diff --git a/opentech/apply/funds/management/commands/migration_base.py b/opentech/apply/funds/management/commands/migration_base.py
index c1cfd452d61844abec1967169f03ca2d57e16ffe..77c2b73cce41807ada8ccab96017d6bd83ba62a7 100644
--- a/opentech/apply/funds/management/commands/migration_base.py
+++ b/opentech/apply/funds/management/commands/migration_base.py
@@ -1,13 +1,17 @@
import argparse
import json
+import os
+from urllib.parse import urlsplit
from datetime import datetime, timezone
+from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.utils import IntegrityError
from django_fsm import FSMField
+from storages.backends.s3boto3 import S3Boto3Storage
from opentech.apply.categories.models import Category, Option
from opentech.apply.categories.categories_seed import CATEGORIES
@@ -16,6 +20,28 @@ from opentech.apply.funds.models.forms import RoundBaseForm, LabBaseForm
from opentech.apply.funds.workflow import INITIAL_STATE
+class MigrationStorage(S3Boto3Storage):
+ if hasattr(settings, 'AWS_MIGRATION_BUCKET_NAME'):
+ bucket_name = settings.AWS_MIGRATION_BUCKET_NAME
+
+ if hasattr(settings, 'AWS_MIGRATION_ACCESS_KEY_ID'):
+ access_key = settings.AWS_MIGRATION_ACCESS_KEY_ID
+
+ if hasattr(settings, 'AWS_MIGRATION_SECRET_ACCESS_KEY_ID'):
+ secret_key = settings.AWS_MIGRATION_SECRET_ACCESS_KEY
+
+ bucket_acl = 'private'
+ custom_domain = False
+ default_acl = 'private'
+ encryption = True
+ file_overwrite = False
+ querystring_auth = True
+ url_protocol = 'https:'
+
+
+migration_storage = MigrationStorage()
+
+
class MigrateCommand(BaseCommand):
help = "Application migration script. Requires a source JSON file."
data = []
@@ -119,7 +145,6 @@ class MigrateCommand(BaseCommand):
self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
except IntegrityError:
self.stdout.write(f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError")
- pass
def get_user(self, uid):
try:
@@ -186,8 +211,7 @@ class MigrateCommand(BaseCommand):
if option:
value.append(option)
elif mapping_type == 'file':
- # TODO finish mapping. Requires access to the files.
- value = {}
+ value = self.process_file(source_value)
return value
@@ -260,3 +284,18 @@ class MigrateCommand(BaseCommand):
def nl2br(self, value):
return value.replace('\r\n', '<br>\n')
+
+ def process_file(self, value):
+ if isinstance(value, dict):
+ value = [value]
+
+ files = []
+
+ for file_data in value:
+ parts = urlsplit(file_data['uri'])
+ file_path = os.path.join('files', 'private', parts.netloc, *parts.path.split('/'))
+ saved_file = migration_storage.open(file_path)
+ saved_file.name = file_data['filename']
+ files.append(saved_file)
+
+ return files
diff --git a/opentech/apply/funds/models/submissions.py b/opentech/apply/funds/models/submissions.py
index 11f745deef22ce452e663f39309bf013b5504e25..707d8409f27c46b19c78b3f53fbfc0288e02ab91 100644
--- a/opentech/apply/funds/models/submissions.py
+++ b/opentech/apply/funds/models/submissions.py
@@ -42,6 +42,35 @@ from ..workflow import (
submission_storage = get_storage_class(getattr(settings, 'PRIVATE_FILE_STORAGE', None))()
+def save_path(file_name, folder):
+ file_path = os.path.join(folder, file_name)
+ return submission_storage.generate_filename(file_path)
+
+
+def handle_file(file, folder):
+ # File is potentially optional
+ if file:
+ try:
+ filename = save_path(file.name, folder)
+ except AttributeError:
+ # file is not changed, it is still the dictionary
+ return file
+
+ saved_name = submission_storage.save(filename, file)
+ return {
+ 'name': file.name,
+ 'path': saved_name,
+ 'url': submission_storage.url(saved_name),
+ }
+
+
+def handle_files(files, folder):
+ if isinstance(files, list):
+ return [handle_file(file, folder) for file in files]
+
+ return handle_file(files, folder)
+
+
class JSONOrderable(models.QuerySet):
json_field = ''
@@ -388,32 +417,6 @@ class ApplicationSubmission(
defaults={'full_name': full_name}
)
- def save_path(self, file_name):
- file_path = os.path.join('submissions', 'user', str(self.user.id), file_name)
- return submission_storage.generate_filename(file_path)
-
- def handle_file(self, file):
- # File is potentially optional
- if file:
- try:
- filename = self.save_path(file.name)
- except AttributeError:
- # file is not changed, it is still the dictionary
- return file
-
- saved_name = submission_storage.save(filename, file)
- return {
- 'name': file.name,
- 'path': saved_name,
- 'url': submission_storage.url(saved_name),
- }
-
- def handle_files(self, files):
- if isinstance(files, list):
- return [self.handle_file(file) for file in files]
-
- return self.handle_file(files)
-
def get_from_parent(self, attribute):
try:
@@ -471,7 +474,7 @@ class ApplicationSubmission(
def clean_submission(self):
self.process_form_data()
self.ensure_user_has_account()
- self.process_file_data()
+ self.process_file_data(self.form_data)
def process_form_data(self):
for field_name, field_id in self.must_include.items():
@@ -479,31 +482,43 @@ class ApplicationSubmission(
if response:
self.form_data[field_name] = response
- def process_file_data(self):
+ def extract_files(self):
+ files = {}
+ for field in self.form_fields:
+ if isinstance(field.block, UploadableMediaBlock):
+ files[field.id] = self.form_data.pop(field.id, {})
+ return files
+
+ def process_file_data(self, data):
for field in self.form_fields:
if isinstance(field.block, UploadableMediaBlock):
- file = self.form_data.get(field.id, {})
- self.form_data[field.id] = self.handle_files(file)
+ file = data.get(field.id, {})
+ self.form_data[field.id] = handle_files(file, os.path.join('submission', str(self.id), field.id))
def save(self, *args, **kwargs):
if self.is_draft:
raise ValueError('Cannot save with draft data')
- self.clean_submission()
-
creating = not self.id
+
if creating:
# We are creating the object default to first stage
self.workflow_name = self.get_from_parent('workflow_name')
# Copy extra relevant information to the child
self.lead = self.get_from_parent('lead')
+ # We need the submission id to correctly save the files
+ files = self.extract_files()
+
+ self.clean_submission()
+
# add a denormed version of the answer for searching
self.search_data = ' '.join(self.prepare_search_values())
super().save(*args, **kwargs)
if creating:
+ self.process_file_data(files)
self.reviewers.set(self.get_from_parent('reviewers').all())
first_revision = ApplicationRevision.objects.create(
submission=self,
diff --git a/opentech/apply/funds/tests/test_models.py b/opentech/apply/funds/tests/test_models.py
index 28842606bac340e3f2aa28d447d4e0e1b0e1227f..f93be582e03f8bdbb523c101f7e2198f5a81d16b 100644
--- a/opentech/apply/funds/tests/test_models.py
+++ b/opentech/apply/funds/tests/test_models.py
@@ -371,8 +371,17 @@ class TestApplicationSubmission(TestCase):
def test_file_gets_uploaded(self):
filename = 'file_name.png'
submission = self.make_submission(form_data__image__filename=filename)
- save_path = os.path.join(settings.MEDIA_ROOT, submission.save_path(filename))
- self.assertTrue(os.path.isfile(save_path))
+ path = os.path.join(settings.MEDIA_ROOT, 'submission', str(submission.id))
+
+ # Check we created the top level folder
+ self.assertTrue(os.path.isdir(path))
+
+ found_files = []
+ for _, _, files in os.walk(path):
+ found_files.extend(files)
+
+ # Check we saved the file somewhere beneath it
+ self.assertIn(filename, found_files)
def test_create_revision_on_create(self):
submission = ApplicationSubmissionFactory()
diff --git a/opentech/settings/base.py b/opentech/settings/base.py
index edaaa1b23ce6bbe88437c3cad7057351142fd0b5..6f69720503e51617b7f9ec97f539129301d8cbe1 100644
--- a/opentech/settings/base.py
+++ b/opentech/settings/base.py
@@ -466,6 +466,12 @@ if 'AWS_STORAGE_BUCKET_NAME' in env:
)
+# Settings to connect to the Bucket from which we are migrating data
+AWS_MIGRATION_BUCKET_NAME = env.get('AWS_MIGRATION_BUCKET_NAME', '')
+AWS_MIGRATION_ACCESS_KEY_ID = env.get('AWS_MIGRATION_ACCESS_KEY_ID', '')
+AWS_MIGRATION_SECRET_ACCESS_KEY = env.get('AWS_MIGRATION_SECRET_ACCESS_KEY', '')
+
+
MAILCHIMP_API_KEY = env.get('MAILCHIMP_API_KEY')
MAILCHIMP_LIST_ID = env.get('MAILCHIMP_LIST_ID')