From 57a1f8931e1b01618acd4d3242f6a1f2f39d5201 Mon Sep 17 00:00:00 2001
From: Todd Dembrey <todd.dembrey@torchbox.com>
Date: Fri, 31 Aug 2018 15:49:24 +0100
Subject: [PATCH] Handle the files being imported into the application for
historic data
---
.../management/commands/migration_base.py | 41 ++++++++++++-
opentech/apply/funds/models/submissions.py | 57 ++++++++++---------
opentech/settings/base.py | 6 ++
3 files changed, 75 insertions(+), 29 deletions(-)
diff --git a/opentech/apply/funds/management/commands/migration_base.py b/opentech/apply/funds/management/commands/migration_base.py
index c1cfd452d..f68846698 100644
--- a/opentech/apply/funds/management/commands/migration_base.py
+++ b/opentech/apply/funds/management/commands/migration_base.py
@@ -3,11 +3,13 @@ import json
from datetime import datetime, timezone
+from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.utils import IntegrityError
from django_fsm import FSMField
+from storages.backends.s3boto3 import S3Boto3Storage
from opentech.apply.categories.models import Category, Option
from opentech.apply.categories.categories_seed import CATEGORIES
@@ -16,6 +18,28 @@ from opentech.apply.funds.models.forms import RoundBaseForm, LabBaseForm
from opentech.apply.funds.workflow import INITIAL_STATE
+class MigrationStorage(S3Boto3Storage):
+ if hasattr(settings, 'AWS_MIGRATION_BUCKET_NAME'):
+ bucket_name = settings.AWS_MIGRATION_BUCKET_NAME
+
+ if hasattr(settings, 'AWS_MIGRATION_ACCESS_KEY_ID'):
+ access_key = settings.AWS_MIGRATION_ACCESS_KEY_ID
+
+ if hasattr(settings, 'AWS_MIGRATION_SECRET_ACCESS_KEY_ID'):
+ secret_key = settings.AWS_MIGRATION_SECRET_ACCESS_KEY
+
+ bucket_acl = 'private'
+ custom_domain = False
+ default_acl = 'private'
+ encryption = True
+ file_overwrite = False
+ querystring_auth = True
+ url_protocol = 'https:'
+
+
+migration_storage = MigrationStorage()
+
+
class MigrateCommand(BaseCommand):
help = "Application migration script. Requires a source JSON file."
data = []
@@ -186,8 +210,7 @@ class MigrateCommand(BaseCommand):
if option:
value.append(option)
elif mapping_type == 'file':
- # TODO finish mapping. Requires access to the files.
- value = {}
+ value = self.process_file(source_value)
return value
@@ -260,3 +283,17 @@ class MigrateCommand(BaseCommand):
def nl2br(self, value):
return value.replace('\r\n', '<br>\n')
+
+ def process_file(self, value):
+ if isinstance(value, dict):
+ value = [value]
+
+ files = []
+
+ for file_data in value:
+ file_path = 'files/private/application_upload/' + file_data['filename']
+ saved_file = migration_storage.open(file_path)
+ saved_file.name = file_data['filename']
+ files.append(saved_file)
+
+ return files
diff --git a/opentech/apply/funds/models/submissions.py b/opentech/apply/funds/models/submissions.py
index 11f745dee..96772bc12 100644
--- a/opentech/apply/funds/models/submissions.py
+++ b/opentech/apply/funds/models/submissions.py
@@ -42,6 +42,35 @@ from ..workflow import (
submission_storage = get_storage_class(getattr(settings, 'PRIVATE_FILE_STORAGE', None))()
+def save_path(file_name, folder):
+ file_path = os.path.join(folder, file_name)
+ return submission_storage.generate_filename(file_path)
+
+
+def handle_file(file, folder):
+ # File is potentially optional
+ if file:
+ try:
+ filename = save_path(file.name, folder)
+ except AttributeError:
+ # file is not changed, it is still the dictionary
+ return file
+
+ saved_name = submission_storage.save(filename, file)
+ return {
+ 'name': file.name,
+ 'path': saved_name,
+ 'url': submission_storage.url(saved_name),
+ }
+
+
+def handle_files(files, folder):
+ if isinstance(files, list):
+ return [handle_file(file, folder) for file in files]
+
+ return handle_file(files, folder)
+
+
class JSONOrderable(models.QuerySet):
json_field = ''
@@ -388,32 +417,6 @@ class ApplicationSubmission(
defaults={'full_name': full_name}
)
- def save_path(self, file_name):
- file_path = os.path.join('submissions', 'user', str(self.user.id), file_name)
- return submission_storage.generate_filename(file_path)
-
- def handle_file(self, file):
- # File is potentially optional
- if file:
- try:
- filename = self.save_path(file.name)
- except AttributeError:
- # file is not changed, it is still the dictionary
- return file
-
- saved_name = submission_storage.save(filename, file)
- return {
- 'name': file.name,
- 'path': saved_name,
- 'url': submission_storage.url(saved_name),
- }
-
- def handle_files(self, files):
- if isinstance(files, list):
- return [self.handle_file(file) for file in files]
-
- return self.handle_file(files)
-
def get_from_parent(self, attribute):
try:
@@ -483,7 +486,7 @@ class ApplicationSubmission(
for field in self.form_fields:
if isinstance(field.block, UploadableMediaBlock):
file = self.form_data.get(field.id, {})
- self.form_data[field.id] = self.handle_files(file)
+ self.form_data[field.id] = handle_files(file, os.path.join('submission', str(self.id), field.id))
def save(self, *args, **kwargs):
if self.is_draft:
diff --git a/opentech/settings/base.py b/opentech/settings/base.py
index edaaa1b23..6f6972050 100644
--- a/opentech/settings/base.py
+++ b/opentech/settings/base.py
@@ -466,6 +466,12 @@ if 'AWS_STORAGE_BUCKET_NAME' in env:
)
+# Settings to connect to the Bucket from which we are migrating data
+AWS_MIGRATION_BUCKET_NAME = env.get('AWS_MIGRATION_BUCKET_NAME', '')
+AWS_MIGRATION_ACCESS_KEY_ID = env.get('AWS_MIGRATION_ACCESS_KEY_ID', '')
+AWS_MIGRATION_SECRET_ACCESS_KEY = env.get('AWS_MIGRATION_SECRET_ACCESS_KEY', '')
+
+
MAILCHIMP_API_KEY = env.get('MAILCHIMP_API_KEY')
MAILCHIMP_LIST_ID = env.get('MAILCHIMP_LIST_ID')
--
GitLab