import argparse import json import os from urllib.parse import urlsplit from datetime import datetime, timezone from django.conf import settings from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.db import transaction from django.db.utils import IntegrityError from django_fsm import FSMField from storages.backends.s3boto3 import S3Boto3Storage from opentech.apply.categories.models import Category, Option from opentech.apply.categories.categories_seed import CATEGORIES from opentech.apply.funds.models import ApplicationSubmission, FundType, Round, LabType from opentech.apply.funds.models.forms import RoundBaseForm, LabBaseForm from opentech.apply.funds.workflow import INITIAL_STATE class MigrationStorage(S3Boto3Storage): if hasattr(settings, 'AWS_MIGRATION_BUCKET_NAME'): bucket_name = settings.AWS_MIGRATION_BUCKET_NAME if hasattr(settings, 'AWS_MIGRATION_ACCESS_KEY_ID'): access_key = settings.AWS_MIGRATION_ACCESS_KEY_ID if hasattr(settings, 'AWS_MIGRATION_SECRET_ACCESS_KEY'): secret_key = settings.AWS_MIGRATION_SECRET_ACCESS_KEY bucket_acl = 'private' custom_domain = False default_acl = 'private' encryption = True file_overwrite = False querystring_auth = True url_protocol = 'https:' migration_storage = MigrationStorage() class MigrateCommand(BaseCommand): help = "Application migration script. Requires a source JSON file." data = [] terms = {} # Monkey patch the status field so it is no longer protected patched_status_field = FSMField(default=INITIAL_STATE, protected=False) setattr(ApplicationSubmission, 'status', patched_status_field) def add_arguments(self, parser): parser.add_argument('source', type=argparse.FileType('r'), help='Migration source JSON file') @transaction.atomic def handle(self, *args, **options): # Prepare the list of categories. for item in CATEGORIES: category, _ = Category.objects.get_or_create(name=item['category']) option, _ = Option.objects.get_or_create(value=item['name'], category=category) self.terms[item['tid']] = option with options['source'] as json_data: self.data = json.load(json_data) counter = 0 for id in self.data: self.process(id) counter += 1 self.stdout.write(f"Imported {counter} submissions.") def process(self, id): node = self.data[id] try: submission = ApplicationSubmission.objects.get(drupal_id=node['nid']) except ApplicationSubmission.DoesNotExist: submission = ApplicationSubmission(drupal_id=node['nid']) # Disable auto_* on date fields so imported dates are used. for field in submission._meta.local_fields: if field.name == "submit_time": field.auto_now_add = False # TODO timezone? submission.submit_time = datetime.fromtimestamp(int(node['created']), timezone.utc) submission.user = self.get_user(node['uid']) if self.CONTENT_TYPE == "fund": FUND = FundType.objects.get(title=self.FUND_NAME) submission.page = FUND ROUND = Round.objects.get(title=self.ROUND_NAME) submission.round = ROUND if self.APPLICATION_TYPE == "request": FORM = RoundBaseForm.objects.get(round=ROUND) elif self.APPLICATION_TYPE == "concept": FORM = RoundBaseForm.objects.filter(round=ROUND)[0] elif self.APPLICATION_TYPE == "proposal": FORM = RoundBaseForm.objects.filter(round=ROUND)[1] submission.form_fields = FORM.form.form_fields elif self.CONTENT_TYPE == "lab": LAB = LabType.objects.get(title=self.LAB_NAME) submission.page = LAB FORM = LabBaseForm.objects.get(lab=LAB) submission.form_fields = FORM.form.form_fields submission.status = self.get_workflow_state(node) if 'proposal_nid' in node: try: submission.next = ApplicationSubmission.objects.get(drupal_id=node['proposal_nid']) except ApplicationSubmission.DoesNotExist: self.stdout.write("No related proposal found, please import proposals before applications.") pass form_data = { 'skip_account_creation_notification': True, } for field in node: if field in self.STREAMFIELD_MAP: try: id = self.STREAMFIELD_MAP[field]['id'] form_data[id] = self.get_field_value(field, node) except TypeError: pass if "value" not in form_data or not form_data["value"]: form_data["value"] = 0 if "duration" not in form_data or not form_data["duration"]: form_data["duration"] = "1" if "email" not in form_data or not form_data["email"]: if hasattr(submission.user, 'email'): form_data["email"] = submission.user.email else: form_data["email"] = f"user+{node['uid']}@example.com" submission.form_data = form_data try: submission.save() self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})") except IntegrityError: self.stdout.write(f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError") def get_user(self, uid): try: User = get_user_model() return User.objects.get(drupal_id=uid) except User.DoesNotExist: return None def get_field_value(self, field, node): """ Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},] """ mapping = self.STREAMFIELD_MAP[field] mapping_type = mapping['type'] key = mapping.get('key', 'value') source_value = node[field] value = None if mapping_type == "direct": value = source_value elif mapping_type == 'value': if key in source_value: value = self.nl2br(source_value[key]) if source_value else '' else: value = self.nl2br(source_value['value']) if source_value else '' elif mapping_type == 'merge_value': values = [] i = 0 for item in source_value: question = self.REQUEST_QUESTION_MAP[node['field_application_request']['target_id']] values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n") i += 1 merged_values = ''.join(values) value = self.nl2br(merged_values) if source_value else '' elif mapping_type == 'map' and 'map' in 'mapping': value = mapping['map'].get(source_value[key]) elif mapping_type == 'address' and 'map' in mapping: try: value_map = mapping['map'] value = {} for item in value_map: value[value_map[item]] = source_value[item] value = json.dumps(value) except TypeError: value = {} elif mapping_type == 'boolean': value = source_value[key] == '1' if source_value else False elif mapping_type == 'category': if not source_value: value = [] else: if isinstance(source_value, dict): option = self.get_referenced_term(source_value[key]) value = [option] if option else [] else: value = [] for item in source_value: option = self.get_referenced_term(item[key]) if option: value.append(option) elif mapping_type == 'file': value = self.process_file(source_value) return value def get_referenced_term(self, tid): try: term = self.terms[tid] return term.id except KeyError: return None def get_referenced_node(self, nid): pass def get_workflow_state(self, node): """ workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}} """ states_request = { "draft": "in_discussion", "published": "in_discussion", "in_discussion": "in_discussion", "council_review": "internal_review", "ready_for_reply": "post_review_discussion", "contract_review": "post_review_discussion", "in_contract": "accepted", "invited_for_proposal": "accepted", "dropped_concept_note": "rejected", "dropped": "rejected", "dropped_without_review": "rejected" } states_concept = { "draft": "in_discussion", "published": "in_discussion", "in_discussion": "in_discussion", "council_review": "concept_internal_review", "ready_for_reply": "concept_review_discussion", "contract_review": "concept_review_discussion", "in_contract": "invited_to_proposal", "invited_for_proposal": "invited_to_proposal", "dropped_concept_note": "concept_rejected", "dropped": "concept_rejected", "dropped_without_review": "concept_rejected" } states_proposal = { "draft": "draft_proposal", "published": "proposal_discussion", "in_discussion": "proposal_discussion", "council_review": "external_review", "ready_for_reply": "proposal_more_info", "contract_review": "post_external_review_discussion", "in_contract": "proposal_accepted", "invited_for_proposal": "proposal_accepted", "dropped_concept_note": "proposal_rejected", "dropped": "proposal_rejected", "dropped_without_review": "proposal_rejected" } if self.APPLICATION_TYPE == "request": workflow_state = states_request.get(node['workbench_moderation']['current']['state'], "in_discussion") elif self.APPLICATION_TYPE == "concept": workflow_state = states_concept.get(node['workbench_moderation']['current']['state'], "in_discussion") elif self.APPLICATION_TYPE == "proposal": workflow_state = states_proposal.get(node['workbench_moderation']['current']['state'], "draft_proposal") else: workflow_state = None return workflow_state def nl2br(self, value): return value.replace('\r\n', '<br>\n') def process_file(self, value): if isinstance(value, dict): value = [value] files = [] for file_data in value: parts = urlsplit(file_data['uri']) file_path = os.path.join('files', 'private', parts.netloc, *parts.path.split('/')) saved_file = migration_storage.open(file_path) saved_file.name = file_data['filename'] files.append(saved_file) return files