From ee0d81c54e5979332d40840a5d5a1f6f40922a12 Mon Sep 17 00:00:00 2001 From: Fredrik Jonsson <frjo@xdeb.org> Date: Wed, 25 Jul 2018 18:19:44 +0200 Subject: [PATCH] Move common migration functions in to a MicrateCommand class. --- .../commands/migrate_rr_applications.py | 584 ++++++------------ .../management/commands/migration_base.py | 215 +++++++ 2 files changed, 421 insertions(+), 378 deletions(-) create mode 100644 opentech/apply/funds/management/commands/migration_base.py diff --git a/opentech/apply/funds/management/commands/migrate_rr_applications.py b/opentech/apply/funds/management/commands/migrate_rr_applications.py index d3262df6e..145c79eb3 100644 --- a/opentech/apply/funds/management/commands/migrate_rr_applications.py +++ b/opentech/apply/funds/management/commands/migrate_rr_applications.py @@ -1,381 +1,209 @@ -import argparse -import json +from opentech.apply.funds.management.commands.migration_base import MigrateCommand -from datetime import datetime, timezone -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from django.db import transaction -from django.db.utils import IntegrityError -from django_fsm import FSMField +class Command(MigrateCommand): + FUND_NAME = "Rapid Response" + ROUND_NAME = "Rapid Response open round" + APPLICATION_TYPE = "request" -from opentech.apply.categories.models import Category, Option -from opentech.apply.categories.categories_seed import CATEGORIES -from opentech.apply.funds.models import ApplicationSubmission, FundType, Round, RoundForm -from opentech.apply.funds.workflow import INITIAL_STATE - -User = get_user_model() - -STREAMFIELD_MAP = { - "title": { - "id": "title", - "type": "direct", - }, - "field_application_name": { - "id": "full_name", - "type": "value", - # If no Drupal value key is specified, we default to 'value' - "key": "safe_value", - }, - "field_application_mail": { - "id": "email", - "type": "value", - "key": "email", - }, - "field_application_preapplied": { - "id": "49a0f5f4-e8e9-4dfc-8575-205ee9675032", - "type": "map", - "map": { - "0": "No", - "1": "Yes", - }, - }, - "field_application_apply_for": { - "id": "c1277029-1718-40e3-8bf5-d80ece7fd343", - "type": "map", - "map": { - "direct": "Direct funding", - "receive": "Requesting to receive services", - "provide": "Requesting to provide services", - }, - }, - "field_application_amount": { - "id": "value", - "type": "value", - }, - "field_application_amount_text": { - "id": "value", - "type": "value", - }, - "field_application_service": { - "id": "ebdf9a22-58c7-4bd6-a58d-e71363357470", - "type": "map", - "map": { - "audit": "Audit of presumably compromised websites", - "ddos": "DDoS response and mitigation", - "hosting": "Secure web hosting", - "hostingevents": "Secure hosting for monitoring and resiliency of websites during special events (elections, campaigns etc.)", - "vpn": "VPN connections", - "isp": "Safe internet connections", - "analysis": "Forensic analysis of digital attacks", - "recovery": "Recovery of compromised websites", - "malware": "Malware analysis", - "equipment": "Equipment replacements (unavailable)", - "legalhelp": "Finding legal representation (unavailable)", - "legalfees": "Payment of legal fees (unavailable)", - }, - }, - "field_application_service_other": { - "id": "c8c329c7-78e4-4cbf-a3b1-77a1324e92ff", - "type": "value", - "key": "safe_value", - }, - "field_application_duration3": { - "id": "806d401c-b52c-46f7-9e96-f36fa70f53d8", - "type": "map", - "map": { - "1": "1 month", - "2": "2 months", - "3": "3 months", - "4": "4 months", - "5": "5 months", - "6": "6 months", - }, - }, - "field_application_who": { - "id": "1ec16cdc-7a68-40be-b17b-9a218def4260", - "type": "value", - "key": "safe_value", - }, - "field_application_how": { - "id": "4fa2ac11-d1cd-4d23-8082-93a14c8f99c8", - "type": "value", - "key": "safe_value", - }, - "field_application_sustainability": { - "id": "3cde39ae-b687-4c4f-b58b-849396c2fdb8", - "type": "value", - "key": "safe_value", - }, - "field_application_dates": { - "id": "0b2a4653-b390-44a6-b92e-fae4647e7ec4", - "type": "value", - "key": "safe_value", - }, - "field_application_why": { - "id": "6d75e412-cf53-4833-9f1d-3e0126512fb9", - "type": "value", - "key": "safe_value", - }, - "field_application_why_rapiid": { - "id": "1b181d1e-ef91-41af-b9c1-d096a991314b", - "type": "value", - "key": "safe_value", - }, - "field_application_focus": { - "id": "efd91eaf-378f-4aab-96cb-c5601155cbee", - "type": "category", - "key": "tid", - }, - "field_application_objectives": { - "id": "4be0c7bd-231d-4d9f-bd47-8589fc005f54", - "type": "category", - "key": "tid", - }, - "field_application_beneficiaries": { - "id": "6e0293ee-218e-4c3b-b82d-5bf91fdb21c9", - "type": "category", - "key": "tid", - }, - "field_term_region": { - "id": "6ff029c6-c6d1-4c37-a49a-46181b1cd33d", - "type": "category", - "key": "tid", - }, - "field_application_problems": { - "id": "7fb1001e-d458-414f-a5bb-006db6f89baf", - "type": "category", - "key": "tid", - }, - "field_application_budget": { - "id": "45d7d38a-9c9d-4c43-98df-bb95d4a1dd77", - "type": "value", - "key": "safe_value", - }, - "field_application_legal_name": { - "id": "632065c5-860f-4751-9b31-52914d7c6448", - "type": "value", - "key": "safe_value", - }, - "field_application_contact": { - "id": "13bb0d64-65f3-4340-8e7e-e5da80d706d5", - "type": "value", - "key": "safe_value", - }, - "field_application_phone": { - "id": "2cb9fe4b-df45-4181-80e5-14382f853081", - "type": "value", - "key": "safe_value", - }, - "field_application_address": { - "id": "bd29eb88-9754-4305-9b2d-406a875ec56a", - "type": "address", - "map": { - "administrative_area": "administrative_area", - "country": "country", - "locality": "locality_name", - "postal_code": "postal_code", - "thoroughfare": "thoroughfare", - "premise": "premise", - } - }, - "field_application_otf_mission": { - "id": "e695f0d7-4c74-4cc6-853f-bd62ecd19d3d", - "type": "boolean", - }, - "field_application_otf_tos": { - "id": "f40d1acc-d802-4cc6-b0e9-fff78dc54223", - "type": "boolean", - }, - "field_application_otf_represent": { - "id": "0b3c0827-38e2-439b-bca5-735835af1019", - "type": "boolean", - }, - "field_application_otf_license": { - "id": "bc9c960e-a6f4-4bc2-b626-efb5bc5552c6", - "type": "boolean", - }, - "field_application_otf_complete": { - "id": "5812b66d-630e-4ca2-8bea-819084278f55", - "type": "boolean", - }, - "field_application_otf_deadline": { - "id": "97d3746c-cf0f-449a-b3a3-7a9cdd45cc6d", - "type": "boolean", - }, - "field_application_otf_list": { - "id": "fc3d2a87-1151-418b-b1cd-9289f00bde35", - "type": "boolean", - }, - "field_application_otf_newsletter": { - "id": "83ecc69a-f47c-495e-bc8f-326e55aed67a", - "type": "boolean", - }, - "field_concept_upload": { - "id": "607daeba-1f33-4ad0-b135-eda743ba8e3a", - "type": "file", - # TODO: finish mapping - }, -} - -FUND = FundType.objects.get(title='Rapid Response') -ROUND = Round.objects.get(title='Rapid Response open round') -FORM = RoundForm.objects.get(round=ROUND) - -# Monkey patch the status field so it is no longer protected -patched_status_field = FSMField(default=INITIAL_STATE, protected=False) -setattr(ApplicationSubmission, 'status', patched_status_field) - - -class Command(BaseCommand): - help = "Rapid response migration script. Requires a source JSON file." - data = [] - terms = {} - - def add_arguments(self, parser): - parser.add_argument('source', type=argparse.FileType('r'), help='Migration source JSON file') - - @transaction.atomic - def handle(self, *args, **options): - # Prepare the list of categories. - for item in CATEGORIES: - category, _ = Category.objects.get_or_create(name=item['category']) - option, _ = Option.objects.get_or_create(value=item['name'], category=category) - self.terms[item['tid']] = option - - with options['source'] as json_data: - self.data = json.load(json_data) - - for id in self.data: - self.process(id) - - def process(self, id): - node = self.data[id] - - try: - submission = ApplicationSubmission.objects.get(drupal_id=node['nid']) - except ApplicationSubmission.DoesNotExist: - submission = ApplicationSubmission(drupal_id=node['nid']) - - # TODO timezone? - submission.submit_time = datetime.fromtimestamp(int(node['created']), timezone.utc) - submission.user = self.get_user(node['uid']) - - submission.page = FUND - submission.round = ROUND - submission.form_fields = FORM.form.form_fields - - submission.status = self.get_workflow_state(node) - - form_data = { - 'skip_account_creation_notification': True, - } - - # Only allow one of application amount or application amount text - if not node['field_application_amount']: - node.pop('field_application_amount', None) - - for field in node: - if field in STREAMFIELD_MAP: - try: - id = STREAMFIELD_MAP[field]['id'] - form_data[id] = self.get_field_value(field, node) - except TypeError: - pass - - submission.form_data = form_data - - try: - submission.save() - self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})") - except IntegrityError: - pass - - def get_user(self, uid): - try: - return User.objects.get(drupal_id=uid) - finally: - return None - - def get_field_value(self, field, node): - """ - Handles the following formats: - field: {(safe_)value: VALUE} - field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. - field: {tid: ID} -- or term ID. fk to Categories - field: [] - field: [{value|target_id|tid: VALUE},] - """ - mapping = STREAMFIELD_MAP[field] - mapping_type = mapping['type'] - key = mapping.get('key', 'value') - source_value = node[field] - value = None - - if mapping_type == "direct": - value = source_value - elif mapping_type == 'value': - value = self.nl2br(source_value[key]) if source_value else '' - elif mapping_type == 'map' and 'map' in 'mapping': - value = mapping['map'].get(source_value[key]) - elif mapping_type == 'address' and 'map' in mapping: - try: - value_map = mapping['map'] - value = {} - for item in value_map: - value[value_map[item]] = source_value[item] - except TypeError: - value = {} - elif mapping_type == 'boolean': - value = source_value[key] == '1' if source_value else False - elif mapping_type == 'category': - if not source_value: - value = [] - else: - if isinstance(source_value, dict): - option = self.get_referenced_term(source_value[key]) - value = [option] if option else [] - else: - value = [] - for item in source_value: - option = self.get_referenced_term(item[key]) - if option: - value.append(option) - elif mapping_type == 'file': - # TODO finish mapping. Requires access to the files. - value = {} - - return value - - def get_referenced_term(self, tid): - try: - term = self.terms[tid] - return term.id - except KeyError: - return None - - def get_referenced_node(self, nid): - pass - - def get_workflow_state(self, node): - """ - workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}} - """ - states = { - "draft": "", - "published": "in_discussion", - "in_discussion": "in_discussion", - "council_review": "internal_review", - "ready_for_reply": "post_review_discussion", - "contract_review": "post_review_discussion", - "in_contract": "accepted", - "invited_for_proposal": "accepted", - "dropped_concept_note": "rejected", - "dropped": "rejected", - "dropped_without_review": "rejected" - } - - return states.get(node['workbench_moderation']['current']['state'], "in_discussion") - - def nl2br(self, value): - return value.replace('\r\n', '<br>\n') + STREAMFIELD_MAP = { + "title": { + "id": "title", + "type": "direct", + }, + "field_application_name": { + "id": "full_name", + "type": "value", + # If no Drupal value key is specified, we default to 'value' + "key": "safe_value", + }, + "field_application_mail": { + "id": "email", + "type": "value", + "key": "email", + }, + "field_application_preapplied": { + "id": "49a0f5f4-e8e9-4dfc-8575-205ee9675032", + "type": "map", + "map": { + "0": "No", + "1": "Yes", + }, + }, + "field_application_apply_for": { + "id": "c1277029-1718-40e3-8bf5-d80ece7fd343", + "type": "map", + "map": { + "direct": "Direct funding", + "receive": "Requesting to receive services", + "provide": "Requesting to provide services", + }, + }, + "field_application_amount": { + "id": "value", + "type": "value", + }, + "field_application_amount_text": { + "id": "value", + "type": "value", + }, + "field_application_service": { + "id": "ebdf9a22-58c7-4bd6-a58d-e71363357470", + "type": "map", + "map": { + "audit": "Audit of presumably compromised websites", + "ddos": "DDoS response and mitigation", + "hosting": "Secure web hosting", + "hostingevents": "Secure hosting for monitoring and resiliency of websites during special events (elections, campaigns etc.)", + "vpn": "VPN connections", + "isp": "Safe internet connections", + "analysis": "Forensic analysis of digital attacks", + "recovery": "Recovery of compromised websites", + "malware": "Malware analysis", + "equipment": "Equipment replacements (unavailable)", + "legalhelp": "Finding legal representation (unavailable)", + "legalfees": "Payment of legal fees (unavailable)", + }, + }, + "field_application_service_other": { + "id": "c8c329c7-78e4-4cbf-a3b1-77a1324e92ff", + "type": "value", + "key": "safe_value", + }, + "field_application_duration3": { + "id": "806d401c-b52c-46f7-9e96-f36fa70f53d8", + "type": "map", + "map": { + "1": "1 month", + "2": "2 months", + "3": "3 months", + "4": "4 months", + "5": "5 months", + "6": "6 months", + }, + }, + "field_application_who": { + "id": "1ec16cdc-7a68-40be-b17b-9a218def4260", + "type": "value", + "key": "safe_value", + }, + "field_application_how": { + "id": "4fa2ac11-d1cd-4d23-8082-93a14c8f99c8", + "type": "value", + "key": "safe_value", + }, + "field_application_sustainability": { + "id": "3cde39ae-b687-4c4f-b58b-849396c2fdb8", + "type": "value", + "key": "safe_value", + }, + "field_application_dates": { + "id": "0b2a4653-b390-44a6-b92e-fae4647e7ec4", + "type": "value", + "key": "safe_value", + }, + "field_application_why": { + "id": "6d75e412-cf53-4833-9f1d-3e0126512fb9", + "type": "value", + "key": "safe_value", + }, + "field_application_why_rapiid": { + "id": "1b181d1e-ef91-41af-b9c1-d096a991314b", + "type": "value", + "key": "safe_value", + }, + "field_application_focus": { + "id": "efd91eaf-378f-4aab-96cb-c5601155cbee", + "type": "category", + "key": "tid", + }, + "field_application_objectives": { + "id": "4be0c7bd-231d-4d9f-bd47-8589fc005f54", + "type": "category", + "key": "tid", + }, + "field_application_beneficiaries": { + "id": "6e0293ee-218e-4c3b-b82d-5bf91fdb21c9", + "type": "category", + "key": "tid", + }, + "field_term_region": { + "id": "6ff029c6-c6d1-4c37-a49a-46181b1cd33d", + "type": "category", + "key": "tid", + }, + "field_application_problems": { + "id": "7fb1001e-d458-414f-a5bb-006db6f89baf", + "type": "category", + "key": "tid", + }, + "field_application_budget": { + "id": "45d7d38a-9c9d-4c43-98df-bb95d4a1dd77", + "type": "value", + "key": "safe_value", + }, + "field_application_legal_name": { + "id": "632065c5-860f-4751-9b31-52914d7c6448", + "type": "value", + "key": "safe_value", + }, + "field_application_contact": { + "id": "13bb0d64-65f3-4340-8e7e-e5da80d706d5", + "type": "value", + "key": "safe_value", + }, + "field_application_phone": { + "id": "2cb9fe4b-df45-4181-80e5-14382f853081", + "type": "value", + "key": "safe_value", + }, + "field_application_address": { + "id": "bd29eb88-9754-4305-9b2d-406a875ec56a", + "type": "address", + "map": { + "administrative_area": "administrative_area", + "country": "country", + "locality": "locality_name", + "postal_code": "postal_code", + "thoroughfare": "thoroughfare", + "premise": "premise", + } + }, + "field_application_otf_mission": { + "id": "e695f0d7-4c74-4cc6-853f-bd62ecd19d3d", + "type": "boolean", + }, + "field_application_otf_tos": { + "id": "f40d1acc-d802-4cc6-b0e9-fff78dc54223", + "type": "boolean", + }, + "field_application_otf_represent": { + "id": "0b3c0827-38e2-439b-bca5-735835af1019", + "type": "boolean", + }, + "field_application_otf_license": { + "id": "bc9c960e-a6f4-4bc2-b626-efb5bc5552c6", + "type": "boolean", + }, + "field_application_otf_complete": { + "id": "5812b66d-630e-4ca2-8bea-819084278f55", + "type": "boolean", + }, + "field_application_otf_deadline": { + "id": "97d3746c-cf0f-449a-b3a3-7a9cdd45cc6d", + "type": "boolean", + }, + "field_application_otf_list": { + "id": "fc3d2a87-1151-418b-b1cd-9289f00bde35", + "type": "boolean", + }, + "field_application_otf_newsletter": { + "id": "83ecc69a-f47c-495e-bc8f-326e55aed67a", + "type": "boolean", + }, + "field_concept_upload": { + "id": "607daeba-1f33-4ad0-b135-eda743ba8e3a", + "type": "file", + # TODO: finish mapping + }, + } diff --git a/opentech/apply/funds/management/commands/migration_base.py b/opentech/apply/funds/management/commands/migration_base.py new file mode 100644 index 000000000..d0c1907f4 --- /dev/null +++ b/opentech/apply/funds/management/commands/migration_base.py @@ -0,0 +1,215 @@ +import argparse +import json + +from datetime import datetime, timezone + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand +from django.db import transaction +from django.db.utils import IntegrityError +from django_fsm import FSMField + +from opentech.apply.categories.models import Category, Option +from opentech.apply.categories.categories_seed import CATEGORIES +from opentech.apply.funds.models import ApplicationSubmission, FundType, Round, RoundForm +from opentech.apply.funds.workflow import INITIAL_STATE + + +class MigrateCommand(BaseCommand): + help = "Application migration script. Requires a source JSON file." + data = [] + terms = {} + + # Monkey patch the status field so it is no longer protected + patched_status_field = FSMField(default=INITIAL_STATE, protected=False) + setattr(ApplicationSubmission, 'status', patched_status_field) + + def add_arguments(self, parser): + parser.add_argument('source', type=argparse.FileType('r'), help='Migration source JSON file') + + @transaction.atomic + def handle(self, *args, **options): + # Prepare the list of categories. + for item in CATEGORIES: + category, _ = Category.objects.get_or_create(name=item['category']) + option, _ = Option.objects.get_or_create(value=item['name'], category=category) + self.terms[item['tid']] = option + + with options['source'] as json_data: + self.data = json.load(json_data) + + for id in self.data: + self.process(id) + + def process(self, id): + node = self.data[id] + + FUND = FundType.objects.get(title=self.FUND_NAME) + ROUND = Round.objects.get(title=self.ROUND_NAME) + FORM = RoundForm.objects.get(round=ROUND) + + try: + submission = ApplicationSubmission.objects.get(drupal_id=node['nid']) + except ApplicationSubmission.DoesNotExist: + submission = ApplicationSubmission(drupal_id=node['nid']) + + # TODO timezone? + submission.submit_time = datetime.fromtimestamp(int(node['created']), timezone.utc) + submission.user = self.get_user(node['uid']) + + submission.page = FUND + submission.round = ROUND + submission.form_fields = FORM.form.form_fields + + submission.status = self.get_workflow_state(node) + + form_data = { + 'skip_account_creation_notification': True, + } + + for field in node: + if field in self.STREAMFIELD_MAP: + try: + id = self.STREAMFIELD_MAP[field]['id'] + form_data[id] = self.get_field_value(field, node) + except TypeError: + pass + + if "value" not in form_data: + form_data["value"] = 0 + + submission.form_data = form_data + + try: + submission.save() + self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})") + except IntegrityError: + self.stdout.write(f"Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError") + pass + + def get_user(self, uid): + try: + User = get_user_model() + return User.objects.get(drupal_id=uid) + except User.DoesNotExist: + return None + + def get_field_value(self, field, node): + """ + Handles the following formats: + field: {(safe_)value: VALUE} + field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. + field: {tid: ID} -- or term ID. fk to Categories + field: [] + field: [{value|target_id|tid: VALUE},] + """ + mapping = self.STREAMFIELD_MAP[field] + mapping_type = mapping['type'] + key = mapping.get('key', 'value') + source_value = node[field] + value = None + + if mapping_type == "direct": + value = source_value + elif mapping_type == 'value': + value = self.nl2br(source_value[key]) if source_value else '' + elif mapping_type == 'map' and 'map' in 'mapping': + value = mapping['map'].get(source_value[key]) + elif mapping_type == 'address' and 'map' in mapping: + try: + value_map = mapping['map'] + value = {} + for item in value_map: + value[value_map[item]] = source_value[item] + except TypeError: + value = {} + elif mapping_type == 'boolean': + value = source_value[key] == '1' if source_value else False + elif mapping_type == 'category': + if not source_value: + value = [] + else: + if isinstance(source_value, dict): + option = self.get_referenced_term(source_value[key]) + value = [option] if option else [] + else: + value = [] + for item in source_value: + option = self.get_referenced_term(item[key]) + if option: + value.append(option) + elif mapping_type == 'file': + # TODO finish mapping. Requires access to the files. + value = {} + + return value + + def get_referenced_term(self, tid): + try: + term = self.terms[tid] + return term.id + except KeyError: + return None + + def get_referenced_node(self, nid): + pass + + def get_workflow_state(self, node): + """ + workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}} + """ + states_request = { + "draft": "", + "published": "in_discussion", + "in_discussion": "in_discussion", + "council_review": "internal_review", + "ready_for_reply": "post_review_discussion", + "contract_review": "post_review_discussion", + "in_contract": "accepted", + "invited_for_proposal": "accepted", + "dropped_concept_note": "rejected", + "dropped": "rejected", + "dropped_without_review": "rejected" + } + + states_concept = { + "draft": "", + "published": "in_discussion", + "in_discussion": "in_discussion", + "council_review": "concept_internal_review", + "ready_for_reply": "concept_review_discussion", + "contract_review": "concept_review_discussion", + "in_contract": "invited_to_proposal", + "invited_for_proposal": "invited_to_proposal", + "dropped_concept_note": "concept_rejected", + "dropped": "concept_rejected", + "dropped_without_review": "concept_rejected" + } + + states_proposal = { + "draft": "draft_proposal", + "published": "proposal_discussion", + "in_discussion": "proposal_discussion", + "council_review": "external_review", + "ready_for_reply": "post_external_review_discussion", + "contract_review": "post_external_review_discussion", + "in_contract": "proposal_accepted", + "invited_for_proposal": "proposal_accepted", + "dropped_concept_note": "proposal_rejected", + "dropped": "proposal_rejected", + "dropped_without_review": "proposal_rejected" + } + + if self.APPLICATION_TYPE == "request": + workflow_state = states_request.get(node['workbench_moderation']['current']['state'], "in_discussion") + elif self.APPLICATION_TYPE == "concept": + workflow_state = states_concept.get(node['workbench_moderation']['current']['state'], "in_discussion") + elif self.APPLICATION_TYPE == "proposal": + workflow_state = states_proposal.get(node['workbench_moderation']['current']['state'], "draft_proposal") + else: + workflow_state = None + + return workflow_state + + def nl2br(self, value): + return value.replace('\r\n', '<br>\n') -- GitLab