diff --git a/opentech/apply/funds/management/commands/migrate_fellowship_application.py b/opentech/apply/funds/management/commands/migrate_fellowship_application.py index 42953903435f14e2d491a3883f7d71939a086e2c..f4c94d1348f1c296d3be220e44b40e0fcc5cf068 100644 --- a/opentech/apply/funds/management/commands/migrate_fellowship_application.py +++ b/opentech/apply/funds/management/commands/migrate_fellowship_application.py @@ -1,323 +1,152 @@ -import argparse -import json +from opentech.apply.funds.management.commands.migration_base import MigrateCommand -from datetime import datetime, timezone -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from django.db import transaction -from django.db.utils import IntegrityError -from django_fsm import FSMField +class Command(MigrateCommand): + CONTENT_TYPE = "fund" + FUND_NAME = "Fellowship archive fund" + ROUND_NAME = "Fellowship archive round" + APPLICATION_TYPE = "concept" -from opentech.apply.categories.models import Category, Option -from opentech.apply.categories.categories_seed import CATEGORIES -from opentech.apply.funds.models import ApplicationSubmission, FundType, Round, RoundForm -from opentech.apply.funds.workflow import INITIAL_STATE - -User = get_user_model() - -STREAMFIELD_MAP = { - "title": { - "id": "title", - "type": "direct", - }, - "field_application_name": { - "id": "full_name", - "type": "value", - # If no Drupal value key is specified, we default to 'value' - "key": "safe_value", - }, - "field_application_mail": { - "id": "email", - "type": "value", - "key": "email", - }, - "field_application_position": { - "id": "1282223d-77f5-4047-be03-4df4c4b2148a", - "type": "value", - "key": "safe_value", - }, - "field_application_role": { - "id": "9c0256e4-42e1-41fe-9880-7f621d6c3458", - "type": "value", - "key": "safe_value", - }, - "field_application_preapplied": { - "id": "f8efef0a-0632-4c81-b4db-7bc6a06caa7d", - "type": "map", - "map": { - "0": "No", - "1": "Yes", + STREAMFIELD_MAP = { + "title": { + "id": "title", + "type": "direct", }, - }, - "field_application_describe": { - "id": "1eb8b4e3-e2bb-4810-a8ce-3fc82a3192c8", - "type": "value", - "key": "safe_value", - }, - "field_application_how": { - "id": "177d56e8-2df1-4ead-8e3d-4916610fbed6", - "type": "value", - "key": "safe_value", - }, - "field_application_insight": { - "id": "05ff1755-947b-4e41-8f71-aae99977c572", - "type": "value", - "key": "safe_value", - }, - "field_application_duration2": { - "id": "611dacd7-553a-4be8-9283-1d006099d0c9", - "type": "map", - "map": { - "3": "3 months", - "6": "6 months", - "9": "9 months", - "12": "12 months", - "18": "18 months", + "field_application_name": { + "id": "full_name", + "type": "value", + # If no Drupal value key is specified, we default to 'value' + "key": "safe_value", }, - }, - "field_application_host_text": { - "id": "0afaf4e1-4556-4e79-aa3d-4990e33620da", - "type": "value", - "key": "safe_value", - }, - "field_application_host2_text": { - "id": "a543b34f-ae6a-4b17-8ac3-ececc14573a0", - "type": "value", - "key": "safe_value", - }, - "field_application_questions": { - "id": "57cc52e2-b3ff-4e9f-a5fe-42e7735e16c2", - "type": "value", - "key": "safe_value", - }, - "field_application_status": { - "id": "ff4d12ff-7b88-4e87-bb5b-81543aef0e25", - "type": "category", - "key": "tid", - }, - "field_application_objectives": { - "id": "30c41288-a762-4003-acce-8c12e7343d90", - "type": "category", - "key": "tid", - }, - "field_application_beneficiaries": { - "id": "56833441-542b-4a06-8ad2-8e7e8fd1a334", - "type": "category", - "key": "tid", - }, - "field_application_focus": { - "id": "6b404851-ce2b-494f-b9f7-62858a937469", - "type": "category", - "key": "tid", - }, - "field_application_problems": { - "id": "590e4b77-c4f4-4bd0-b5be-2ad2851da4f5", - "type": "category", - "key": "tid", - }, - "field_term_region": { - "id": "81c01278-8ba4-4d84-a1da-e05a07aad874", - "type": "category", - "key": "tid", - }, - "field_concept_upload": { - "id": "25740b9d-0f8f-4ce1-88fa-c6ee831c6aef", - "type": "file", - # TODO: finish mapping - }, - "field_application_otf_mission": { - "id": "5178e15f-d442-4d36-824d-a4292ef77062", - "type": "boolean", - }, - "field_application_otf_tos": { - "id": "bd91e220-4cdb-4392-8054-7b7dfe667d46", - "type": "boolean", - }, - "field_application_otf_represent": { - "id": "8d000129-ca8b-48cf-8dc2-4651bcbe46e8", - "type": "boolean", - }, - "field_application_otf_license": { - "id": "92f0801e-b9dc-4edc-9716-3f1709ae1c9b", - "type": "boolean", - }, - "field_application_otf_complete": { - "id": "3a3f2da3-4e32-4b86-9060-29c606927114", - "type": "boolean", - }, - "field_application_otf_deadline": { - "id": "19395179-ed9f-4556-9b6b-ab5caef4f610", - "type": "boolean", - }, - "field_application_otf_list": { - "id": "1345a8eb-4dcc-4170-a5ac-edda42d4dafc", - "type": "boolean", - }, - "field_application_otf_newsletter": { - "id": "4ca22ebb-daba-4fb6-a4a6-b130dc6311a8", - "type": "boolean", - }, -} - -FUND = FundType.objects.get(title='Fellowship archive fund') -ROUND = Round.objects.get(title='Fellowship archive round') -FORM = RoundForm.objects.filter(round=ROUND)[0] - -# Monkey patch the status field so it is no longer protected -patched_status_field = FSMField(default=INITIAL_STATE, protected=False) -setattr(ApplicationSubmission, 'status', patched_status_field) - - -class Command(BaseCommand): - help = "Fellowship application migration script. Requires a source JSON file." - data = [] - terms = {} - - def add_arguments(self, parser): - parser.add_argument('source', type=argparse.FileType('r'), help='Migration source JSON file') - - @transaction.atomic - def handle(self, *args, **options): - # Prepare the list of categories. - for item in CATEGORIES: - category, _ = Category.objects.get_or_create(name=item['category']) - option, _ = Option.objects.get_or_create(value=item['name'], category=category) - self.terms[item['tid']] = option - - with options['source'] as json_data: - self.data = json.load(json_data) - - for id in self.data: - self.process(id) - - def process(self, id): - node = self.data[id] - - try: - submission = ApplicationSubmission.objects.get(drupal_id=node['nid']) - except ApplicationSubmission.DoesNotExist: - submission = ApplicationSubmission(drupal_id=node['nid']) - - # TODO timezone? - submission.submit_time = datetime.fromtimestamp(int(node['created']), timezone.utc) - submission.user = self.get_user(node['uid']) - - submission.page = FUND - submission.round = ROUND - submission.form_fields = FORM.form.form_fields - - submission.status = self.get_workflow_state(node) - - form_data = { - 'skip_account_creation_notification': True, - } - - for field in node: - if field in STREAMFIELD_MAP: - try: - id = STREAMFIELD_MAP[field]['id'] - form_data[id] = self.get_field_value(field, node) - except TypeError: - pass - - if "value" not in form_data: - form_data["value"] = 0 - - submission.form_data = form_data - - try: - submission.save() - self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})") - except IntegrityError: - self.stdout.write(f"Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError") - pass - - def get_user(self, uid): - try: - return User.objects.get(drupal_id=uid) - except User.DoesNotExist: - return None - - def get_field_value(self, field, node): - """ - Handles the following formats: - field: {(safe_)value: VALUE} - field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. - field: {tid: ID} -- or term ID. fk to Categories - field: [] - field: [{value|target_id|tid: VALUE},] - """ - mapping = STREAMFIELD_MAP[field] - mapping_type = mapping['type'] - key = mapping.get('key', 'value') - source_value = node[field] - value = None - - if mapping_type == "direct": - value = source_value - elif mapping_type == 'value': - value = self.nl2br(source_value[key]) if source_value else '' - elif mapping_type == 'map' and 'map' in 'mapping': - value = mapping['map'].get(source_value[key]) - elif mapping_type == 'address' and 'map' in mapping: - try: - value_map = mapping['map'] - value = {} - for item in value_map: - value[value_map[item]] = source_value[item] - except TypeError: - value = {} - elif mapping_type == 'boolean': - value = source_value[key] == '1' if source_value else False - elif mapping_type == 'category': - if not source_value: - value = [] - else: - if isinstance(source_value, dict): - option = self.get_referenced_term(source_value[key]) - value = [option] if option else [] - else: - value = [] - for item in source_value: - option = self.get_referenced_term(item[key]) - if option: - value.append(option) - elif mapping_type == 'file': - # TODO finish mapping. Requires access to the files. - value = {} - - return value - - def get_referenced_term(self, tid): - try: - term = self.terms[tid] - return term.id - except KeyError: - return None - - def get_referenced_node(self, nid): - pass - - def get_workflow_state(self, node): - """ - workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}} - """ - states = { - "draft": "", - "published": "in_discussion", - "in_discussion": "in_discussion", - "council_review": "concept_internal_review", - "ready_for_reply": "concept_review_discussion", - "contract_review": "concept_review_discussion", - "in_contract": "invited_to_proposal", - "invited_for_proposal": "invited_to_proposal", - "dropped_concept_note": "concept_rejected", - "dropped": "concept_rejected", - "dropped_without_review": "concept_rejected" - } - - return states.get(node['workbench_moderation']['current']['state'], "in_discussion") - - def nl2br(self, value): - return value.replace('\r\n', '<br>\n') + "field_application_mail": { + "id": "email", + "type": "value", + "key": "email", + }, + "field_application_position": { + "id": "1282223d-77f5-4047-be03-4df4c4b2148a", + "type": "value", + "key": "safe_value", + }, + "field_application_role": { + "id": "9c0256e4-42e1-41fe-9880-7f621d6c3458", + "type": "value", + "key": "safe_value", + }, + "field_application_preapplied": { + "id": "f8efef0a-0632-4c81-b4db-7bc6a06caa7d", + "type": "map", + "map": { + "0": "No", + "1": "Yes", + }, + }, + "field_application_describe": { + "id": "1eb8b4e3-e2bb-4810-a8ce-3fc82a3192c8", + "type": "value", + "key": "safe_value", + }, + "field_application_how": { + "id": "177d56e8-2df1-4ead-8e3d-4916610fbed6", + "type": "value", + "key": "safe_value", + }, + "field_application_insight": { + "id": "05ff1755-947b-4e41-8f71-aae99977c572", + "type": "value", + "key": "safe_value", + }, + "field_application_duration2": { + "id": "611dacd7-553a-4be8-9283-1d006099d0c9", + "type": "map", + "map": { + "3": "3 months", + "6": "6 months", + "9": "9 months", + "12": "12 months", + "18": "18 months", + }, + }, + "field_application_host_text": { + "id": "0afaf4e1-4556-4e79-aa3d-4990e33620da", + "type": "value", + "key": "safe_value", + }, + "field_application_host2_text": { + "id": "a543b34f-ae6a-4b17-8ac3-ececc14573a0", + "type": "value", + "key": "safe_value", + }, + "field_application_questions": { + "id": "57cc52e2-b3ff-4e9f-a5fe-42e7735e16c2", + "type": "value", + "key": "safe_value", + }, + "field_application_status": { + "id": "ff4d12ff-7b88-4e87-bb5b-81543aef0e25", + "type": "category", + "key": "tid", + }, + "field_application_objectives": { + "id": "30c41288-a762-4003-acce-8c12e7343d90", + "type": "category", + "key": "tid", + }, + "field_application_beneficiaries": { + "id": "56833441-542b-4a06-8ad2-8e7e8fd1a334", + "type": "category", + "key": "tid", + }, + "field_application_focus": { + "id": "6b404851-ce2b-494f-b9f7-62858a937469", + "type": "category", + "key": "tid", + }, + "field_application_problems": { + "id": "590e4b77-c4f4-4bd0-b5be-2ad2851da4f5", + "type": "category", + "key": "tid", + }, + "field_term_region": { + "id": "81c01278-8ba4-4d84-a1da-e05a07aad874", + "type": "category", + "key": "tid", + }, + "field_concept_upload": { + "id": "25740b9d-0f8f-4ce1-88fa-c6ee831c6aef", + "type": "file", + # TODO: finish mapping + }, + "field_application_otf_mission": { + "id": "5178e15f-d442-4d36-824d-a4292ef77062", + "type": "boolean", + }, + "field_application_otf_tos": { + "id": "bd91e220-4cdb-4392-8054-7b7dfe667d46", + "type": "boolean", + }, + "field_application_otf_represent": { + "id": "8d000129-ca8b-48cf-8dc2-4651bcbe46e8", + "type": "boolean", + }, + "field_application_otf_license": { + "id": "92f0801e-b9dc-4edc-9716-3f1709ae1c9b", + "type": "boolean", + }, + "field_application_otf_complete": { + "id": "3a3f2da3-4e32-4b86-9060-29c606927114", + "type": "boolean", + }, + "field_application_otf_deadline": { + "id": "19395179-ed9f-4556-9b6b-ab5caef4f610", + "type": "boolean", + }, + "field_application_otf_list": { + "id": "1345a8eb-4dcc-4170-a5ac-edda42d4dafc", + "type": "boolean", + }, + "field_application_otf_newsletter": { + "id": "4ca22ebb-daba-4fb6-a4a6-b130dc6311a8", + "type": "boolean", + }, + } diff --git a/opentech/apply/funds/management/commands/migrate_fellowship_proposals.py b/opentech/apply/funds/management/commands/migrate_fellowship_proposals.py index 2f3d5fda01b2c6eef18203283339c5c4e9feeba0..7b46c1502da8153335834996c27488903646481f 100644 --- a/opentech/apply/funds/management/commands/migrate_fellowship_proposals.py +++ b/opentech/apply/funds/management/commands/migrate_fellowship_proposals.py @@ -1,237 +1,63 @@ -import argparse -import json - -from datetime import datetime, timezone - -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from django.db import transaction -from django.db.utils import IntegrityError -from django_fsm import FSMField - -from opentech.apply.categories.models import Category, Option -from opentech.apply.categories.categories_seed import CATEGORIES -from opentech.apply.funds.models import ApplicationSubmission, FundType, Round, RoundForm -from opentech.apply.funds.workflow import INITIAL_STATE - -User = get_user_model() - -STREAMFIELD_MAP = { - "title": { - "id": "title", - "type": "direct", - }, - "field_proposal_common_name": { - "id": "full_name", - "type": "value", - # If no Drupal value key is specified, we default to 'value' - "key": "safe_value", - }, - - "field_proposal_host_text": { - "id": "bc03235e-3c78-4770-9fc2-97feb93c2c8c", - "type": "value", - "key": "safe_value", - }, - - "field_proposal_start_date": { - "id": "672cb6f1-335c-4005-a0f1-46c414feda06", - "type": "value", - "key": "value", - }, - "field_proposal_completion_date": { - "id": "8262f209-f084-4a79-9dfa-2d18137119bb", - "type": "value", - "key": "value", - }, - "field_proposal_objectives": { - "id": "af2c5f38-7257-4295-87fa-787060e845ef", - "type": "value", - "key": "safe_value", - }, - "field_proposal_activities": { - "id": "3c521847-7642-4cae-aca9-d5336ad8962d", - "type": "value", - "key": "safe_value", - }, - "field_proposal_sustainability": { - "id": "fd0eb7ea-e054-4bcf-9580-eb672d44745c", - "type": "value", - "key": "safe_value", - }, - "field_proposal_request_questions": { - "id": "b6d71932-98c2-4ce8-a5e6-454a1f800d21", - "type": "value", - "key": "safe_value", - }, - "field_proposal_upload": { - "id": "30dfa46e-f656-46c9-9efc-bab9029f2008", - "type": "file", - # TODO: finish mapping - }, -} - -FUND = FundType.objects.get(title='Fellowship archive fund') -ROUND = Round.objects.get(title='Fellowship archive round') -FORM = RoundForm.objects.filter(round=ROUND)[1] - -from pprint import pprint -pprint(FORM) - -# Monkey patch the status field so it is no longer protected -patched_status_field = FSMField(default=INITIAL_STATE, protected=False) -setattr(ApplicationSubmission, 'status', patched_status_field) - - -class Command(BaseCommand): - help = "Fellowship proposal migration script. Requires a source JSON file." - data = [] - terms = {} - - def add_arguments(self, parser): - parser.add_argument('source', type=argparse.FileType('r'), help='Migration source JSON file') - - @transaction.atomic - def handle(self, *args, **options): - # Prepare the list of categories. - for item in CATEGORIES: - category, _ = Category.objects.get_or_create(name=item['category']) - option, _ = Option.objects.get_or_create(value=item['name'], category=category) - self.terms[item['tid']] = option - - with options['source'] as json_data: - self.data = json.load(json_data) - - for id in self.data: - self.process(id) - - def process(self, id): - node = self.data[id] - - try: - submission = ApplicationSubmission.objects.get(drupal_id=node['nid']) - except ApplicationSubmission.DoesNotExist: - submission = ApplicationSubmission(drupal_id=node['nid']) - - # TODO timezone? - submission.submit_time = datetime.fromtimestamp(int(node['created']), timezone.utc) - submission.user = self.get_user(node['uid']) - - submission.page = FUND - submission.round = ROUND - submission.form_fields = FORM.form.form_fields - - submission.status = self.get_workflow_state(node) - - form_data = { - 'skip_account_creation_notification': True, - } - - for field in node: - if field in STREAMFIELD_MAP: - try: - id = STREAMFIELD_MAP[field]['id'] - form_data[id] = self.get_field_value(field, node) - except TypeError: - pass - - if "value" not in form_data: - form_data["value"] = 0 - - submission.form_data = form_data - - try: - submission.save() - self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})") - except IntegrityError: - self.stdout.write(f"Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError") - pass - - def get_user(self, uid): - try: - return User.objects.get(drupal_id=uid) - except User.DoesNotExist: - return None - - def get_field_value(self, field, node): - """ - Handles the following formats: - field: {(safe_)value: VALUE} - field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. - field: {tid: ID} -- or term ID. fk to Categories - field: [] - field: [{value|target_id|tid: VALUE},] - """ - mapping = STREAMFIELD_MAP[field] - mapping_type = mapping['type'] - key = mapping.get('key', 'value') - source_value = node[field] - value = None - - if mapping_type == "direct": - value = source_value - elif mapping_type == 'value': - value = self.nl2br(source_value[key]) if source_value else '' - elif mapping_type == 'map' and 'map' in 'mapping': - value = mapping['map'].get(source_value[key]) - elif mapping_type == 'address' and 'map' in mapping: - try: - value_map = mapping['map'] - value = {} - for item in value_map: - value[value_map[item]] = source_value[item] - except TypeError: - value = {} - elif mapping_type == 'boolean': - value = source_value[key] == '1' if source_value else False - elif mapping_type == 'category': - if not source_value: - value = [] - else: - if isinstance(source_value, dict): - option = self.get_referenced_term(source_value[key]) - value = [option] if option else [] - else: - value = [] - for item in source_value: - option = self.get_referenced_term(item[key]) - if option: - value.append(option) - elif mapping_type == 'file': - # TODO finish mapping. Requires access to the files. - value = {} - - return value - - def get_referenced_term(self, tid): - try: - term = self.terms[tid] - return term.id - except KeyError: - return None - - def get_referenced_node(self, nid): - pass - - def get_workflow_state(self, node): - """ - workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}} - """ - states = { - "draft": "draft_proposal", - "published": "proposal_discussion", - "in_discussion": "proposal_discussion", - "council_review": "external_review", - "ready_for_reply": "post_external_review_discussion", - "contract_review": "post_external_review_discussion", - "in_contract": "proposal_accepted", - "invited_for_proposal": "proposal_accepted", - "dropped_concept_note": "proposal_rejected", - "dropped": "proposal_rejected", - "dropped_without_review": "proposal_rejected" - } - - return states.get(node['workbench_moderation']['current']['state'], "draft_proposal") - - def nl2br(self, value): - return value.replace('\r\n', '<br>\n') +from opentech.apply.funds.management.commands.migration_base import MigrateCommand + + +class Command(MigrateCommand): + CONTENT_TYPE = "fund" + FUND_NAME = "Fellowship archive fund" + ROUND_NAME = "Fellowship archive round" + APPLICATION_TYPE = "proposal" + + STREAMFIELD_MAP = { + "title": { + "id": "title", + "type": "direct", + }, + "field_proposal_common_name": { + "id": "full_name", + "type": "value", + # If no Drupal value key is specified, we default to 'value' + "key": "safe_value", + }, + + "field_proposal_host_text": { + "id": "bc03235e-3c78-4770-9fc2-97feb93c2c8c", + "type": "value", + "key": "safe_value", + }, + + "field_proposal_start_date": { + "id": "672cb6f1-335c-4005-a0f1-46c414feda06", + "type": "value", + "key": "value", + }, + "field_proposal_completion_date": { + "id": "8262f209-f084-4a79-9dfa-2d18137119bb", + "type": "value", + "key": "value", + }, + "field_proposal_objectives": { + "id": "af2c5f38-7257-4295-87fa-787060e845ef", + "type": "value", + "key": "safe_value", + }, + "field_proposal_activities": { + "id": "3c521847-7642-4cae-aca9-d5336ad8962d", + "type": "value", + "key": "safe_value", + }, + "field_proposal_sustainability": { + "id": "fd0eb7ea-e054-4bcf-9580-eb672d44745c", + "type": "value", + "key": "safe_value", + }, + "field_proposal_request_questions": { + "id": "b6d71932-98c2-4ce8-a5e6-454a1f800d21", + "type": "value", + "key": "safe_value", + }, + "field_proposal_upload": { + "id": "30dfa46e-f656-46c9-9efc-bab9029f2008", + "type": "file", + # TODO: finish mapping + }, + }