pull_from_submittable.py

#!/usr/bin/env python3

__doc__ = """\
Pull fresh data from submittable for the Build a World of Play competition

Usage:

  $ compose-and-upload \\
       --proposals-json=FILE
       --mapping-json=FILE

Command-line options:
  --proposals-json FILE           FILE is the json file that this will get output to.

  --mappings-json FILE            FILE is the json file that holds the mappings between
                                  submittables registration numbers and the application
                                  numbers we assign.
"""

from etl import submittable
import config
import sys
import getopt
import json

torque_mappings = {
    # Fields for Action For Women's Health Registration:
    "2614d3f6-2a08-413c-aa8c-578e60568f6b": "Organization Legal Status", #What is the legal status of the Applicant Organization? (Single Response)
    "a65db225-1542-490b-bc4f-6efc00bce31c": "Institution Affiliation", #Is the Applicant Organization a specific lab or center housed within an institution of higher learning (e.g., a college or university) or a specific department or program housed within a larger organization? (Single Response)
    "777a45ee-2a7d-4150-a32f-5ada009a9ee7": "Project Name", #Project Name (Short Answer)
    "eec25dda-cfd8-4bc5-adac-c2d952ec8ab1": "Department Name", #Lab, Center, Program, or Department Name (Short Answer)
    "f96a58a9-47af-4448-9fb6-d48eb6f47928": False, #Is your organization based in the United States or U.S. Territories? (Fiscally sponsored organizations: please provide the location of the fiscal sponsor.) (Single Response)
    "d1f30e10-e7f4-4e62-8522-715c12d7aab1": "Organization Location|Domestic", #Applicant Organization Address: (Fiscally sponsored projects: please provide the physical mailing address of the fiscal sponsor.) (Address)
    "650dfe1e-500b-48ba-843e-6b354f56624f": False, #Does the Applicant Organization exist within a Tribal Territory? (Single Response)
    "eff67f7e-fca4-486b-93b0-078334297c45": "Organization Location|International", #Applicant Organization Address: (Address).  This seems unused (the above Org Location)
    "9702eb63-7489-4d68-9df9-47258195b422": "Applicant Tax Identification Number|Charity Check", #Organization Identification Number: (Charity Check)
    "2692e1ba-a87d-436d-b821-155f211a4999": "Applicant Tax Identification Number|Selection", #Organization Identification Number: The Applicant Organization must be incorporated in an appropriate jurisdiction. Typically, an identifying number is provided upon incorporation or for tax filings. Select the Country and Identification Type of number that the Applicant Organization has been provided. (Single Response)
    "be883998-134c-4604-bbda-736348abae77": "Applicant Tax Identification Number|Australia Business Number", #Australia Business Number (ABN)  (Short Answer)
    "d324121f-9416-45e9-9ea0-7bb15060e3e8": "Applicant Tax Identification Number|CPNJ", #CNPJ (for companies) (Short Answer)
    "0946f368-8edd-4b49-a550-605bc4d57793": "Applicant Tax Identification Number|Revenue Agency Business", #Revenue Agency Business/Registration Number  (Short Answer)
    "1e68de34-3d2a-4274-bd39-81571cacb769": "Applicant Tax Identification Number|FCRA Registration Number", #FCRA Registration Number  (Short Answer)
    "28f5c6a8-78f1-4047-b0ca-0007dda1190c": "Applicant Tax Identification Number|KVK Number", # KVK Number (Short Answer)
    "44bdfd96-4171-4447-ae3f-b5b73c7f6803": "Applicant Tax Identification Number|Nonprofit Organzation Number", #Nonprofit Organization Number (Short Answer)
    "47e81fc3-1466-46de-bf60-644656454bc6": "Applicant Tax Identification Number|Chartity Number", #Charity Number  (Short Answer)
    "cd1e7c62-c23b-43ff-a281-298ce1767787": "Applicant Tax Identification Number|Company Number", #Company Number  (Short Answer)
    "3b9984af-f565-4def-8a12-25726d64ba3e": "Applicant Tax Identification Number|Country Name", #Enter name of country: (and Identification Number) (Short Answer)
    "e2c2d563-b182-4fc3-a9ed-70f874c89c31": "Year of Formation", #Year of Formation (Short Answer)
    "a1d3cb44-f636-4914-bad8-d1c65ccfe96b": "Equivalency Determination", #Has the Applicant Organization previously been determined to be the equivalent of a 501(c)(3) and 509(a)(1) or (2) organization by NGOsource for which equivalency determination (ED) has not expired and would be in effect or could extend through the date of award? (Single Response)
    "5b73fb6b-ecd9-4f0e-9481-db16e35530b2": "Organization Website", #Applicant Organization Website or Social Media Page (Website)
    "1b7fe176-72fe-4841-8e1e-8117e9662955": "Primary Contact|Name", #Primary contact name  (Name)
    "288766ff-4167-48ab-ad89-ad0a2ae9122f": "Primary Contact|Title", #Primary contact title (Short Answer)
    "52c5c438-08c1-4074-a29a-7391f322896f": "Primary Contact|Phone", #Primary contact phone (Phone)
    "a7716a6c-3d16-47a9-b68a-c14ad776181f": "Primary Contact|Email", #Primary contact email (Email)
    "53b9d67c-a9aa-4112-81ed-32a171a0226e": "Secondary Contact|Name", #Secondary contact name (Name)
    "f933e93a-9263-43ed-a3ef-5be130217627": "Secondary Contact|Title", #Secondary contact title (Short Answer)
    "6a8e2977-69a9-41be-8226-86dde66e823a": "Secondary Contact|Phone", #Secondary contact phone (Phone)
    "53b4a60e-201b-45b5-9133-fd5dd0788161": "Secondary Contact|Email", #Secondary contact email (Email)
    "b85a2bd7-79e8-43e0-8249-36969fc669d3": "Source|Selection", #How did you hear about the Action For Women's Health? You may select more than one. (Multiple Response)
    "53035468-38de-4f2e-87c6-858e471b9b1f": "Source|Other Direct Outreach", #(Other Direct Outreach) Please describe: (Short Answer)
    "3e30426d-e6a2-4583-b92d-183a55199492": "Source|Other News Source", #(Other News Source) Please describe: (Short Answer)
    "b83e4702-87fb-4cce-9cb2-8eb869c6d972": "Source|Social Media", #(Social Media) Please describe: (Short Answer)
    "7f3694ef-e28e-47a1-9937-c5522f020a14": "Source|Partner Organization", #(Partner Organization) Please describe: (Short Answer)
    "378e9d94-b668-4583-a270-2843db4b98f7": "Source|Event", #(Event) Please describe: (Short Answer)
    "2b8a583e-b9f8-43c7-8524-4fbed767e2f4": "Source|Other", #(Other) Please describe: (Short Answer)
    "8158a878-56c4-45f3-8e50-6bf090999247": False, #I consent to receiving email communications from Lever for Change, including updates on future challenges, news, and opportunities. (Single Response)
    "ff07bc10-ba77-420f-b41b-07c09b5736cf": False, #Acceptance of Rules (Checkbox)

    # Fields for Action For Women's Health Application:
    "52ea08b2-7d5b-4515-9727-d03257c241d5": "Annual Operating Budget", #Annual Operating Budget:  What is the annual operating budget in United States Dollars (USD) of the Applicant Organization? (Single Response)
    "e9e98537-aa87-4ab5-b5c0-f3074d504898": "Number of Employees", #Number of Employees: How many full-time employees does the Applicant Organization employ? (Single Response)
    "fda56f5f-f6a7-4b02-a33c-1ecf165e4975": False, #Is the Applicant Organization based in the United States or U.S. Territories? (Fiscally sponsored projects: please provide the location of the fiscal sponsor.) (Single Response)
    "dd1e3815-b92f-4755-98a4-3cc5292eff03": False, #Is the Applicant Organization a nonprofit organization applying on behalf of its fiscally sponsored project? (Single Response)
    "6f4217d3-180e-4378-a7fa-b72110e6752a": "Primary Subject Area", #Primary Subject Area: Select the primary subject area of your organization’s work. The following subject area categories are standard across Lever for Change initiatives. (Dropdown)
    "72f30094-702e-4493-bdf8-b06686f99ff3": "Additional Subject Areas", #Secondary Subject Area: Select the secondary subject area of your organization’s work. The following subject area categories are standard across Lever for Change initiatives. (Optional) (Dropdown)
    "69a3ba67-9357-47d0-9238-ae4a2f69539f": "Key Words and Phrases", #Key Words and Phrases: Provide a list of up to 5 key words or phrases that can best be used to describe your organization’s work [ex. Social-enterprise, vocation, internship, training]. Choose key words that capture the essence of your organization and its mission. The key words should be different from the selected “primary subject area.”  (Short Answer)
    "67216a87-3b7f-4227-8e2c-883a4b36ad96": "Current Work #1 Location|Country", #Location of Current Work 1 Country: (Dropdown)
    "92101e1c-6a2c-4f9b-b67d-6eb88cbdc2d3": "Current Work #1 Location|State/Province", # Location of Current Work 1 State: (Dropdown)
    "d5f46baa-2354-4006-b12b-f65526ee8c6a": "Current Work #2 Location|Country", #Location of Current Work 2 Country: (optional) (Dropdown)
    "60ca9929-6a87-4a1a-a5ca-af16f6e87d1a": "Current Work #2 Location|State/Province", # Location of Current Work 2 State: (Dropdown)
    "c27175a7-4110-4e62-85d2-1a486ff360b7": "Current Work #3 Location|Country", #Location of Current Work 3 Country: (optional) (Dropdown)
    "d528c9dc-6b9d-460a-9918-86f3786a1ec3": "Current Work #3 Location|State/Province", # Location of Current Work 3 State: (Dropdown)
    "feee74ce-227e-4aba-97fc-5e05cb06eadd": "Current Work #4 Location|Country", #Location of Current Work 4 Country: (optional) (Dropdown)
    "b0f2aba5-fbd4-4d68-b216-6cceb0cafd01": "Current Work #4 Location|State/Province", # Location of Current Work 4 State: (Dropdown)
    "f0fb06ce-339b-4def-950c-30b12a1d01ec": "Current Work #5 Location|Country", #Location of Current Work 5 Country: (optional) (Dropdown)
    "899438cc-9dac-4646-9c1d-2c733ceb3cb0": "Current Work #5 Location|State/Province", # Location of Current Work 5 State: (Dropdown)
    "6ea69235-eb81-41f4-ab1a-43a600703c03": "Priority Populations|1", #POPULATION #1: (Dropdown)
    "f01a3793-e1d7-447e-8f38-ef3b95d81fc5": "Priority Populations|2", #POPULATION #2: (Optional) (Dropdown)
    "45b8947f-0b5d-4b93-babc-220b4ab07dfc": "Priority Populations|3", #POPULATION #3: (Optional) (Dropdown)
    "d47d708c-56ba-439e-a5c6-21da8605e0e7": "Project Description", #Description of Work (35 words):  Provide a short description of your organization’s work in one sentence using language that can be easily understood by non-experts and avoiding jargon and acronyms.  (Short Answer)
    "6fdf3b92-7e20-4f02-9fae-af899aad7bae": "Executive Summary", #Executive Summary (Long Answer)
    "5b0228f9-488f-4ad1-819f-e2692b9aa3f7": "Affirmation of Approach", #Affirmation of Approach (250 words): What gives you confidence that your approach works?  (Long Answer)
    "a3d353eb-ebb2-40fb-9c03-a783932237fb": "External References", #External References (150 words)  (Long Answer)
    "f2374f9f-fe22-4637-a91d-48c957da2f8c": "Video", #Paste the full YouTube URL in the box below. (Website)
    "971c0f86-d283-48e2-92f4-8768704b3e89": "Why Your Team", #Why Your Team (250 words): (Long Answer)
    "f333a080-bcfe-4c9b-83c7-cb6e5be6320f": "Key Staff #1|Name", #Key Staff #1: (Name)
    "f64dc136-2a89-4ff1-b84b-9a2d9af392c1": "Key Staff #1|Title", #Title: (Short Answer)
    "329461ea-4c66-4fb0-828a-e3b80e3d6fc5": "Key Staff #1|Biography", #Biography: (Long Answer)
    "f588fc83-9c1e-4ad6-83cd-776ffb13f28f": "Key Staff #2|Name", #Key Staff #2: (Name)
    "98d6f7da-a906-4da9-9499-d06e3557b04b": "Key Staff #2|Title", #Title: (Short Answer)
    "8ba6917c-3b3b-4d14-9350-9732cb77a344": "Key Staff #2|Biography", #Biography: (Long Answer)
    "c456ed92-d282-42e3-9ce2-e3549eafc75e": "Key Staff #3|Name", #Key Staff #3: (Name)
    "5ffd945a-b546-4d7d-8a64-8ed4132ca2c3": "Key Staff #3|Title", #Title: (Short Answer)
    "a8bef610-d7c3-4cff-900c-34c39337a49c": "Key Staff #3|Biography", #Biography: (Long Answer)
    "9b9cd174-ae7d-42df-a8e9-fd41e05684f5": "Overview of Work", #Overview of Work (250 words): (Long Answer)
    "ae0f5cdb-3528-49bd-bcb3-d544df46ddf8": "Issue Landscape", #Issue Landscape (250 words): (Long Answer)
    "aba07d60-6b84-414d-8a67-c3a97b2a070f": "Stakeholder Engagement", #Stakeholder Engagement (200 words): (Long Answer)
    "09c3eabb-5fef-476c-99aa-07d5a0a2af7c": "Diversity-Equity-Inclusion", #Diversity, Equity, and Inclusion (300 words): (Long Answer)
    "f341a36d-755a-4aa5-a3a0-f5b5dcac3120": "Measurement of Results", #Assessment of Results (250 words): (Long Answer)
    "ad11584c-413a-4525-ad5c-041a0c3410f3": "Why Now?", #Why Now? (250 words): (Long Answer)
    "f96e6312-c6db-4c69-abf3-800197ce27dc": "Vision for Growth", #Vision for Growth (250 words): (Long Answer)
    "a702a766-af20-42d7-a769-aec2cf9e14d7": "Other Considerations", #Other Considerations (150 words): (Long Answer)
    "ec1af885-855d-4218-b491-0004ed16ee28": "Attachments|Governance Structure", #Governance Structure (File Upload)
    "08d517b2-662f-4c9a-a71f-4b60c805943d": "Attachments|Proof of Registration", #Proof of Registration (File Upload)
    "2c859b2c-8179-49c0-bcf8-5396dfb932e5": "Attachments|Audited Finacial Statements", #Audited Financial Statements (File Upload)
    "16989570-7f97-44f3-993e-014faa7cac4e": "Attachments|Form 990", #Form 990 (File Upload)
    "22c4f250-c260-48f8-8df3-f195b9dcf0c8": "Attachments|Audited Financial Statements", #Audited Financial Statements (File Upload)
    "bccecfc3-e0ae-495b-b926-21bed832dd62": False, #Language – Audited Financial Records (Short Answer)
    "332b30be-65be-46ef-88cd-4cc6f4724c12": "Attachments|Fiscal Sponsorship Agreement", #Fiscal Sponsorship Agreement (in English) (File Upload)
    "7b3bd89a-dbf6-48c9-83d8-8764b3707920": "Attachments|Audited Financial Statements of Fiscal Sponsor", #Audited Financial Statements of Fiscal Sponsor  (File Upload)
    "044a011a-9145-41db-9cc6-e6f395e8104d": "Age of Organization", #Age of Organization: Is your organization less than five years old (as of January 1, 2025)? Fiscally sponsored projects: please answer for the fiscal sponsor.  (Single Response)
    "2b53b147-d113-4524-95f0-a7dd94e29c63": "Funder #1", #FUNDER #1 NAME:    (Short Answer)
    "e8dd37a3-ea3a-41fc-925c-fa2880e0bf5e": "Funder #1: First Year of Funding", #FIRST YEAR OF FUNDING:  (Short Answer)
    "c1f2db47-cbeb-4898-b1ed-9289de33277d": "Funder #1: Last Year of Funding", #LAST YEAR OF FUNDING:  (Short Answer)
    "c0f5f908-be24-459c-85a1-cf983845cf8c": "Funder #1: Amount of Funding", #AMOUNT OF FUNDING (Number)
    "b42e1810-5bfe-44c2-b76e-e38f4fdafe97": "Funder #2", #FUNDER #2 NAME:    (Short Answer)
    "f5ee6044-515a-4705-a79d-bb009852cb76": "Funder #2: First Year of Funding", #FIRST YEAR OF FUNDING:  (Short Answer)
    "a49fc071-1790-45f4-92fb-272669d639bf": "Funder #2: Last Year of Funding", #LAST YEAR OF FUNDING:  (Short Answer)
    "bbb0316b-fc69-44a9-a3b9-85f1a3cb34ac": "Funder #2: Amount of Funding", #AMOUNT OF FUNDING (Number)
    "ee105954-7cae-4cf9-a0e6-6dc574422129": "Funder #3", #FUNDER #3 NAME:    (Short Answer)
    "7d1f5974-5ae6-45cc-aaf1-046cd8a1a3af": "Funder #3: First Year of Funding", #FIRST YEAR OF FUNDING:  (Short Answer)
    "12f816fc-b71a-4b64-b38c-4c7efd35c7a5": "Funder #3: Last Year of Funding", #LAST YEAR OF FUNDING:  (Short Answer)
    "dcfda2d7-575b-4535-9701-d67668fcf984": "Funder #3: Amount of Funding", #AMOUNT OF FUNDING (Number)
    "93951338-40fd-47f9-933a-d15701d49190": "Funder #4", #FUNDER #4 NAME:    (Short Answer)
    "96314c98-80c2-4f26-8da2-ca3e50626921": "Funder #4: First Year of Funding", #FIRST YEAR OF FUNDING:  (Short Answer)
    "c4d0c56c-bd7d-4b20-b2d5-0ff5f83c7b1c": "Funder #4: Last Year of Funding", #LAST YEAR OF FUNDING:  (Short Answer)
    "1a161165-54d0-4a3e-9186-1bb3560b62c8": "Funder #4: Amount of Funding", #AMOUNT OF FUNDING (Number)
    "58817833-6c9a-4cfb-bad7-12e59e47aaba": "Funder #5", #FUNDER #5 NAME:    (Short Answer)
    "979964e3-f31c-4ad2-a32b-8bb11951ad3c": "Funder #5: First Year of Funding", #FIRST YEAR OF FUNDING:  (Short Answer)
    "2920afc8-22f3-487f-ba85-df8156f3ca72": "Funder #5: Last Year of Funding", #LAST YEAR OF FUNDING:  (Short Answer)
    "53a2cd41-058e-4056-8479-34888e96e475": "Funder #5: Amount of Funding", #AMOUNT OF FUNDING (Number)
    "9bc186e7-d004-4f44-a278-a5ceb15d5686": False, #Are any of the following true of the Applicant Organization, either currently or in the last three (3) years? (Multiple Response)
    "6f3b441c-e91e-45a8-8049-cebbd67e34d2": "Additional Information", #Please feel free to use the text box below to provide any additional information regarding your response to the Organizational Circumstance question. (Long Answer)

    # Fields for Participatory Review:
    "ba349d5c-cdaa-4a76-90f3-8598a118f7f5": False, # Does assessing this application constitute a conflict of interest? (Single Response)
    "e753d26d-295b-4fd0-bfa4-29889cee4205": False, # #1 IMPACTFUL (1-5) (Single Rating)
    "af2db550-0097-46fd-a20d-f382587f5ad0": False, # #1 IMPACTFUL: Reviewer Comments: (Long Answer)
    "626ea6ba-9ffe-4708-8dd8-12174ebe7977": False, # #2 SCALABLE (Single Rating)
    "10b78d67-8f00-4e2c-b2e3-cb156cb2d3e8": False, # #2 SCALABLE: Reviewer Comments: (Long Answer)
    "5c7b9dc1-1aef-4d2f-8cf1-ac114c2708e0": False, # #3 EQUITABLE (Single Rating)
    "e3163de4-4cb6-4013-ab4c-8e35a0377485": False, # #3 EQUITABLE: Reviewer Comments: (Long Answer)
    "77a18506-2288-4e1f-868d-37abbca21ec9": False, # #4 FEASIBLE  (Single Rating)
    "ced3ff61-843d-4ed2-9ff4-99227840bd19": False, # #4 FEASIBLE: Reviewer Comments: (Long Answer)

    # Fields for Administrative Review:
    "db901c70-78b7-46db-8aaa-edadda01952c": False, # Is this organization an ineligible entity type per the Open Call Rules? (Single Response)
    "2a22e126-a201-4a65-9e88-106a76041c96": False, # Does the organization appear to be any of the following ineligible entity types? (Single Response)
    "18bb0e9c-0ae9-441c-a047-d779e64883d4": False, # Is the applicant based in any of the countries or regions on the OFAC sanctions list?  (Single Response)
    "003a6d52-8f86-4910-9539-a2d395abbcd3": False, # Which of these countries on the OFAC sanctions list is the applicant based in? (Dropdown)
    "55d99d4a-31dc-447b-a41c-1df488d6914c": False, # Is the applicant performing work in any of the countries or regions on the OFAC sanctions list?  (Single Response)
    "8533defc-4a98-4c7d-bebf-5678a72813c4": False, # Which of these countries on the OFAC sanctions list is the applicant performing work in? (Dropdown)
    "32e7f7ad-80a8-473b-9f44-0be6f24f421a": False, # Does the applicant engage in any of the ineligible activities? (Single Response)
    "2d79f2b7-604c-40ea-bc0d-2ba350fe8cb6": False, # Which of the following ineligible activities does the applicant engage in?  (Single Response)
    "4fe68c5c-c41b-4473-9e7d-10bcff1fe494": False, # Does the video have issues playing? (Single Response)
    "b6831d3c-da6d-4799-9390-2afd57e5c5a6": False, # Is the video missing English captions?   (Single Response)
    "ee94296e-6246-4340-80fe-4c364a295b17": False, # Is the video greater than 90 seconds?  (Single Response)
    "e9876594-d8b7-4d48-88ce-9d2ad335b801": False, # Was this application submitted in a language other than English?  (Single Response)
    "67ca0447-90da-499f-bded-20bdff015d41": False, # Is the application INCOMPLETE (i.e., all sections have not been completed to a sufficient degree and/or do not make reasonable sense)?  (Single Response)
    "8b75a017-8385-4e42-a687-b88f40034df0": False, # Does the application include profane or inappropriate language and/or disparaging or discriminatory content? (Single Response)
    "ecbaeeae-cf45-4452-ad50-7f855e253c28": False, # Is the applicant organization’s main purpose something other than improving women’s mental and/or physical health? (Single Response)
    "3945216b-1b13-449a-a0d0-c7ec7475953f": False, # Is this application ready for Participatory Review? (Single Rating)
    "73fb3150-f5d5-4445-854b-e4f2d5ebbae5": False, # Please provide relevant information regarding your recommendation. (Long Answer)

    # Extra fields
    "bb26f981-1806-40a4-b3a9-7e20d6ae6d5d": False, # This was weirdly not in the export forms output and seemed to be a duplicate of Acceptance of Rules
    "c95f4a10-1b6e-4cec-9f75-6cfe230d8afa": "Organization Name", # This was weirdly not in the export forms output and seemed to be a duplicate

    "3588cc24-b17b-4087-9b9c-a8706098f19d": False, # Part of extra export stuff - Message Type
    "14881e8c-76bb-4ae2-b13d-2a4ed44e4489": False, # Part of extra export stuff - recipient Info

}

torque_allowlist_exceptions = [
    "Current Work #1 Location",
    "Current Work #2 Location",
    "Current Work #3 Location",
    "Current Work #4 Location",
    "Current Work #5 Location",
    "External References",
    "Key Staff #1",
    "Key Staff #2",
    "Key Staff #3",
    "Issue Landscape",
    "Stakeholder Engagement",
    "Vision for Growth",
    "Funder #1",
    "Funder #1: First Year of Funding",
    "Funder #1: Last Year of Funding",
    "Funder #1: Amount of Funding",
    "Funder #2",
    "Funder #2: First Year of Funding",
    "Funder #2: Last Year of Funding",
    "Funder #2: Amount of Funding",
    "Funder #3",
    "Funder #3: First Year of Funding",
    "Funder #3: Last Year of Funding",
    "Funder #3: Amount of Funding",
    "Funder #4",
    "Funder #4: First Year of Funding",
    "Funder #4: Last Year of Funding",
    "Funder #4: Amount of Funding",
    "Funder #5",
    "Funder #5: First Year of Funding",
    "Funder #5: Last Year of Funding",
    "Funder #5: Amount of Funding",
    "Year of Formation",
    "Department Name",
    "Overview of Work",
    "Age of Organization",
    "Additional Information",
    "Year of Formation",
    "Department Name",
    "Institution Affiliation",
]

attachment_cols = [
    "Attachments|Governance Structure",
    "Attachments|Proof of Registration",
    "Attachments|Audited Finacial Statements",
    "Attachments|Form 990",
    "Attachments|Audited Financial Statements",
    "Attachments|Fiscal Sponsorship Agreement",
    "Attachments|Audited Financial Statements of Fiscal Sponsor",
]

def main():
    try:
        opts, args = getopt.getopt(
            sys.argv[1:],
            "",
            [
                "applications-json=",
                "registrations-json=",
                "mapping-json=",
                "cache-file=",
                "num-days-cache-invalidate=",
                "attachments-dir=",
                "submittable-attachments-dir=",
            ],
        )
    except getopt.GetoptError as err:
        sys.stderr.write("ERROR: '%s'\n" % err)
        sys.exit(2)

    applications_json = None
    registrations_json = None
    mapping_json = None
    cache_file = None
    num_days_cache_invalidate = None
    attachments_dir = None
    submittable_attachments_dir = None
    for o, a in opts:
        if o == "--applications-json":
            applications_json = a
        elif o == "--registrations-json":
            registrations_json = a
        elif o == "--mapping-json":
            mapping_json = a
        elif o == "--cache-file":
            cache_file = a
        elif o == "--num-days-cache-invalidate":
            num_days_cache_invalidate = int(a)
        elif o == "--attachments-dir":
            attachments_dir = a
        elif o == "--submittable-attachments-dir":
            submittable_attachments_dir = a
        else:
            sys.stderr.write("ERROR: unrecognized option '%s'\n" % o)
            sys.exit(2)

    if applications_json is None:
        sys.stderr.write("ERROR: need --proposals-json option.\n\n")
        sys.stderr.write(__doc__)
        sys.exit(1)

    for (status, output_file) in [("anyCompleted", applications_json), (False, registrations_json),]:
        submissions_payload = {}
        if status:
            submissions_payload["AdditionalFormStatuses"] = status

        sub = submittable.Submittable(
            config.submittable_api_key,
            torque_mappings,
            [
                "f313369f-1b44-4696-b9e7-d1cbe60495e9", # Peer review
                #"fe0cc464-8278-4c2f-8bc5-f8c2ede14d27", # Panel review
            ],
            None,
            "7a7fdb3f-1c57-422c-93ea-e64460c744de",
            mapping_json,
            num_days_cache_invalidate,
            allowlist_exceptions=torque_allowlist_exceptions,
            cache_file=cache_file,
            attachments_dir=submittable_attachments_dir,
            submissions_payload=submissions_payload,
        )

        sub.output_data_files(output_file, mapping_json)
        sub.convert_attachments(attachment_cols, attachments_dir)

    application_data = json.load(open(applications_json))
    traits = ['EQUITY-FOCUSED', 'TRACK RECORD', 'COMMUNITY LEADERSHIP', 'TEAM CAPACITY']

#    for application in application_data:
#        trait_data = list(application["Peer Judge Data|%s|Comments" % traits[0]])
#
#        if(len(trait_data) > 5):
#            if(len(trait_data) > 6):
#                raise Exception("More than 6?! We need to handle that!")
#            trait_data = sorted(trait_data, key=lambda t: t["completedAt"])
#            removing_email = trait_data[5]["creator"]["email"]
#            print("Removing Review from %s (%s) on submission %s" % (trait_data[5]["creator"]["name"]["full"], trait_data[5]["creator"]["email"], application["Submittable ID"]))
#            for trait in traits:
#                application["Peer Judge Data|%s|Comments" % trait] = [ comment for comment in application["Peer Judge Data|%s|Comments" % trait] if comment["creator"]["email"] != removing_email]
#                application["Peer Judge Data|%s|Score" % trait] = [ score for score in application["Peer Judge Data|%s|Score" % trait] if score["creator"]["email"] != removing_email]

    json.dump(application_data, open(applications_json, "w"))


if __name__ == "__main__":
    main()