Add finalist attachments, and corrections

From the ETL pipeline work on June 22, 2021

competitions/RacialEquity2030/etl/compose-and-upload

@@ -169,16 +169,16 @@ def main():
         proposals_csv, "RacialEquity2030", "Application #", pare
     )
 
-    correction_processor = competition.CorrectionData("Application #", correction_file)
-    for column in correction_processor.columns_affected():
-        comp.process_cells_special(column, correction_processor)
-
     comp.add_supplemental_information(
         competition.LinkedSecondSheet(
             admin_review_csv, "Application #", [{"source_name": "Organization name"}]
         )
     )
 
+    correction_processor = competition.CorrectionData("Application #", correction_file)
+    for column in correction_processor.columns_affected():
+        comp.process_cells_special(column, correction_processor)
+
     comp.filter_proposals(
         competition.ColumnEqualsProposalFilter("Admin Review Status", "Not Applicable")
     )

competitions/RacialEquity2030/etl/deploy

@@ -32,6 +32,7 @@ PROPOSALS_CSV="RE2020_Registration_Application_2021-03-24_06-29-20.csv"
 P2P_RANKING_CSV="RE2020_Peer_Review_2021-05-13_14-27-30_No_Judge_Names.csv"
 EP_RANKING_CSV="RE2020_Evaluation_Panel_2021-06-14_07-31-58.csv"
 BASE_ATTACHMENTS="RE2020_Registration_Application_exports_2021-03-24_08-08-31.zip"
+FINALIST_ATTACHMENTS="Finalist Attachments.zip"
 CORRECTION_FILE="OTS_Corrections.csv"
 ADMIN_REVIEW_CSV="Kellogg-AdminReview.csv"
 
@@ -67,6 +68,7 @@ if [ ! -d "${DATA_DIR}" ] ; then
 
   decrypt "${DATA_DIR}/${PROPOSALS_CSV}" "${ENCRYPTED_DIR}/${PROPOSALS_CSV}.gpg" "${GPG_PASSPHRASE}"
   decrypt "${DATA_DIR}/${BASE_ATTACHMENTS}" "${ENCRYPTED_DIR}/${BASE_ATTACHMENTS}.gpg" "${GPG_PASSPHRASE}"
+  decrypt "${DATA_DIR}/${FINALIST_ATTACHMENTS}" "${ENCRYPTED_DIR}/${FINALIST_ATTACHMENTS}.gpg" "${GPG_PASSPHRASE}"
   decrypt "${DATA_DIR}/${ADMIN_REVIEW_CSV}" "${ENCRYPTED_DIR}/${ADMIN_REVIEW_CSV}.gpg" "${GPG_PASSPHRASE}"
   decrypt "${DATA_DIR}/${CORRECTION_FILE}" "${ENCRYPTED_DIR}/${CORRECTION_FILE}.gpg" "${GPG_PASSPHRASE}"
   decrypt "${DATA_DIR}/${P2P_RANKING_CSV}" "${ENCRYPTED_DIR}/${P2P_RANKING_CSV}.gpg" "${GPG_PASSPHRASE}"
@@ -78,6 +80,15 @@ if [ ! -d "${DATA_DIR}" ] ; then
   mkdir -p $TMP_ATTACHMENTS_DIR
   unzip -d ${TMP_ATTACHMENTS_DIR} ${DATA_DIR}/${BASE_ATTACHMENTS}
   mv "${TMP_ATTACHMENTS_DIR}/"*"/Valid" $ATTACHMENTS_DIR
+  rm -rf $TMP_ATTACHMENTS_DIR
+
+  mkdir -p $TMP_ATTACHMENTS_DIR
+  unzip -d $TMP_ATTACHMENTS_DIR "${DATA_DIR}/${FINALIST_ATTACHMENTS}"
+  ls "$TMP_ATTACHMENTS_DIR/Finalist Attachments" | while read file ; do
+    NUM=${file/_*/};
+    mv -v $TMP_ATTACHMENTS_DIR/*/$NUM*/* ${ATTACHMENTS_DIR}/$NUM/
+  done
+  rm -rf $TMP_ATTACHMENTS_DIR
 
   # Clean up large zips to save disk space
   rm ${DATA_DIR}/${BASE_ATTACHMENTS}