Skip to content
Snippets Groups Projects
Commit cee50d19 authored by Frank Duncan's avatar Frank Duncan
Browse files

Add finalist attachments, and corrections

From the ETL pipeline work on June 22, 2021
parent f74e80f0
No related branches found
No related tags found
No related merge requests found
......@@ -169,16 +169,16 @@ def main():
proposals_csv, "RacialEquity2030", "Application #", pare
)
correction_processor = competition.CorrectionData("Application #", correction_file)
for column in correction_processor.columns_affected():
comp.process_cells_special(column, correction_processor)
comp.add_supplemental_information(
competition.LinkedSecondSheet(
admin_review_csv, "Application #", [{"source_name": "Organization name"}]
)
)
correction_processor = competition.CorrectionData("Application #", correction_file)
for column in correction_processor.columns_affected():
comp.process_cells_special(column, correction_processor)
comp.filter_proposals(
competition.ColumnEqualsProposalFilter("Admin Review Status", "Not Applicable")
)
......
......@@ -32,6 +32,7 @@ PROPOSALS_CSV="RE2020_Registration_Application_2021-03-24_06-29-20.csv"
P2P_RANKING_CSV="RE2020_Peer_Review_2021-05-13_14-27-30_No_Judge_Names.csv"
EP_RANKING_CSV="RE2020_Evaluation_Panel_2021-06-14_07-31-58.csv"
BASE_ATTACHMENTS="RE2020_Registration_Application_exports_2021-03-24_08-08-31.zip"
FINALIST_ATTACHMENTS="Finalist Attachments.zip"
CORRECTION_FILE="OTS_Corrections.csv"
ADMIN_REVIEW_CSV="Kellogg-AdminReview.csv"
......@@ -67,6 +68,7 @@ if [ ! -d "${DATA_DIR}" ] ; then
decrypt "${DATA_DIR}/${PROPOSALS_CSV}" "${ENCRYPTED_DIR}/${PROPOSALS_CSV}.gpg" "${GPG_PASSPHRASE}"
decrypt "${DATA_DIR}/${BASE_ATTACHMENTS}" "${ENCRYPTED_DIR}/${BASE_ATTACHMENTS}.gpg" "${GPG_PASSPHRASE}"
decrypt "${DATA_DIR}/${FINALIST_ATTACHMENTS}" "${ENCRYPTED_DIR}/${FINALIST_ATTACHMENTS}.gpg" "${GPG_PASSPHRASE}"
decrypt "${DATA_DIR}/${ADMIN_REVIEW_CSV}" "${ENCRYPTED_DIR}/${ADMIN_REVIEW_CSV}.gpg" "${GPG_PASSPHRASE}"
decrypt "${DATA_DIR}/${CORRECTION_FILE}" "${ENCRYPTED_DIR}/${CORRECTION_FILE}.gpg" "${GPG_PASSPHRASE}"
decrypt "${DATA_DIR}/${P2P_RANKING_CSV}" "${ENCRYPTED_DIR}/${P2P_RANKING_CSV}.gpg" "${GPG_PASSPHRASE}"
......@@ -78,6 +80,15 @@ if [ ! -d "${DATA_DIR}" ] ; then
mkdir -p $TMP_ATTACHMENTS_DIR
unzip -d ${TMP_ATTACHMENTS_DIR} ${DATA_DIR}/${BASE_ATTACHMENTS}
mv "${TMP_ATTACHMENTS_DIR}/"*"/Valid" $ATTACHMENTS_DIR
rm -rf $TMP_ATTACHMENTS_DIR
mkdir -p $TMP_ATTACHMENTS_DIR
unzip -d $TMP_ATTACHMENTS_DIR "${DATA_DIR}/${FINALIST_ATTACHMENTS}"
ls "$TMP_ATTACHMENTS_DIR/Finalist Attachments" | while read file ; do
NUM=${file/_*/};
mv -v $TMP_ATTACHMENTS_DIR/*/$NUM*/* ${ATTACHMENTS_DIR}/$NUM/
done
rm -rf $TMP_ATTACHMENTS_DIR
# Clean up large zips to save disk space
rm ${DATA_DIR}/${BASE_ATTACHMENTS}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment