deploy

#!/bin/bash

# Copyright (C) 2017, 2019, 2020  Open Tech Strategies, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

SCRIPT_DIR=`dirname "$BASH_SOURCE"`
COMMON_SCRIPT_DIR="${SCRIPT_DIR}/../../../etl/common/deploy"
source "${COMMON_SCRIPT_DIR}/utils"
source "${COMMON_SCRIPT_DIR}/init"

COMPETITION="RacialEquity2030"
LFC_DIR=`dirname "${0}"`

DATA_DIR="${BASE_DATA_DIR}/${COMPETITION}"

# Make sure we have python3
python3 -V > /dev/null 2>&1 || { echo "python3 required, aborting" ; exit 1; }

PROPOSALS_CSV="RE2020_Registration_Application_2021-03-24_06-29-20.csv"
P2P_RANKING_CSV="RE2020_Peer_Review_2021-05-13_14-27-30_No_Judge_Names.csv"
EP_RANKING_CSV="RE2020_Evaluation_Panel_2021-06-14_07-31-58.csv"
BASE_ATTACHMENTS="RE2020_Registration_Application_exports_2021-03-24_08-08-31.zip"
FINALIST_ATTACHMENTS="Finalist Attachments.zip"
CORRECTION_FILE="OTS_Corrections.csv"
ADMIN_REVIEW_CSV="Kellogg-AdminReview.csv"
WILDCARDS="wildcards.dat"

TMP_ATTACHMENTS_DIR="${DATA_DIR}/tmpattachments"
ATTACHMENTS_DIR="${DATA_DIR}/attachments/"
TDC_CONFIG_DIR="${DATA_DIR}/tdcconfig"
RUNNER="${LFC_DIR}"/compose-and-upload

if [ ! -d "${DATA_DIR}" ] ; then
  echo "Stage 0: Setting up data directory..."

  if [ "${OTS_DIR}" = "" ] ; then
    echo "ERROR: \$OTS_DIR is not set up"
    echo ""
    echo "See bureaucracy/onboarding about setting the OTS_DIR environment variable up"
    exit 1
  fi

  ENCRYPTED_DIR="${OTS_DIR}/clients/lever-for-change/torque-sites/${COMPETITION}/data/bigdata"
  if [ ! -d "${ENCRYPTED_DIR}" ] ; then
    echo "ERROR: it looks like your encrypted dir isn't checked out"
    echo ""
    echo "It was expected to be in:"
    echo "  $ENCRYPTED_DIR"
    echo ""
    echo "You will need to check out the subversion torque-sites data, and then"
    echo "run get-bigdata from ${OTS_DIR}/clients/lever-for-change/torque-sites/${COMPETITION}/data/"
    exit 1
  fi

  mkdir -p $DATA_DIR
  echo "Decrypting..."

  decrypt "${DATA_DIR}/${PROPOSALS_CSV}" "${ENCRYPTED_DIR}/${PROPOSALS_CSV}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${BASE_ATTACHMENTS}" "${ENCRYPTED_DIR}/${BASE_ATTACHMENTS}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${FINALIST_ATTACHMENTS}" "${ENCRYPTED_DIR}/${FINALIST_ATTACHMENTS}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${ADMIN_REVIEW_CSV}" "${ENCRYPTED_DIR}/${ADMIN_REVIEW_CSV}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${CORRECTION_FILE}" "${ENCRYPTED_DIR}/${CORRECTION_FILE}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${P2P_RANKING_CSV}" "${ENCRYPTED_DIR}/${P2P_RANKING_CSV}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${EP_RANKING_CSV}" "${ENCRYPTED_DIR}/${EP_RANKING_CSV}.gpg" "${GPG_PASSPHRASE}"
  decrypt "${DATA_DIR}/${WILDCARDS}" "${ENCRYPTED_DIR}/${WILDCARDS}.gpg" "${GPG_PASSPHRASE}"

  mkdir -p ${TDC_CONFIG_DIR}

  echo "Setting up attachments..."
  mkdir -p $TMP_ATTACHMENTS_DIR
  unzip -d ${TMP_ATTACHMENTS_DIR} ${DATA_DIR}/${BASE_ATTACHMENTS}
  mv "${TMP_ATTACHMENTS_DIR}/"*"/Valid" $ATTACHMENTS_DIR
  rm -rf $TMP_ATTACHMENTS_DIR

  mkdir -p $TMP_ATTACHMENTS_DIR
  unzip -d $TMP_ATTACHMENTS_DIR "${DATA_DIR}/${FINALIST_ATTACHMENTS}"
  ls "$TMP_ATTACHMENTS_DIR/Finalist Attachments" | while read file ; do
    NUM=${file/_*/};
    mv -v $TMP_ATTACHMENTS_DIR/*/$NUM*/* ${ATTACHMENTS_DIR}/$NUM/
  done
  rm -rf $TMP_ATTACHMENTS_DIR

  # Clean up large zips to save disk space
  rm ${DATA_DIR}/${BASE_ATTACHMENTS}
  rm -rf ${DATA_DIR}/${TMP_ATTACHMENTS_DIR}
fi

if ! echo "01adcee5467c6a4f12a2e8a0644b285d882c4c93  $DATA_DIR/$PROPOSALS_CSV" | sha1sum -c &> /dev/null ; then
  echo "ERROR: SHA of PROPOSALS_CSV did not match expectation!"
  echo "This means that we most likely got a new file, which may have the old"
  echo "errors in it (specifically the organization proposal 829)."
  echo "If we got a new file and there are no further errors, you"
  echo "can remove this check from torque-refresh!  If not, please"
  echo "make adjustments in a new file and update the sha."
  exit 1
fi

mkdir -p $TDC_CONFIG_DIR

${RUNNER} --proposals-csv="${DATA_DIR}/${PROPOSALS_CSV}" \
          --attachments-dir=${ATTACHMENTS_DIR} \
          --admin-review-csv="${DATA_DIR}/${ADMIN_REVIEW_CSV}" \
          --peer-to-peer-review="${DATA_DIR}/${P2P_RANKING_CSV}" \
          --expert-panel-review="${DATA_DIR}/${EP_RANKING_CSV}" \
          --correction-file="${DATA_DIR}/${CORRECTION_FILE}" \
          --wildcards="${DATA_DIR}/${WILDCARDS}" \
          $PARE \
          $CSV_ONLY \
          --tdc-config-dir="${TDC_CONFIG_DIR}"
if [ $? -ne 0 ]; then
		echo Failure!
		exit 1
fi

echo "Done creating wiki."