compose-and-upload

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017, 2019, 2020 Open Tech Strategies, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

__doc__ = """\
Compose all of the Lever for Change 2017 Proposal CSV files.

Usage:

  $ compose-and-upload \\
       --proposals-csv=PROPOSALS_CSV \\
       --attachments-dir=ATTACHMENTS_DIR \\
       --tdc-config-dir=TDC_CONFIG_DIR \\
       --correction-file=CORRECTION_FILE \\
       --pare=PARE \\
       --csv-only

Command-line options:
  --proposals-csv FILE            FILE is a CSV file representing the bulk
                                  of the proposal information

  --attachments-dir DIR           DIR is a directory for compose-csvs to look in for what attachments
                                  will be uploaded to the torque wiki.  It needs to have subdirectories
                                  by proposal number.

  --tdc-config-dir DIR            DIR is the location for files that are the base configuration files
                                  needed by TorqueDataConnect, and can be optionally, manually, put on
                                  the torque wiki.  We don't automatically do that because we want to
                                  overwrite the configuration out there.

  --pare ARG                      If ARG is a number, reduce the number of items to 1/ARG.  If
                                  ARG begins with +, then ARG is a comma separated list of
                                  keys to include.  If ARG begins with @, then ARG is a
                                  file with a list of keys to include.  For both + and @,
                                  the list of keys will be limited to only the ones provided.

  --correction-file FILE          FILE is a csv of corrections to the main data.  The header
                                  must match the header of the original proposals file, and any
                                  one of the columns must contain the review number.  Then
                                  the data from the correction file will override the
                                  source data for output.  There can be multiple correction
                                  files, and each one overwrites the previous.

                                  If the data cells have the empty string, no correction is applied.

  --csv-only                      Only upload the created CSV file.  Don't upload attachments or
                                  create wiki pages.  For use to speed up process when wiki has been
                                  created already.
"""

from etl import competition, wiki, toc, tdc
import config
import getopt
import sys
import os


def main():
    """Compose the LFC input and emit it as html-ized csv."""
    try:
        opts, args = getopt.getopt(
            sys.argv[1:],
            "",
            [
                "proposals-csv=",
                "tdc-config-dir=",
                "attachments-dir=",
                "pare=",
                "csv-only",
                "correction-file=",
            ],
        )
    except getopt.GetoptError as err:
        sys.stderr.write("ERROR: '%s'\n" % err)
        sys.exit(2)

    proposals_csv = None
    attachments_dir = None
    tdc_config_dir = None
    pare = None
    csv_only = False
    correction_files = []
    pare = None
    csv_only = False
    for o, a in opts:
        if o == "--proposals-csv":
            proposals_csv = a
        elif o == "--pare":
            pare = a
        elif o == "--csv-only":
            csv_only = True
        elif o == "--tdc-config-dir":
            tdc_config_dir = a
        elif o == "--attachments-dir":
            attachments_dir = a
        elif o == "--correction-file":
            correction_files.append(a)
        else:
            sys.stderr.write("ERROR: unrecognized option '%s'\n" % o)
            sys.exit(2)

    if proposals_csv is None:
        sys.stderr.write("ERROR: need --proposals-csv option.\n\n")
        sys.stderr.write(__doc__)
        sys.exit(1)

    comp = competition.Competition(
        proposals_csv, "LFC100Change2017", "Review_Number", pare
    )
    comp.add_supplemental_information(
        competition.MediaWikiTitleAdder("Registered Organization Name")
    )
    comp.add_supplemental_information(
        competition.GlobalViewMediaWikiTitleAdder(
            "100Change2017", "Registered Organization Name"
        )
    )
    comp.add_supplemental_information(
        competition.StaticColumnAdder("Competition Name", "100Change2017")
    )

    fix_cell_processor = competition.FixCellProcessor()
    comp.process_all_cells_special(fix_cell_processor)
    fix_cell_processor.report()

    for correction_file in correction_files:
        correction_processor = competition.CorrectionData(
            "Review_Number", correction_file
        )
        for column in correction_processor.columns_affected():
            comp.process_cells_special(column, correction_processor)

    attachments = competition.RegexSpecifiedAttachments(
        comp.sorted_proposal_keys, attachments_dir
    )
    attachments.specify_new_column("^\\d*_mou", "MOU Attachment")
    attachments.specify_new_column("^\\d*_team", "Team Attachment")
    attachments.specify_new_column("financial", "Financials Attachment")
    comp.add_supplemental_information(attachments)

    comp.sort("Registered Organization Name")

    comp.add_toc(toc.GenericToc("Topic_TOC", "Primary Thematic Area"))

    comp.process_tocs()

    if tdc_config_dir is not None:
        tdc.AllProposals(comp).generate(tdc_config_dir)
        tdc.AllColumns(comp).generate(tdc_config_dir)
        tdc.ProcessedSpreadsheet(comp).generate(tdc_config_dir)

    my_wiki = wiki.WikiSession(
        config.username, config.password, comp.name, config.wiki_url
    )
    my_wiki.csv_only = csv_only
    my_wiki.upload_sheet(comp)
    my_wiki.upload_attachments(attachments.attachments)


if __name__ == "__main__":
    main()