#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017, 2019, 2020 Open Tech Strategies, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
__doc__ = """\
Compose all of the Lever for Change Racial Equity 2030 Proposal CSV files.
Usage:
$ compose-and-upload \\
--proposals-csv=PROPOSALS_CSV \\
--admin-review-csv=ADMIN_REVIEW_CSV \\
--attachments-dir=ATTACHMENTS_DIR \\
--tdc-config-dir=TDC_CONFIG_DIR \\
--correction-file=CORRECTION_FILE \\
--pare=PARE \\
--csv-only
Command-line options:
--proposals-csv FILE FILE is a CSV file representing the bulk
of the proposal information
--attachments-dir DIR DIR is a directory for compose-csvs to look in for what attachments
will be uploaded to the torque wiki. It needs to have subdirectories
by proposal number.
--admin-review-csv FILE FILE is a CSV file representing which applications
in PROPOSALS_CSV should be included, as well as the Organization
information
--tdc-config-dir DIR DIR is the location for files that are the base configuration files
needed by TorqueDataConnect, and can be optionally, manually, put on
the torque wiki. We don't automatically do that because we want to
overwrite the configuration out there.
--correction-file FILE FILE is a csv of corrections to the main data. The header
must match the header of the original proposals file, and any
one of the columns must contain the review number. Then
the data from the correction file will override the
source data for output. There can be multiple correction
files, and each one overwrites the previous.
If the data cells have the empty string, no correction is applied.
--peer-to-peer-review FILE FILE is a CSV file with a many to one relationshp
between peers and the proposals they evaluated,
with the extra data being their evaluation
--expert-panel-review FILE FILE is a CSV file with a many to one relationshp
between experts and the proposals they evaluated,
with the extra data being their evaluation
--wildcards FILE FILE is a simle newline separated list of proposal keys
for the wildcard proposals.
--pare ARG If ARG is a number, reduce the number of items to 1/ARG. If
ARG begins with +, then ARG is a comma separated list of
keys to include. If ARG begins with @, then ARG is a
file with a list of keys to include. For both + and @,
the list of keys will be limited to only the ones provided.
--csv-only Only upload the created CSV file. Don't upload attachments or
create wiki pages. For use to speed up process when wiki has been
created already.
"""
from etl import competition, wiki, toc, tdc
import config
import getopt
import sys
import os
import csv
class LFCAnalysisAdder(competition.InformationAdder):
"""Adds the columns for the LFC Analysis, all empty because these
cells are there to be edited upon later"""
def column_names(self):
return [
"LFC Analysis: Short Description",
"LFC Analysis: Overview",
"LFC Analysis: Diversity, Equity and Inclusion",
"LFC Analysis: Funder Relationship History",
"LFC Analysis: Strength of Approach",
"LFC Analysis: Observations",
"LFC Analysis: Internet Scan",
]
def cell(self, proposal, column_name):
return ""
def main():
"""Compose the LFC input and emit it as html-ized csv."""
try:
opts, args = getopt.getopt(
sys.argv[1:],
"",
[
"proposals-csv=",
"tdc-config-dir=",
"admin-review-csv=",
"judge-evaluation-csv=",
"expert-panel-evaluation-csv=",
"application-data=",
"financial-sheets-dir=",
"attachments-dir=",
"correction-file=",
"peer-to-peer-review=",
"expert-panel-review=",
"wildcards=",
"pare=",
"csv-only",
],
)
except getopt.GetoptError as err:
sys.stderr.write("ERROR: '%s'\n" % err)
sys.exit(2)
proposals_csv = None
admin_review_csv = None
attachments_dir = None
tdc_config_dir = None
correction_file = None
peer_to_peer_review_csv = None
expert_panel_review_csv = None
wildcards = None
pare = None
csv_only = False
for o, a in opts:
if o == "--proposals-csv":
proposals_csv = a
elif o == "--pare":
pare = a
elif o == "--csv-only":
csv_only = True
elif o == "--tdc-config-dir":
tdc_config_dir = a
elif o == "--attachments-dir":
attachments_dir = a
elif o == "--admin-review-csv":
admin_review_csv = a
elif o == "--correction-file":
correction_file = a
elif o == "--peer-to-peer-review":
peer_to_peer_review_csv = a
elif o == "--expert-panel-review":
expert_panel_review_csv = a
elif o == "--wildcards":
wildcards = a
else:
sys.stderr.write("ERROR: unrecognized option '%s'\n" % o)
sys.exit(2)
if proposals_csv is None:
sys.stderr.write("ERROR: need --proposals-csv option.\n\n")
sys.stderr.write(__doc__)
sys.exit(1)
comp = competition.Competition(
proposals_csv, "RacialEquity2030", "Application #", pare
)
comp.add_supplemental_information(
competition.LinkedSecondSheet(
admin_review_csv, "Application #", [{"source_name": "Organization name"}]
)
)
correction_processor = competition.CorrectionData("Application #", correction_file)
for column in correction_processor.columns_affected():
comp.process_cells_special(column, correction_processor)
comp.filter_proposals(
competition.ColumnEqualsProposalFilter("Admin Review Status", "Not Applicable")
)
fix_cell_processor = competition.FixCellProcessor()
comp.process_all_cells_special(fix_cell_processor)
fix_cell_processor.report()
comp.process_cells_special(
"Organization name", competition.RemoveHTMLBRsProcessor()
)
comp.add_supplemental_information(
competition.EvaluationAdder(
"Peer to Peer",
peer_to_peer_review_csv,
app_col_name="Application #",
score_rank_normalized_col_name="Overall Score Rank Normalized ",
sum_of_scores_normalized_col_name="Sum Of Scores Normalized ",
trait_col_name="Trait",
score_normalized_col_name="Trait Score Normalized",
comments_col_name="Trait Judge Comment ",
comments_score_normalized_col_name="Trait Score Normalized",
)
)
comp.add_supplemental_information(
competition.EvaluationAdder(
"Panel",
expert_panel_review_csv,
app_col_name="Application #",
score_rank_normalized_col_name="Overall Score Rank Normalized ",
sum_of_scores_normalized_col_name="Sum Of Scores Normalized ",
trait_col_name="Trait",
score_normalized_col_name="Trait Score Normalized",
comments_col_name="Trait Judge Comment ",
comments_score_normalized_col_name="Trait Score Normalized",
)
)
comp.add_supplemental_information(LFCAnalysisAdder())
comp.process_cells_special("Project Title", competition.RemoveHTMLBRsProcessor())
comp.add_supplemental_information(competition.MediaWikiTitleAdder("Project Title"))
comp.add_supplemental_information(
competition.GlobalViewMediaWikiTitleAdder("RacialEquity2030", "Project Title")
)
comp.add_supplemental_information(
competition.StaticColumnAdder("Competition Name", "RacialEquity2030")
)
comp.process_cells_special("Priority Populations", competition.MultiLineProcessor())
comp.process_cells_special(
"Sustainable Development Goals",
competition.MultiLineProcessor(split_string="|"),
)
comp.process_cells_special(
"Sustainable Development Goals",
competition.SustainableDevelopmentGoalProcessor(
competition.SustainableDevelopmentGoalProcessor.official_sdgs
),
)
comp.process_cells_special("Total Projected Costs", competition.NumberCommaizer())
comp.process_cells_special("Budget Data", competition.BudgetTableProcessor())
comp.process_cells_special(
"Annual Operating Budget",
competition.AnnualBudgetProcessor(
{
competition.AnnualBudget.LESS_THAN_1_MIL: "Less than $1 Million",
competition.AnnualBudget.BETWEEN_1_MIL_AND_5_MIL: "$1.0 to 5.0 Million",
competition.AnnualBudget.BETWEEN_5_MIL_AND_10_MIL: "$5.1 to 10 Million",
competition.AnnualBudget.BETWEEN_10_MIL_AND_25_MIL: "$10.1 to 25 Million",
competition.AnnualBudget.BETWEEN_25_MIL_AND_50_MIL: "$25.1 to 50 Million",
competition.AnnualBudget.BETWEEN_50_MIL_AND_100_MIL: "$50.1 to 100 Million",
competition.AnnualBudget.BETWEEN_100_MIL_AND_500_MIL: "$100.1 to 500 Million",
competition.AnnualBudget.BETWEEN_500_MIL_AND_1_BIL: "$500.1 Million to $1 Billion",
competition.AnnualBudget.MORE_THAN_1_BIL: "$1 Billion +",
}
),
)
attachments = competition.BasicAttachments(
comp.sorted_proposal_keys, attachments_dir
)
comp.add_supplemental_information(attachments)
comp.sort("Panel Overall Score Rank Normalized", True)
list_toc = toc.ListToc("All_Proposals")
list_toc.proposal_formatter = toc.WikiTableTocProposalFormatter(
[
{
"name": "Organization name",
"heading": "Organization",
},
{
"name": "Project Title",
"heading": "Title",
"link": True,
},
{
"name": "Application #",
"heading": "ID #",
"right_aligned": True,
},
{
"name": "Overall Rank",
"heading": "Rank",
"right_aligned": True,
},
]
)
comp.add_toc(list_toc)
comp.add_toc(
toc.GenericToc("Funders", ["FUNDER 1 Name", "FUNDER 2 Name", "FUNDER 3 Name"])
)
comp.add_toc(toc.GenericMultiLineToc("Populations", "Priority Populations"))
comp.add_toc(
toc.GenericMultiLineToc(
"Sustainable_Development_Goals",
"Sustainable Development Goals",
competition.SustainableDevelopmentGoalProcessor.official_sdgs,
)
)
comp.add_toc(toc.GenericToc("Subject_Areas", "Primary Subject Area"))
comp.add_toc(
toc.GeographicToc(
"Current_Work_Locations",
[
[
"Location of Current Work #1 Country",
"Location of Current Work #1 State / Provence",
],
[
"Location of Current Work #2 Country",
"Location of Current Work #2 State / Provence",
],
[
"Location of Current Work #3 Country",
"Location of Current Work #3 State / Provence",
],
[
"Location of Current Work #4 Country",
"Location of Current Work #4 State / Provence",
],
[
"Location of Current Work #5 Country",
"Location of Current Work #5 State / Provence",
],
],
)
)
comp.add_toc(
toc.GeographicToc(
"Future_Work_Locations",
[
[
"Location of Future Work #1 Country",
"Location of Future Work #1 State / Provence",
],
[
"Location of Future Work #2 Country",
"Location of Future Work #2 State / Provence",
],
[
"Location of Future Work #3 Country",
"Location of Future Work #3 State / Provence",
],
[
"Location of Future Work #4 Country",
"Location of Future Work #4 State / Provence",
],
[
"Location of Future Work #5 Country",
"Location of Future Work #5 State / Provence",
],
],
)
)
comp.add_toc(toc.AnnualBudgetToc("Annual Operating Budget"))
comp.process_tocs()
if tdc_config_dir is not None:
tdc.AllProposals(comp).generate(tdc_config_dir)
tdc.ValidProposals(comp, "Admin Review Status", "Valid").generate(
tdc_config_dir
)
tdc.AllColumns(comp).generate(tdc_config_dir)
tdc.ProcessedSpreadsheet(comp).generate(tdc_config_dir)
my_wiki = wiki.WikiSession(
config.username, config.password, comp.name, config.wiki_url
)
my_wiki.csv_only = csv_only
my_wiki.upload_sheet(comp)
my_wiki.upload_attachments(attachments.attachments)
for proposal in comp.proposals.values():
my_wiki.create_page(
"Evaluations of %s" % proposal.cell("MediaWiki Title"),
"{{ #tdcrender:RacialEquity2030/id/"
+ proposal.key()
+ ".mwiki|Evaluations }}",
)
for key in comp.sorted_proposal_keys[0:20]:
proposal = comp.proposals[key]
my_wiki.create_page(
"LFC Analysis of %s" % proposal.cell("MediaWiki Title"),
"{{ #tdcrender:RacialEquity2030/id/" + key + ".mwiki|LFCAnalysis }}",
)
with open(wildcards) as f:
for key in f.read().splitlines():
if key in comp.proposals:
proposal = comp.proposals[key]
my_wiki.create_page(
"LFC Analysis of %s" % proposal.cell("MediaWiki Title"),
"{{ #tdcrender:RacialEquity/id/" + key + ".mwiki|LFCAnalysis }}",
)
if __name__ == "__main__":
main()