Newer
Older
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017, 2019, 2020 Open Tech Strategies, LLC
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
__doc__ = """\
Compose all of the Lever for Change <LONG_COMPETITION_NAME> Proposal CSV files.
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
Usage:
$ compose-and-upload \\
--proposals-csv=PROPOSALS_CSV \\
--attachments-dir=ATTACHMENTS_DIR \\
--tdc-config-dir=TDC_CONFIG_DIR \\
--pare=PARE \\
--csv-only
Command-line options:
--proposals-csv FILE FILE is a CSV file representing the bulk
of the proposal information
--attachments-dir DIR DIR is a directory for compose-csvs to look in for what attachments
will be uploaded to the torque wiki. It needs to have subdirectories
by proposal number.
--tdc-config-dir DIR DIR is the location for files that are the base configuration files
needed by TorqueDataConnect, and can be optionally, manually, put on
the torque wiki. We don't automatically do that because we want to
overwrite the configuration out there.
--pare ARG If ARG is a number, reduce the number of items to 1/ARG. If
ARG begins with +, then ARG is a comma separated list of
keys to include. If ARG begins with @, then ARG is a
file with a list of keys to include. For both + and @,
the list of keys will be limited to only the ones provided.
--csv-only Only upload the created CSV file. Don't upload attachments or
create wiki pages. For use to speed up process when wiki has been
created already.
"""
from etl import competition, wiki, toc, tdc
import config
import getopt
import sys
import os
import csv
def main():
"""Compose the LFC input and emit it as html-ized csv."""
try:
opts, args = getopt.getopt(
sys.argv[1:],
"",
[
"proposals-csv=",
"tdc-config-dir=",
"attachments-dir=",
"pare=",
"csv-only",
],
)
except getopt.GetoptError as err:
sys.stderr.write("ERROR: '%s'\n" % err)
sys.exit(2)
proposals_csv = None
attachments_dir = None
tdc_config_dir = None
pare = None
csv_only = False
for o, a in opts:
if o == "--proposals-csv":
proposals_csv = a
elif o == "--pare":
pare = a
elif o == "--csv-only":
csv_only = True
elif o == "--tdc-config-dir":
tdc_config_dir = a
elif o == "--attachments-dir":
attachments_dir = a
else:
sys.stderr.write("ERROR: unrecognized option '%s'\n" % o)
sys.exit(2)
if proposals_csv is None:
sys.stderr.write("ERROR: need --proposals-csv option.\n\n")
sys.stderr.write(__doc__)
sys.exit(1)
comp = competition.Competition(
proposals_csv, "<COMPETITION_NAME>", "Application #", pare
fix_cell_processor = competition.FixCellProcessor()
comp.process_all_cells_special(fix_cell_processor)
fix_cell_processor.report()
comp.process_cells_special(
"Organization Name", competition.RemoveHTMLBRsProcessor()
)
comp.process_cells_special("Project Title", competition.RemoveHTMLBRsProcessor())
comp.add_supplemental_information(competition.MediaWikiTitleAdder("Project Title"))
comp.add_supplemental_information(
competition.GlobalViewMediaWikiTitleAdder("<COMPETITION_NAME>", "Project Title")
comp.filter_proposals(
competition.ColumnEqualsProposalFilter("Admin Review Status", "Not submitted")
)
comp.process_cells_special("Priority Populations", competition.MultiLineProcessor())
comp.process_cells_special("Total Projected Costs", competition.NumberCommaizer())
comp.process_cells_special("Budget data", competition.BudgetTableProcessor())
comp.process_cells_special(
"Annual Operating Budget",
competition.AnnualBudgetProcessor(
{
competition.AnnualBudget.LESS_THAN_1_MIL: "Less than $1 Million",
competition.AnnualBudget.BETWEEN_1_MIL_AND_5_MIL: "$1.0 to 5.0 Million",
competition.AnnualBudget.BETWEEN_5_MIL_AND_10_MIL: "$5.1 to 10 Million",
competition.AnnualBudget.BETWEEN_10_MIL_AND_25_MIL: "$10.1 to 25 Million",
competition.AnnualBudget.BETWEEN_25_MIL_AND_50_MIL: "$25.1 to 50 Million",
competition.AnnualBudget.BETWEEN_50_MIL_AND_100_MIL: "$50.1 to 100 Million",
competition.AnnualBudget.BETWEEN_100_MIL_AND_500_MIL: "$100.1 to 500 Million",
competition.AnnualBudget.BETWEEN_500_MIL_AND_1_BIL: "$500.1 Million to $1 Billion",
competition.AnnualBudget.MORE_THAN_1_BIL: "$1 Billion +",
}
),
)
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
attachments = competition.BasicAttachments(
comp.sorted_proposal_keys, attachments_dir
)
comp.add_supplemental_information(attachments)
comp.sort("Organization Name")
list_toc = toc.ListToc("All_Proposals")
list_toc.proposal_formatter = toc.WikiTableTocProposalFormatter(
[
{
"name": "Organization Name",
"heading": "Organization",
},
{
"name": "Project Title",
"heading": "Title",
"link": True,
},
{
"name": "Application #",
"heading": "ID #",
"right_aligned": True,
},
]
)
comp.add_toc(list_toc)
comp.add_toc(toc.GenericMultiLineToc("Populations", "Priority Populations"))
comp.add_toc(toc.GenericToc("Regions", "Application Data Region"))
comp.add_toc(toc.AnnualBudgetToc("Annual Operating Budget"))
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
comp.add_toc(
toc.GenericToc(
"Number_of_Employees",
"Number of Employees",
[
"Fewer than 10 Full-time Employees",
"10 to 25 Full-time Employees",
"26 to 50 Full-time Employees",
"51 to 100 Full-time Employees",
"101 to 300 Full-time Employees",
"301 to 500 Full-time Employees",
"501 to 1,000 Full-time Employees",
"1,000+ Full-time Employees",
],
)
)
comp.add_toc(
toc.GeographicToc(
"Current_Work_Locations",
[
[
"Location of Current Work #1 Country",
"Location of Current Work #1 State /Province",
],
[
"Location of Current Work #2 Country",
"Location of Current Work #2 State /Province",
],
[
"Location of Current Work #3 Country",
"Location of Current Work #3 State /Province",
],
[
"Location of Current Work #4 Country",
"Location of Current Work #4 State /Province",
],
[
"Location of Current Work #5 Country",
"Location of Current Work #5 State /Province",
],
],
)
)
comp.add_toc(
toc.GeographicToc(
"Future_Work_Locations",
[
[
"Location of Future Work #1 Country",
"Location of Future Work #1 State / Province",
],
[
"Location of Future Work #2 Country",
"Location of Future Work #2 State /Province",
],
[
"Location of Future Work #3 Country",
"Location of Future Work #3 State /Province",
],
[
"Location of Future Work #4 Country",
"Location of Future Work #4 State /Province",
],
[
"Location of Future Work #5 Country",
"Location of Future Work #5 State /Province",
],
],
)
)
comp.process_tocs()
if tdc_config_dir is not None:
tdc.AllProposals(comp).generate(tdc_config_dir)
tdc.ValidProposals(comp, "Admin Review Status", "Valid").generate(
tdc_config_dir
)
tdc.AllColumns(comp).generate(tdc_config_dir)
tdc.ProcessedSpreadsheet(comp).generate(tdc_config_dir)
my_wiki = wiki.WikiSession(
config.username, config.password, comp.name, config.wiki_url
)
my_wiki.csv_only = csv_only
my_wiki.upload_sheet(comp)
my_wiki.upload_attachments(attachments.attachments)
if __name__ == "__main__":
main()