diff --git a/competitions/100Change2025/ansible/roles/100Change2025/files/LLM_Prompts.yml b/competitions/100Change2025/ansible/roles/100Change2025/files/LLM_Prompts.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ee5eb466fe0aecd0d4d83e056470d12913fafc77
--- /dev/null
+++ b/competitions/100Change2025/ansible/roles/100Change2025/files/LLM_Prompts.yml
@@ -0,0 +1,174 @@
+title: LLM LFC Analysis
+prompts:
+ - title: Project Overview
+ prompt: |
+ Please summarize in 3 bullet points the key
+ details of the proposal in the following areas: The Challenge, The
+ Solution and The Plan.
+ prompts:
+ - title: The Challenge
+ prompt: |
+ Please summarize in 2-3 sentences the challenge the proposal
+ tackles. This should only include facts and no analysis.
+ blocks:
+ - the_problem
+ - title: The Solution
+ prompt: |
+ Please summarize in 2-3 sentences describing the solution and
+ impact. This should include what, how, and with whom. This
+ should only include facts and no analysis.
+ blocks:
+ - the_solution
+ - title: The Plan
+ prompt: |
+ Please summarize in 2-3 sentences how the award will be used
+ and what will be achieved. This should only include facts from
+ the application and no analysis.
+
+ Include the following, if available a breakdown of the award
+ e.g., How much is earmarked for staff, the solution, DEI,
+ evaluation, etc. What are the targets, outcomes, and results?
+ Expected direct and/or indirect reach of key stakeholder
+ groups.
+ - title: Strength of Approach
+ prompt: |
+ Please summarize in 3 bullet points
+ the key strengths of the proposal in the following areas:
+ People, Solution and Process.
+ prompts:
+ - title: People
+ prompt: |
+ Please summarize in 2-4 sentences the strengths, assets, and
+ experience the named leaders and any partners bring to this
+ project. How has the organization
+ involved stakeholders in solution development? How are
+ diversity, equity, and inclusion embedded in the solution and
+ lead organization? Please use evidence and name why this
+ matters, is unique or uncommon.
+ - title: Solution
+ prompt: |
+ Please summarize in 2-4 sentences the strengths of the
+ proposed solution. Is the problem compelling? Is the market
+ addressable? Does the solution make sense? What is the 3-5
+ year impact goal? Is the goal specific and ambitious?
+ What is the enduring impact, beyond 5 years? Please use
+ evidence and name why this matters, is unique or uncommon.
+ blocks:
+ - the_solution
+ - panel_comments
+ - title: Process
+ prompt: |
+ Please summarize in 2-4 sentences the strengths of the
+ proposed process. Does the organization have a credible path
+ to achieve its goals? What is the organization’s approach to
+ scale? How does this solution solve a specific problem while
+ changing systems? Does the budget support the
+ implementation plan? How does the solution use data to measure,
+ evaluate, learn, and adapt? Please use evidence
+ and name why this matters, is unique or uncommon.
+ blocks:
+ - the_solution
+ - panel_comments
+ - title: Potential Challenges
+ prompt: |
+ Please summarize in 3 bullet points the
+ potential challenges of the proposal. Each bullet point should address
+ a specific area that can include: Impact, Basis in Evidence,
+ Feasibility, Durability and Justice.
+ prompts:
+ - title: Impact
+ prompt: |
+ Please summarize in 2-4 sentences any potential challenges with
+ the impact of the proposed solution. Please use evidence and
+ name why this is an important consideration.
+ blocks:
+ - projected_impact
+ - panel_comments
+ - title: Basis in Evidence
+ prompt: |
+ Please summarize in 2-4 sentences any potential challenges with
+ the evidence based nature of the proposed solution. Please use
+ evidence and name why this is an important consideration.
+ blocks:
+ - the_solution
+ - panel_comments
+ - title: Feasibility
+ prompt: |
+ Please summarize in 2-4 sentences any potential challenges with
+ the feasibility of the proposed solution. Please use evidence
+ and name why this is an important consideration.
+ blocks:
+ - the_solution
+ - panel_comments
+ - title: Durability
+ prompt: |
+ Please summarize in 2-4 sentences any potential challenges with
+ the durability of the proposed solution. Please use evidence
+ and name why this is an important consideration.
+ blocks:
+ - the_solution
+ - panel_comments
+ - title: Justice
+ prompt: |
+ Please summarize in 2-4 sentences any potential challenges with
+ the justice of the proposed solution. Please use evidence and
+ name why this is an important consideration.
+ blocks:
+ - the_solution
+ - panel_comments
+ - title: Diversity, Equity, Inclusion, and Accessibility
+ prompt: |
+ Please summarize in 3 bullet points
+ the key details of the proposal as they relate to Diversity,
+ Equity, Inclusion, and Accessibility in the following areas:
+ Leadership, Community Engagement and Accessibility.
+ prompts:
+ - title: Leardership
+ prompt: |
+ Please summarize in 1-3 sentences key details of the
+ leadership of the proposed solution through the lens of
+ diversity, equity, and inclusion. Please use
+ facts and provide no analysis.
+ - title: Community Engagement
+ prompt: |
+ Please summarize in 1-3 sentences key details about the
+ of the community engagement of the proposed solution through
+ the lens diversity, equity, and inclusion. Please use facts
+ and provide no analysis.
+ - title: Accessibility
+ prompt: |
+ If the proposal include plans or funds
+ for accessibility, please summarize the proposed
+ solution or accomodation, including the budget if there is
+ one.
+ - title: Reputational Risks
+ prompt: |
+ Please summarize in less than 3 bullet points
+ any potential reputational risks contained in the provided search
+ results. If no reputational risks are surfaced, please state that. Please don't
+ suggest reputational risks if not directly stated.
+ prompts:
+ - title: Reputational Risks
+ prompt: |
+ Please summarize in 3 bullet points with 2-4 sentences each any
+ potential reputational risks associated surfaced in the provided search
+ results. Please use evidence and name why this is an important consideration.
+ If no reputational risks are surfaced, please state that. Please don't
+ suggest reputational risks if not directly stated.
+ blocks:
+ - quick_pitch
+ - title: Other Considerations
+ prompt: |
+ Please summarize the proposal in 4-6 bullet points
+ highlighting review and Peer and Expert Review comments that pose
+ questions that remain about the proposal and are important
+ considerations.
+ prompts:
+ - title: Other Considerations
+ prompt: |
+ Please summarize the proposal in 4-6 reflections highlighting
+ review and Peer and Expert Review comments that pose questions
+ that remain about the proposal and are important
+ considerations.
+ blocks:
+ - panel_comments
diff --git a/competitions/GFC2024/ansible/roles/GFC2024/files/LLM_Prompts.yml b/competitions/GFC2024/ansible/roles/GFC2024/files/LLM_Prompts.yml
new file mode 100644
index 0000000000000000000000000000000000000000..8611667ee919d4b29baa91b0daf51c83a63a17f4
--- /dev/null
+++ b/competitions/GFC2024/ansible/roles/GFC2024/files/LLM_Prompts.yml
@@ -0,0 +1,169 @@
+title: LLM LFC Analysis
+prompts:
+ - title: The Challenge
+ prompt: |
+ Please summarize in 2-3 sentences the challenge the proposal
+ tackles. This should only include facts and no analysis.
+ blocks:
+ - the_problem
+ - title: The Solution
+ prompt: |
+ Please provide 2-3 sentence summary describing the solution and impact, including what,
+ how, and with whom. This should only include facts from the application and no analysis.
+ blocks:
+ - the_solution
+ - title: Implementation Plan
+ prompt: |
+ Please provide a two bullet point summary of the implementation plan and
+ specific targets, outcomes and results.
+ prompts:
+ - title: Implementation Plan
+ prompt: |
+ Please provide a 5 sentence summary of the implementation plan.
+ - title: Targets, Outcomes, and Results
+ prompt: |
+ Please provide a 5 sentence summary of specific targets, outcomes or results mentioned in the proposal.
+ - title: Partnerships
+ prompt: |
+ Please list any partners involved in the proposed project, separated by commas.
+ If the proposal does not involve any partners, or if no text from the proposal
+ is provided, please write “Not Applicableâ€.
+ blocks:
+ - partners
+ - title: The Three Futures
+ prompt: |
+ Summarize in bullet point form the Futures the team identifies in its application.
+ In each bullet point, describe how the project aims to address the specific Future in 1-3 sentences.
+ prompts:
+ - title: The Future of Energy Transition
+ prompt: |
+ Does this project align with the The Future of Energy Transition? If so, please describe how. How
+ does the team plan to address it? If not, please write “Does not align with The Future of Energy
+ Transition.â€
+ - title: The Future of Environmental Change along the Coast
+ prompt: |
+ Does this project align with the The Future of Environmental Change along the Coast? If so, please describe how.
+ How does the team plan to address it? If not, please write “Does not align with The Future of
+ Environmental Change along the Coast.â€
+ - title: The Future of Healthy and Resilient Communities
+ prompt: |
+ Does this project align with the The Future of Healthy and Resilient Communities? If so, please describe how.
+ How does the team plan to address it? If not, please write “Does not align with The Future of
+ Healthy and Resilient Communities.â€
+ - title: Strength of Approach
+ prompt: |
+ Please summarize the following proposal, highlighting in bullet point form
+ 3-5 key strengths of the application, and taking into consideration the following:
+ prompts:
+ - title: Impactful
+ prompt: |
+ Does the proposal address a critical issue affecting the Gulf Coast
+ region? Will the proposal have a transformative and long-lasting impact that
+ supports a safe, resilient, and sustainable future for those who call the region
+ home?
+ - title: Bridging Knowledge to Action
+ prompt: |
+ Does the proposal effectively bridge knowledge to
+ action? Does the proposal apply, translate, or communicate sciences, engineering,
+ or medical information in a way that will result in a safe, resilient, sustainable
+ future for the Gulf and those who call the Gulf Region home?
+ - title: Innovative
+ prompt: |
+ Does the proposal offer a new strategy, approach, process, or
+ partnership that will more effectively or efficiently support people in the Gulf
+ Coast region to address, understand, anticipate, and/or adapt to current or future
+ challenges?
+ - title: Inclusive
+ prompt: |
+ Does the proposal authentically represent, engage, and increase the
+ participation of people, communities, sectors, and demographic groups in which it
+ will work? Does the proposal demonstrate an understanding of historical context
+ or the challenges it seeks to address? Will the proposal support meaningful,
+ diverse, and mutually beneficial partnerships and collaborations?
+ - title: Three Futures
+ prompt: |
+ Does the proposal align with the three Futures? They are:
+ The Future of Energy Transition
+ The Future of Environmental Change along the Coast
+ The Future of Healthy and Resilient Communities
+ - title: Scientific Rigor
+ prompt: |
+ Does the proposal demonstrate scientific rigor?
+ Does the proposal use sound scientific methods and data to support its claims?
+ Does the proposal demonstrate a clear understanding of the scientific literature
+ and context in which it is working?
+ - title: Panel Strengths
+ prompt: |
+ What are some pecific strengths that have been brought-up by reviewers
+ (both Participatory and Evaluation Panel reviewers)?
+ blocks:
+ - panel_comments
+ - title: Potential Challenges
+ prompt: |
+ Please summarize the following proposal, highlighting in bullet point form
+ 3-5 key weaknesses/areas of improvement, taking into consideration the following:
+
+ Please follow the format:
+ 2-5-word phrase: 2-4 sentences explaining the potential challenge, citing evidence, and
+ naming why it is an important consideration.
+ prompts:
+ - title: Impactful
+ prompt: |
+ Does the proposal address a critical issue affecting the Gulf Coast
+ region? Will the proposal have a transformative and long-lasting impact that
+ supports a safe, resilient, and sustainable future for those who call the region
+ home?
+ - title: Bridging Knowledge to Action
+ prompt: |
+ Does the proposal effectively bridge knowledge to
+ action? Does the proposal apply, translate, or communicate sciences, engineering,
+ or medical information in a way that will result in a safe, resilient, sustainable
+ future for the Gulf and those who call the Gulf Region home?
+ - title: Innovative
+ prompt: |
+ Does the proposal offer a new strategy, approach, process, or
+ partnership that will more effectively or efficiently support people in the Gulf
+ Coast region to address, understand, anticipate, and/or adapt to current or future
+ challenges?
+ - title: Inclusive
+ prompt: |
+ Does the proposal authentically represent, engage, and increase the
+ participation of people, communities, sectors, and demographic groups in which it
+ will work? Does the proposal demonstrate an understanding of historical context
+ or the challenges it seeks to address? Will the proposal support meaningful,
+ diverse, and mutually beneficial partnerships and collaborations?
+ - title: Three Futures
+ prompt: |
+ Does the proposal align with the three Futures? They are:
+ The Future of Energy Transition
+ The Future of Environmental Change along the Coast
+ The Future of Healthy and Resilient Communities
+ - title: Scientific Rigor
+ prompt: |
+ Does the proposal demonstrate scientific rigor?
+ Does the proposal use sound scientific methods and data to support its claims?
+ Does the proposal demonstrate a clear understanding of the scientific literature
+ and context in which it is working?
+ - title: Panel Weaknesses
+ prompt: |
+ What are some pecific weaknesses that have been brought-up by reviewers
+ (both Participatory and Evaluation Panel reviewers)?
+ blocks:
+ - panel_comments
+ - title: Reputational Risks
+ prompt: |
+ Please summarize in less than 3 bullet points any potential reputational risks contained
+ in the provided search results. Please don't invent reputational risks if not directly
+ stated.
+
+ If {Organization Name} is a university, only report information about the
+ specific center, lab, etc. that is most directly responsible for the application.
+
+ If multiple credible sources report a specific issue about {Organization Name}
+ respond with 2-5 word heading for each issue found, followed by "Multiple articles and publications
+ have raised this concern about {Organization Name}." Then explain the issue in 2-4 sentences,
+ and cite sources.
+
+ If nothing concerning surfaced respond with "No concerning news on this organizations was uncovered."
+ blocks:
+ - quick_pitch
diff --git a/roles/torque-llm-bridge/files/LLMProposal b/roles/torque-llm-bridge/files/LLMProposal
index 9445fafb3f1f23f1ae3b0a01ab65e289eb2647e7..88da2b418c8ed9c3273ed4c4fb5b47c4339d7cd8 100644
--- a/roles/torque-llm-bridge/files/LLMProposal
+++ b/roles/torque-llm-bridge/files/LLMProposal
@@ -68,12 +68,18 @@
## Team Collaboration: Partnership Status
{{ convert_md_to_mwiki(proposal['Team Collaboration']) }}
{% endif -%}
+
+{% block partners %}
+
{% if 'Key Partners' in proposal and proposal['Key Partners'] -%}
## Partners
{% for key_partner in proposal['Key Partners'] %}
* {{ key_partner['Name'] }}
{% endfor %}
{% endif -%}
+
+{% endblock partners %}
+
{% if 'Why Your Team' in proposal and proposal['Why Your Team'] -%}
## Why Your Team
{{ convert_md_to_mwiki(proposal['Why Your Team']) }}
@@ -202,7 +208,7 @@ ${{ commaize_number(proposal['Total Projected Costs']) }}
## Budget
{% for line in proposal['Budget Data'] -%}
-* {{ line['description'] }}: ${{ commaize_number(line['amount']) }}
+* {{ line['description'] }}
{% endfor %}
{% endif -%}
@@ -448,10 +454,7 @@ ${{ commaize_number(proposal['Total Projected Costs']) }}
{% endif -%}
{% endfor -%}
{% endif -%}
-{% if 'Source' in proposal and proposal['Source'] -%}
-## Source
-{{ convert_md_to_mwiki(proposal['Source']) }}
-{% endif -%}
+
{% if 'Bibliography' in proposal and proposal['Bibliography'] %}
## Bibliography
diff --git a/scripts/llm/analysis_support.py b/scripts/llm/analysis_support.py
index e2c7a8d86b84d083e2b2fd72e3bbdb53c30312d7..96309c0d26b469bb81e929dfa8986d4041be13d6 100755
--- a/scripts/llm/analysis_support.py
+++ b/scripts/llm/analysis_support.py
@@ -1,33 +1,50 @@
-#!/usr/bin/env python3
+#!/usr/bin/env pipx run
+
+# PEP 723 inline script metadata https://packaging.python.org/en/latest/specifications/inline-script-metadata/
+# deps auto-install in dedicated env on script execution with `pipx run`` or `uv run``
+# pipx is now the default script runner in the shebang, but
+# manual Python env setup still works, just run:
+# `$ python -m venv venv`
+# `$ source ./venv/bin/activate`
+# `$ pip3 install` the following:
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+# "jinja2",
+# "nltk",
+# "pydantic",
+# "pyyaml",
+# "requests",
+# "rich",
+# "torqueclient==0.6.13",
+# "typer",
+# ]
+# ///
"""
Description:
This script generates Analysis Support for proposals in a specified
competition.
-Prerequisites:
- 1. Optionally, set up a `config.py` file to provide configuration
- - copy `config.py.tmpl` to `config.py` and fill in the values
- 2. Install dependencies:
- - `torqueclient`, `jinja2`, `requests`, `nltk` Python libraries
- - `pandoc`
+Pre-requisites:
+ 1. A Python script runner like pipx https://pipx.pypa.io/latest/installation/
+ also compatible with uv run https://docs.astral.sh/uv/guides/scripts/
+ 2. Optionally, set up a `config.py` file to provide configuration
+ - e.g. `cp config.py.tmpl config.py` and fill in the values
Installation:
- $ python -m venv venv
- $ source ./venv/bin/activate
- $ pip3 install torqueclient jinja2 requests nltk
- $ sudo apt-get install pandoc
+ 1. Install pandoc, e.g. with `sudo apt-get install pandoc` or `brew install pandoc`
Usage:
- $ ./analysis_support.py [--dry-run] [--competition COMPETITION]
- [--proposals PROPOSALS]
+ $ ./analysis_support.py [--competition COMPETITION]
+ [--proposals PROPOSALS] [--dry-run]
Options:
--dry-run Output the response from the LLM rather
than committing back to torque.
--log-level LOG_LEVEL Set the logging level.
--competition COMPETITION Competition to generate analysis support
- for, can alternatively be set in config.py.
+ for.
--proposals PROPOSALS Comma-separated list of proposal IDs to
generate analysis support for, can
alternatively be set in config.py.
@@ -52,8 +69,8 @@ Example:
--proposals 123,456,789
Notes:
- If `--dry-run` is set, LLM Analysis Support output will be pretty-printed to
- the terminal.
+ If `--log-level=DEBUG` is set, LLM Analysis Support output will be
+ pretty-printed to the terminal.
"""
try:
@@ -61,433 +78,195 @@ try:
except ImportError:
config = object()
-from dataclasses import asdict, dataclass, field
-import re
-import nltk
-import requests
-from requests.adapters import HTTPAdapter, Retry
-import textwrap
-import pprint
-import argparse
import logging
+import re
import subprocess
import sys
-import os
-from torqueclient import Torque
+import warnings
+from collections.abc import Mapping, MutableMapping
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, Literal, Self, Union
+import nltk # type: ignore
+import requests
+import typer
+import yaml
from jinja2 import Environment, FileSystemLoader
+from pydantic import (
+ BaseModel,
+ Field,
+ model_validator,
+)
+from requests.adapters import HTTPAdapter, Retry
+from rich import print as pprint
+from rich.console import Console
+from rich.logging import RichHandler
+from rich.markdown import Markdown
+from rich.progress import (
+ BarColumn,
+ Progress,
+ SpinnerColumn,
+ TaskProgressColumn,
+ TextColumn,
+ TimeElapsedColumn,
+)
+from rich.table import Table
+from torqueclient import Torque # type: ignore
-logging.basicConfig(level=logging.INFO, format="%(message)s")
+console = Console(width=80)
+
+logging.basicConfig(
+ level=logging.INFO,
+ format="%(message)s",
+ handlers=[
+ RichHandler(
+ rich_tracebacks=True, show_time=False, show_level=False, show_path=False
+ )
+ ],
+)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("mwclient").setLevel(logging.ERROR)
-nltk.download("punkt_tab", quiet=True)
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="mwclient.client")
-parser = argparse.ArgumentParser(
- prog="analysis_support.py",
- description="Generate LLM Analysis Support for proposals",
-)
+PROJECT_ROOT = Path(__file__).parent.parent.parent
-parser.add_argument(
- "-d",
- "--dry-run",
- action="store_true",
- help="Output the response from the LLM rather than committing back to Torque",
-)
-parser.add_argument(
- "-e",
- "--evaluate",
- action="store_true",
- help="Evaluate the similarity between the LLM and LFC analyses",
-)
-parser.add_argument(
- "-s",
- "--similarity",
- type=float,
- default=os.getenv(
- "SEMANTIC_SIMILARITY", getattr(config, "SEMANTIC_SIMILARITY", 0.77)
- ),
- help="Parts of the analysis are considered similar if the score is greater than this value",
-)
-parser.add_argument(
- "-l",
- "--log-level",
- help="Set the logging level",
- choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-)
-parser.add_argument(
- "-p",
- "--proposals",
- default=os.getenv("PROPOSALS", getattr(config, "PROPOSALS", None)),
- help=(
- "Comma separated list of proposal IDs to generate Analysis "
- "Support for, can alternatively be set in config.py or as an "
- "environment variable"
- ),
- type=lambda x: [int(i) for i in x.split(",")],
-)
-parser.add_argument(
- "-c",
- "--competition",
- default=os.getenv("COMPETITION", getattr(config, "COMPETITION", None)),
- help=(
- "Competition to generate Analysis Support for, can alternatively"
- "be set in config.py or as an environment variable"
- ),
- type=str,
-)
+class LogLevel(str, Enum):
+ DEBUG = "DEBUG"
+ INFO = "INFO"
+ WARNING = "WARNING"
+ ERROR = "ERROR"
+ CRITICAL = "CRITICAL"
-parser.add_argument(
- "--torque-user",
- default=os.getenv("TORQUE_USER", getattr(config, "TORQUE_USER", None)),
- help=(
- "The Torque user used to fetch and store proposal data, can"
- "alternatively be set in config.py or as an environment variable"
- ),
- type=str,
-)
-parser.add_argument(
- "--torque-password",
- default=os.getenv("TORQUE_PASSWORD", getattr(config, "TORQUE_PASSWORD", None)),
- help=(
- "The Torque API key used to fetch and store proposal data, can"
- "alternatively be set in config.py or as an environment variable"
- ),
- type=str,
-)
-parser.add_argument(
- "--torque-url",
- default=os.getenv("TORQUE_URL", getattr(config, "TORQUE_URL", None)),
- help=(
- "URL of the Torque instance from which to fetch and store proposal"
- "data, can alternatively be set in config.py or as an environment"
- "variable"
- ),
- type=str,
-)
-parser.add_argument(
- "--llm-endpoint",
- default=os.getenv("LLM_ENDPOINT", getattr(config, "LLM_ENDPOINT", None)),
- help=(
- "URL of the LLM API from which used to generate the Analysis Support,"
- "can alternatively be set in config.py or as an environment variable"
- ),
- type=str,
-)
-parser.add_argument(
- "--llm-api-key",
- default=os.getenv("LLM_API_KEY", getattr(config, "LLM_API_KEY", None)),
- help=(
- "Key for the LLM API from which used to generate the Analysis Support,"
- "can alternatively be set in config.py or as an environment variable"
- ),
- type=str,
-)
+class Config(BaseModel):
+ PROPOSALS: list[str] | None = None
+ COMPETITION: str | None = None
+ TORQUE_USER: str
+ TORQUE_PASSWORD: str
+ TORQUE_URL: str
+ LLM_ENDPOINT: str
+ LLM_API_KEY: str
+ SEARCH_ENDPOINT: str
+ SEARCH_API_KEY: str
+ SEMANTIC_SIMILARITY: float = 0.77
+ @classmethod
+ def validate_config(cls, config: object):
+ fields = cls.model_fields
+ config_values = {}
+ for field in fields:
+ config_values[field] = getattr(config, field, fields[field].default)
-parser.add_argument(
- "--search-endpoint",
- default=os.getenv("SEARCH_ENDPOINT", getattr(config, "SEARCH_ENDPOINT", None)),
- help=(
- "URL of the web search API used to look for reputational risks,"
- "can alternatively be set in config.py or as an environment variable"
- ),
- type=str,
-)
-parser.add_argument(
- "--search-api-key",
- default=os.getenv("SEARCH_API_KEY", getattr(config, "SEARCH_API_KEY", None)),
- help=(
- "Key for the web search API used to look for reputational risks,"
- "can alternatively be set in config.py or as an environment variable"
- ),
- type=str,
-)
+ return cls.model_validate(config_values, from_attributes=True)
-@dataclass
-class AnalysisPrompt:
- display: str
- prefix: str = ""
- postfix: str = "Create this summary using the following text:"
- template_blocks: list[str] = field(default_factory=list)
- sent: str = field(init=False)
- extrainstructions: str = ""
+class MarkdownRenderer:
+ template_path: Path
+ template_name: str
- def __post_init__(self):
- self.sent = (
- self.prefix + "\n\n" + self.display.strip() + "\n\n" + self.postfix
- ).strip()
+ @staticmethod
+ def commaize_number(number: int | float | str):
+ try:
+ return f"{int(float(number)):,}"
+ except (ValueError, TypeError):
+ return number
+ @staticmethod
+ def convert_md_to_mwiki(markdown: str):
+ if not markdown:
+ return ""
-@dataclass
-class AnalysesPrompts(AnalysisPrompt):
- prompts: dict[str, AnalysisPrompt] = field(default_factory=dict)
-
-
-sections = {
- "Project Overview": AnalysesPrompts(
- display="""
- Please summarize in 3 bullet points the key
- details of the proposal in the following areas: The Challenge, The
- Solution and The Plan.
- """,
- prompts={
- "The Challenge": AnalysisPrompt(
- display="""
- Please summarize in 2-3 sentences the challenge the proposal
- tackles. This should only include facts and no analysis.
- """,
- template_blocks=["the_problem"],
- ),
- "The Solution": AnalysisPrompt(
- display="""
- Please summarize in 2-3 sentences describing the solution and
- impact. This should include what, how, and with whom. This
- should only include facts and no analysis.
- """,
- template_blocks=["the_solution"],
- ),
- "The Plan": AnalysisPrompt(
- display="""
- Please summarize in 2-3 sentences how the award will be used
- and what will be achieved. This should only include facts from
- the application and no analysis.
-
- Include the following, if available a breakdown of the award
- e.g., How much is earmarked for staff, the solution, DEI,
- evaluation, etc. What are the targets, outcomes, and results?
- Expected direct and/or indirect reach of key stakeholder
- groups.
- """
- ),
- },
- ),
- "Strength of Approach": AnalysesPrompts(
- display="""
- Please summarize in 3 bullet points
- the key strengths of the proposal in the following areas:
- People, Solution and Process.
- """,
- prompts={
- "People": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences the strengths, assets, and
- experience the named leaders and any partners bring to this
- project. How has the organization
- involved stakeholders in solution development? How are
- diversity, equity, and inclusion embedded in the solution and
- lead organization? Please use evidence and name why this
- matters, is unique or uncommon.
- """,
- ),
- "Solution": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences the strengths of the
- proposed solution. Is the problem compelling? Is the market
- addressable? Does the solution make sense? What is the 3-5
- year impact goal? Is the goal specific and ambitious?
- What is the enduring impact, beyond 5 years? Please use
- evidence and name why this matters, is unique or uncommon.
- """,
- template_blocks=["the_solution", "panel_comments"],
- ),
- "Process": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences the strengths of the
- proposed process. Does the organization have a credible path
- to achieve its goals? What is the organization’s approach to
- scale? How does this solution solve a specific problem while
- changing systems? Does the budget support the
- implementation plan? How does the solution use data to measure,
- evaluate, learn, and adapt? Please use evidence
- and name why this matters, is unique or uncommon.
- """,
- template_blocks=["the_solution", "panel_comments"],
- ),
- },
- ),
- "Potential Challenges": AnalysesPrompts(
- display="""
- Please summarize in 3 bullet points the
- potential challenges of the proposal. Each bullet point should address
- a specific area that can include: Impact, Basis in Evidence,
- Feasibility, Durability and Justice.
- """,
- prompts={
- "Impact": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences any potential challenges with
- the impact of the proposed solution. Please use evidence and
- name why this is an important consideration.
- """,
- template_blocks=["projected_impact", "panel_comments"],
- ),
- "Basis in Evidence": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences any potential challenges with
- the evidence based nature of the proposed solution. Please use
- evidence and name why this is an important consideration.
- """,
- template_blocks=["the_solution", "panel_comments"],
- ),
- "Feasibility": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences any potential challenges with
- the feasibility of the proposed solution. Please use evidence
- and name why this is an important consideration.
- """,
- template_blocks=["the_solution", "panel_comments"],
- ),
- "Durability": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences any potential challenges with
- the durability of the proposed solution. Please use evidence
- and name why this is an important consideration.
- """,
- template_blocks=["the_solution", "panel_comments"],
- ),
- "Justice": AnalysisPrompt(
- display="""
- Please summarize in 2-4 sentences any potential challenges with
- the justice of the proposed solution. Please use evidence and
- name why this is an important consideration.
- """,
- template_blocks=["the_solution", "panel_comments"],
- ),
- },
- ),
- "Diversity, Equity, Inclusion, and Accessibility": AnalysesPrompts(
- display="""
- Please summarize in 3 bullet points
- the key details of the proposal as they relate to Diversity,
- Equity, Inclusion, and Accessibility in the following areas:
- Leadership, Community Engagement and Accessibility.
- """,
- prompts={
- "Leardership": AnalysisPrompt(
- display="""
- Please summarize in 1-3 sentences key details of the
- leadership of the proposed solution through the lens of
- diversity, equity, and inclusion. Please use
- facts and provide no analysis.
- """,
- ),
- "Community Engagement": AnalysisPrompt(
- display="""
- Please summarize in 1-3 sentences key details about the
- of the community engagement of the proposed solution through
- the lens diversity, equity, and inclusion. Please use facts
- and provide no analysis.
- """,
- ),
- "Accessibility": AnalysisPrompt(
- display="""
- If the proposal include plans or funds
- for accessibility, please summarize the proposed
- solution or accomodation, including the budget if there is
- one.
- """,
- ),
- },
- ),
- "Reputational Risks": AnalysesPrompts(
- display="""
- Please summarize in less than 3 bullet points
- any potential reputational risks contained in the provided search
- results. If no reputational risks are surfaced, please state that. Please don't
- suggest reputational risks if not directly stated.
- """,
- prompts={
- "Reputational Risks": AnalysisPrompt(
- display="""
- Please summarize in 3 bullet points with 2-4 sentences each any
- potential reputational risks associated surfaced in the provided search
- results. Please use evidence and name why this is an important consideration.
- If no reputational risks are surfaced, please state that. Please don't
- suggest reputational risks if not directly stated.
- """,
- template_blocks=["panel_comments"],
- )
- },
- ),
- "Other Considerations": AnalysesPrompts(
- display="""
- Please summarize the proposal in 4-6 bullet points
- highlighting review and Peer and Expert Review comments that pose
- questions that remain about the proposal and are important
- considerations.
- """,
- prompts={
- "Other Considerations": AnalysisPrompt(
- display="""
- Please summarize the proposal in 4-6 reflections highlighting
- review and Peer and Expert Review comments that pose questions
- that remain about the proposal and are important
- considerations.
- """,
- template_blocks=["panel_comments"],
+ try:
+ p = subprocess.run(
+ ["pandoc", "-t", "mediawiki"],
+ capture_output=True,
+ input=markdown,
+ text=True,
+ check=True,
)
- },
- ),
-}
+ return p.stdout
+ except subprocess.CalledProcessError as e:
+ logging.error(f"Error converting markdown to mwiki: {e.stderr}")
+ return ""
+ def __init__(self):
+ template_path = PROJECT_ROOT / self.template_path
+ self.env = Environment(
+ extensions=["jinja2.ext.do"],
+ loader=FileSystemLoader(template_path),
+ )
+ self.template = self.env.get_template(
+ self.template_name,
+ globals={
+ "convert_md_to_mwiki": self.convert_md_to_mwiki,
+ "commaize_number": self.commaize_number,
+ },
+ )
-@dataclass
-class AnalysisRequest:
- text: str
- considerations: str
+ def render_block(self, block: str, **kwargs: dict[str, Any]):
+ block_fn = self.template.blocks[block]
+ return "".join(block_fn(self.template.new_context(kwargs)))
+ def render(self, **kwargs: dict[str, Any]):
+ return self.template.render(kwargs)
-@dataclass
-class AnalysesRequest:
- list: list[AnalysisRequest]
- considerations: str
+class Proposal(Mapping[str, Any], MarkdownRenderer):
+ """
+ A wrapper around proposals that can be accessed like a dictionary.
+ Torque Client proposals are dict-ish, but with some exceptions. This class
+ allows us to access the proposal data in a more Pythonic way. It is read-only,
+ so we can't change the proposal data by accident. See `MutableProposal`
+ for a mutable version.
+ """
-@dataclass
-class AnalysisResponse:
- id: str
- value: str
+ template_path = PROJECT_ROOT / "roles" / "torque-llm-bridge" / "files"
+ template_name = "LLMProposal"
- def without_intro(self, sep=":"):
- """
- We want to remove the first line of the text if it's something like
- "Here is a summary of the proposal:", so we check if the first line
- ends with a colon and remove it if it does. Otherwise, we return the
- text as is.
- """
- text = self.value
+ _data: dict[str, Any]
- lines = text.strip().splitlines()
- if lines[0].endswith(sep) or not (
- lines[0].strip().startswith("•") or lines[0].strip().startswith("*")
- ):
- text = "\n".join(lines[1:]).strip()
+ def __init__(self, data: dict[str, Any]):
+ self._data = data
+ super().__init__()
- return AnalysisResponse(value=text, id=self.id)
+ def __getitem__(self, key: str):
+ return self._data[key]
+ def __iter__(self):
+ return iter(self._data.keys())
-@dataclass
-class LLMAnalysis(AnalysisResponse):
- prompt: AnalysesPrompts
- llmgenerated: str = "llm"
- generating: bool = False
+ def __len__(self):
+ return len(self._data.keys())
+ def render_markdown(self, blocks: list[str] | None = None):
+ if blocks and len(blocks) > 0:
+ return "\n\n".join(
+ [self.render_block(block, proposal=dict(self)) for block in blocks]
+ )
-@dataclass
-class SearchResult:
+ return self.render(proposal=dict(self))
+
+
+class MutableProposal(Proposal, MutableMapping[str, Any]):
+ def __setitem__(self, key: str, value: Any):
+ self._data[key] = value
+
+ def __delitem__(self, key: str) -> None:
+ raise NotImplementedError("Cannot delete items from a proposal")
+
+
+class SearchResult(BaseModel):
title: str
url: str
description: str
- extra_snippets: list[str] = field(default_factory=list)
+ extra_snippets: list[str] = Field(default_factory=list)
-@dataclass
-class SearchResults:
+class SearchResults(BaseModel):
results: list[SearchResult]
def __str__(self):
@@ -506,17 +285,106 @@ class SearchResults:
return "No results found."
-@dataclass
-class EmbeddingRequest:
+class AnalysisPrompt(BaseModel):
+ title: str
+ display: str | None = Field(default=None, alias="prompt")
+ prefix: str = (
+ "You are a helpful assistant who provides tips to staff analyzing "
+ "proposals to potentially fund. You have 20 years of experience. "
+ "You respond in a friendly, professional, and helpful manner, "
+ "with concrete details and specific numbers."
+ "You never make things up."
+ "You format your responses in Markdown but never use headers."
+ "You provide a 2-5 word bold title followd immediately by ':' in"
+ "each bullet point when using bullet points."
+ )
+ postfix: str = ""
+ template_blocks: list[str] = Field(default_factory=list, alias="blocks")
+ sent: str = ""
+ extrainstructions: str = ""
+
+ @model_validator(mode="after")
+ def set_sent(self) -> Self:
+ prefix = self.prefix.strip()
+ display = (self.display or "").strip()
+ postfix = self.postfix.strip()
+ self.sent = f"{prefix}\n\n{display}\n\n{postfix}".strip()
+ return self
+
+
+class AnalysesPrompts(AnalysisPrompt):
+ prompts: list[Union[AnalysisPrompt, "AnalysesPrompts"]]
+
+
+class AnalysisRequest(BaseModel):
+ text: str
+ considerations: str
+
+ @classmethod
+ def create(
+ cls,
+ proposal: Proposal,
+ prompt: AnalysisPrompt,
+ search_results: SearchResults | None = None,
+ ):
+ if not search_results:
+ text = "Summarize based on this text from the proposal:\n"
+ else:
+ text = "For context, here is some background about the project:\n"
+
+ text += proposal.render_markdown(prompt.template_blocks)
+
+ if search_results:
+ text += "\n\n"
+ text += str(search_results)
+
+ considerations = prompt.sent.format(**proposal)
+ return cls(text=clean_text(text), considerations=considerations)
+
+
+class AnalysesRequest(BaseModel):
+ list: list[AnalysisRequest]
+ considerations: str
+
+
+class AnalysisResponse(BaseModel):
+ id: str
+ value: str
+
+ def without_intro(self, sep: str = ":"):
+ """
+ We want to remove the first line of the text if it's something like
+ "Here is a summary of the proposal:", so we check if the first line
+ ends with a colon and remove it if it does.
+ """
+ text = self.value
+
+ lines = text.strip().splitlines()
+ if (
+ lines[0].strip().endswith(sep)
+ # or lines[0].strip().startswith("###")
+ # or lines[0].strip().startswith("**")
+ ):
+ text = "\n".join(lines[1:]).strip()
+
+ return AnalysisResponse(value=text, id=self.id)
+
+
+class LLMAnalysis(AnalysisResponse):
+ prompt: AnalysesPrompts | AnalysisPrompt
+ llmgenerated: str = "llm"
+ generating: bool = False
+
+
+class EmbeddingRequest(BaseModel):
input: str | list[str]
-@dataclass
-class EvaluationStats:
+class EvaluationStats(BaseModel):
total: int = 0
similar: int = 0
- def __add__(self, other):
+ def __add__(self, other: "EvaluationStats"):
return EvaluationStats(
total=self.total + other.total,
similar=self.similar + other.similar,
@@ -526,15 +394,12 @@ class EvaluationStats:
def percent(self):
return round((self.similar / self.total) * 100)
- def __str__(self):
- return f"{self.similar} / {self.total} = {self.percent}%"
-
class APIClient:
endpoint: str
api_key: str
- def __init__(self, endpoint, api_key):
+ def __init__(self, endpoint: str, api_key: str):
self.endpoint = endpoint
self.api_key = api_key
self.session = requests.Session()
@@ -549,7 +414,13 @@ class APIClient:
self.session.mount("http://", adapter)
self.session.mount("https://", adapter)
- def make_request(self, method, path, **kwargs):
+ def make_request(
+ self,
+ path: str,
+ *,
+ method: Literal["get", "post"] = "get",
+ **kwargs: dict[str, Any],
+ ):
http_method = getattr(self.session, method)
response = http_method(
f"{self.endpoint}{path}",
@@ -566,10 +437,16 @@ class Brave(APIClient):
API client for searching the web, which uses the Brave search engine.
"""
- def make_request(self, path, **kwargs):
+ def make_request(
+ self,
+ path: str,
+ *,
+ method: Literal["get", "post"] = "get",
+ **kwargs: dict[str, Any],
+ ):
return super().make_request(
- "get",
path,
+ method=method,
headers={
"accept": "application/json",
"X-Subscription-Token": self.api_key,
@@ -577,9 +454,7 @@ class Brave(APIClient):
**kwargs,
)
- def search(self, query):
- logging.info(f" Searching for {query}...")
-
+ def search(self, query: str):
response = self.make_request(
"web/search",
params={"q": query},
@@ -588,27 +463,24 @@ class Brave(APIClient):
if "web" not in response:
raise ValueError("No web pages in response")
- results = response["web"]["results"]
+ results = response["web"]
- return SearchResults(
- results=[
- SearchResult(
- title=result["title"],
- url=result["url"],
- description=result["description"],
- extra_snippets=result.get("extra_snippets", []),
- )
- for result in results
- ]
- )
+ return SearchResults.model_validate(results)
class LLM(APIClient):
- def make_request(self, path, data, **kwargs):
+ def make_request(
+ self,
+ path: str,
+ *,
+ data: BaseModel,
+ method: Literal["get", "post"] = "post",
+ **kwargs: dict[str, Any],
+ ):
return super().make_request(
- "post",
path,
- json=asdict(data),
+ method=method,
+ json=data.model_dump(),
headers={
"accept": "application/json",
"Authorization": "Bearer " + self.api_key,
@@ -617,17 +489,17 @@ class LLM(APIClient):
**kwargs,
)
- def get_analysis(self, text, prompt):
+ def get_analysis(self, text: str, prompt: str):
response = self.make_request(
"draft-analysis",
- AnalysisRequest(text=text, considerations=prompt),
+ data=AnalysisRequest(text=text, considerations=prompt),
)
return AnalysisResponse(value=response["draft-analysis"], id=response["id"])
- def get_analyses(self, list, prompt):
+ def get_analyses(self, list: list[AnalysisRequest], prompt: str):
response = self.make_request(
"draft-analyses",
- AnalysesRequest(list=list, considerations=prompt),
+ data=AnalysesRequest(list=list, considerations=prompt),
)
if "result" not in response:
@@ -636,118 +508,42 @@ class LLM(APIClient):
summary = response["result"]["summary"]
return AnalysisResponse(value=summary["content"], id=summary["id"])
- def get_embeddings(self, text):
+ def get_embeddings(self, text: str | list[str]):
response = self.make_request(
"embeddings",
- EmbeddingRequest(input=text),
+ data=EmbeddingRequest(input=text),
)
return [embedding["embedding"] for embedding in response["data"]]
-class MarkdownRenderer:
- template_path: str
- template_name: str
-
- @staticmethod
- def commaize_number(number):
- try:
- return "{:,}".format(int(float(number)))
- except (ValueError, TypeError):
- return number
-
- @staticmethod
- def convert_md_to_mwiki(markdown):
- if not markdown:
- return ""
-
- try:
- p = subprocess.run(
- ["pandoc", "-t", "mediawiki"],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- input=markdown,
- text=True,
- check=True,
- )
- return p.stdout
- except subprocess.CalledProcessError as e:
- logging.error(f"Error converting markdown to mwiki: {e.stderr}")
- return ""
-
- def __init__(self):
- dirname = os.path.dirname(__file__)
- template_path = os.path.join(dirname, self.template_path)
- self.env = Environment(
- extensions=["jinja2.ext.do"],
- loader=FileSystemLoader(template_path),
- )
- self.template = self.env.get_template(
- self.template_name,
- globals={
- "convert_md_to_mwiki": self.convert_md_to_mwiki,
- "commaize_number": self.commaize_number,
- },
- )
-
- def render_block(self, block, **kwargs):
- block = self.template.blocks[block]
- return "".join(block(self.template.new_context(kwargs)))
-
- def render(self, **kwargs):
- return self.template.render(kwargs)
-
-
-class LLMProposal(MarkdownRenderer):
- template_path = "../../roles/torque-llm-bridge/files/"
- template_name = "LLMProposal"
-
- def __init__(self, proposal):
- self.proposal = proposal
- super().__init__()
-
- def render_markdown(self, blocks=[]):
- variables = {
- "proposal": {key: self.proposal[key] for key in self.proposal.keys()}
- }
- if len(blocks) > 0:
- blocks = "\n\n".join(
- [self.render_block(block, **variables) for block in blocks]
- )
- return self.render(**variables)
-
-
-def wrap_text(text, width=60, indent=""):
- wrapper = textwrap.TextWrapper(
- width=width, initial_indent=indent, subsequent_indent=indent
+def spinner():
+ return Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ TimeElapsedColumn(),
+ BarColumn(),
+ TaskProgressColumn(),
+ transient=True,
)
- lines = text.splitlines()
- wrapped_lines = [wrapper.fill(line) for line in lines]
- return "\n".join(wrapped_lines)
-def clean_text(text):
+def clean_text(text: str):
text = re.sub(r"http[s]?://\S+", " ", text) # Remove URLs
- text = re.sub(r"\([0-9\.,]+\)", " ", text) # Remove numbers in parentheses
- text = re.sub(r"\[[0-9\.,]+\]", " ", text) # Remove numbers in brackets
+ text = re.sub(r"\([0-9\.,]*\)", " ", text) # Remove numbers in parentheses
+ text = re.sub(r"\[[0-9\.,]*\]", " ", text) # Remove numbers in brackets
text = text.replace("<strong>", " ")
text = text.replace("</strong>", " ")
text = text.replace("{", " ")
text = text.replace("}", " ")
- text = re.sub(r"\s+", " ", text).strip() # Remove extra whitespace
+ text = re.sub(r" +", " ", text).strip() # Remove extra spaces
return text
-def split_text(text):
- return nltk.sent_tokenize(text)
-
-
-def remove_backslashes(text):
- # Replace escaped characters
- text = re.sub(r"\\([,.\-()$:+?])", r"\1", text)
- return text
+def split_text(text: str):
+ return nltk.sent_tokenize(text) # type: ignore
-def dot_product(a, b):
+def dot_product(a: list[float], b: list[float]):
result = 0
for i in range(len(a)):
result += a[i] * b[i]
@@ -755,18 +551,32 @@ def dot_product(a, b):
return result
-def calculate_evaluation_stats(llm, llm_key, llm_value, proposal, **kwargs):
- if "LFC Analysis" not in proposal.keys():
+def parse_list(values: list[str] | None):
+ if not values:
+ return None
+ return [item for value in values for item in value.split(",")]
+
+
+def calculate_evaluation_stats(
+ llm: LLM,
+ llm_key: str,
+ llm_value: str,
+ proposal: Proposal,
+ *,
+ similarity: float = 0.77,
+):
+ if "LFC Analysis" not in proposal:
logging.error("LFC Analysis not found, not evaluating")
return EvaluationStats()
+ # Adjusts for mismatches between Analysis Support and LFC Analysis keys
key_mapping = {
"Project Overview": "Overview",
- "Diversity, Equity, Inclusion, and Accessibility": "Diversity, Equity and Inclusion",
+ "Diversity, Equity, Inclusion, and Accessibility": "Diversity, Equity and Inclusion", # noqa: E501
}
lfc_key = key_mapping.get(llm_key, llm_key)
- lfc_value = remove_backslashes(proposal["LFC Analysis"].get(lfc_key, ""))
+ lfc_value = proposal["LFC Analysis"].get(lfc_key, "")
llm_parts = split_text(llm_value)
lfc_parts = split_text(lfc_value)
@@ -779,19 +589,21 @@ def calculate_evaluation_stats(llm, llm_key, llm_value, proposal, **kwargs):
llm_embeddings = llm.get_embeddings(llm_parts)
lfc_embeddings = llm.get_embeddings(lfc_parts)
- for lfc_part, lfc_embedding in zip(lfc_parts, lfc_embeddings):
+ for lfc_part, lfc_embedding in zip(lfc_parts, lfc_embeddings, strict=False):
max_score, llm_part = max(
(
(
dot_product(llm_embedding, lfc_embedding),
llm_part,
)
- for llm_part, llm_embedding in zip(llm_parts, llm_embeddings)
+ for llm_part, llm_embedding in zip(
+ llm_parts, llm_embeddings, strict=False
+ )
),
key=lambda item: item[0],
)
- if max_score >= kwargs.get("similarity", 0.77):
+ if max_score >= similarity:
logging.debug(
f" * Similar sentence found ({max_score}):\n"
f" - LLM: {llm_part}\n"
@@ -805,113 +617,238 @@ def calculate_evaluation_stats(llm, llm_key, llm_value, proposal, **kwargs):
return stats
-def generate_analysis_support(llm, proposal, search_engine, **kwargs):
- llm_analysis = {}
- summary_stats = {}
+def generate_analysis_support(
+ llm: LLM,
+ prompts: list[AnalysesPrompts | AnalysisPrompt],
+ proposal: Proposal,
+ search_engine: Brave,
+ evaluate: bool = False,
+ similarity: float = 0.77,
+ log_level: LogLevel = LogLevel.INFO,
+ proposals_completed: int = 0,
+ total_proposals: int = 0,
+):
+ llm_analysis: dict[str, dict[str, Any]] = {}
+ summary_stats: dict[str, EvaluationStats] = {}
- for name, section in sections.items():
- logging.info(f" * {name}...")
+ total_prompts = len(prompts)
- prompts = []
+ for i, prompt in enumerate(prompts):
+ logging.info("")
+ logging.info(f"[bold]{prompt.title}[/bold]", extra={"markup": True})
+ logging.info("")
- for prompt in section.prompts.values():
- if name == "Reputational Risks":
- text = str(
- search_engine.search(f"{proposal['Organization Name']} controversy")
- )
- else:
- text = LLMProposal(proposal).render_markdown(prompt.template_blocks)
+ search_results = None
+ if prompt.title == "Reputational Risks":
+ search_results = search_engine.search(
+ f"{proposal['Organization Name']} controversy"
+ )
- prompts.append(
- AnalysisRequest(
- text=clean_text(text),
- considerations=prompt.sent,
- )
+ with spinner() as progress:
+ progress.add_task(
+ description=f"{
+ 'Waking up' if proposals_completed == 0 and i == 0 else 'Generating'
+ }[yellow]...[/yellow]",
+ completed=i + proposals_completed * total_prompts,
+ total=total_prompts * total_proposals,
)
- response = llm.get_analyses(prompts, section.sent)
+ if isinstance(prompt, AnalysesPrompts) and prompt.prompts:
+ requests: list[AnalysisRequest] = [
+ AnalysisRequest.create(proposal, subprompt, search_results)
+ for subprompt in prompt.prompts
+ ]
- value = response.without_intro().value.replace("• ", "* ")
+ formatted_prompt = AnalysisRequest.create(
+ proposal, prompt
+ ).considerations
- llm_analysis[name] = asdict(
- LLMAnalysis(
- id=response.id,
- value=value,
- prompt=section,
- )
- )
+ response = llm.get_analyses(requests, formatted_prompt)
- logging.info(f"{wrap_text(value, indent=' ')}")
+ else:
+ request = AnalysisRequest.create(proposal, prompt, search_results)
- if kwargs.get("evaluate"):
- summary_stats[name] = calculate_evaluation_stats(
- llm, name, value, proposal, **kwargs
+ response = llm.get_analysis(request.text, request.considerations)
+
+ value = f"{response.without_intro().value.replace('• ', '* ').strip()}\n"
+
+ llm_analysis[prompt.title] = LLMAnalysis(
+ id=response.id,
+ value=value,
+ prompt=prompt,
+ ).model_dump()
+
+ if log_level == LogLevel.INFO or log_level == LogLevel.DEBUG:
+ console.print(Markdown(value))
+
+ if evaluate:
+ summary_stats[prompt.title] = calculate_evaluation_stats(
+ llm, prompt.title, value, proposal, similarity=similarity
)
logging.debug("")
- logging.debug("*** Prompt and Analysis Support: ***")
+ logging.debug("[bold]Prompt and Analysis Support:[/bold]", extra={"markup": True})
logging.debug("")
- logging.debug(pprint.pformat(llm_analysis))
+ if log_level == LogLevel.DEBUG:
+ pprint(llm_analysis)
logging.debug("")
return llm_analysis, summary_stats
-def cli():
- args = parser.parse_args()
-
- if args.dry_run:
- logging.getLogger().setLevel(logging.INFO)
-
- if args.log_level:
- logging.getLogger().setLevel(getattr(logging, args.log_level))
+app = typer.Typer(
+ name="Analysis Support", help="Generates helpful tips for analysts with LLMs"
+)
- if not args.torque_user or not args.torque_password or not args.torque_url:
- parser.error("Torque credentials not set")
+valid_config = Config.validate_config(config)
- if not args.llm_endpoint or not args.llm_api_key:
- parser.error("LLM credentials not set")
- if not args.proposals or not args.competition:
- parser.error("Competition or proposals not set")
+@app.command()
+def main(
+ dry_run: Annotated[
+ bool,
+ typer.Option(
+ "--dry-run",
+ "-d",
+ help=(
+ "Output the response from the LLM rather than committing back to Torque"
+ ),
+ ),
+ ] = False,
+ evaluate: Annotated[
+ bool,
+ typer.Option(
+ "--evaluate",
+ "-e",
+ help="Evaluate the similarity between the LLM and LFC analyses",
+ ),
+ ] = False,
+ log_level: Annotated[
+ LogLevel,
+ typer.Option(
+ "--log-level",
+ "-l",
+ help="Set the logging level",
+ case_sensitive=False,
+ show_choices=True,
+ ),
+ ] = LogLevel.INFO,
+ proposals: Annotated[
+ list[str] | None,
+ typer.Option(
+ "--proposals",
+ "-p",
+ help=("List of proposal IDs to generate Analysis Support for"),
+ callback=parse_list,
+ envvar="PROPOSALS",
+ ),
+ ] = valid_config.PROPOSALS,
+ competition: Annotated[
+ str | None,
+ typer.Option(
+ "--competition",
+ "-c",
+ help="Competition to generate Analysis Support for",
+ envvar="COMPETITION",
+ ),
+ ] = valid_config.COMPETITION,
+):
+ logging.getLogger().setLevel(getattr(logging, log_level))
+
+ if not proposals or not competition:
+ logging.error("Competition or proposals not set")
+ exit(1)
torque = Torque(
- args.torque_url,
- args.torque_user,
- args.torque_password,
+ f"{valid_config.TORQUE_URL}{competition}",
+ valid_config.TORQUE_USER,
+ valid_config.TORQUE_PASSWORD,
)
- llm = LLM(endpoint=args.llm_endpoint, api_key=args.llm_api_key)
+ llm = LLM(endpoint=valid_config.LLM_ENDPOINT, api_key=valid_config.LLM_API_KEY)
+
+ brave = Brave(
+ endpoint=valid_config.SEARCH_ENDPOINT,
+ api_key=valid_config.SEARCH_API_KEY,
+ )
- brave = Brave(endpoint=args.search_endpoint, api_key=args.search_api_key)
+ with open(
+ PROJECT_ROOT
+ / "competitions"
+ / competition
+ / "ansible"
+ / "roles"
+ / competition
+ / "files"
+ / "LLM_Prompts.yml"
+ ) as file:
+ prompts = AnalysesPrompts.model_validate(yaml.safe_load(file)).prompts
+
+ if evaluate:
+ nltk.download("punkt_tab", quiet=True) # type: ignore
try:
- competition = torque.competitions[args.competition]
- except KeyError:
- print(f"Competition {args.competition} not found")
+ competition = torque.competitions[competition] # type: ignore
+ except TypeError:
+ logging.error(f"Competition {competition} not found")
sys.exit(1)
- logging.info("")
- logging.info("Generating Analysis Support for:")
- logging.info("")
+ logging.warning("")
+ logging.warning(
+ (
+ "Generating [bold]:sparkles:Analysis Support:sparkles:[/bold] for "
+ f"{len(proposals)} proposal{'s' if len(proposals) > 1 else ''}."
+ ),
+ extra={"markup": True},
+ )
+ logging.warning("")
+ logging.warning(
+ (
+ "[yellow]Note:[/yellow] May be slow to wake up on first response, "
+ "please be patient."
+ ),
+ extra={"markup": True},
+ )
+ logging.warning("")
- summary_stats = {}
+ summary_stats: dict[str, EvaluationStats] = {}
- for proposal_id in args.proposals:
- logging.info(f" * #{proposal_id}")
+ for i, id in enumerate(proposals):
+ logging.info("")
+ logging.info(
+ f"[bold]Proposal #{id}[/bold] ({i + 1}/{len(proposals)})",
+ extra={"markup": True},
+ )
+ logging.info("")
try:
- proposal = competition.proposals[proposal_id]
- except KeyError:
+ document = competition.proposals[id] # type: ignore
+ proposal = Proposal(document) if dry_run else MutableProposal(document) # type: ignore
+ except TypeError:
logging.error("Proposal not found")
continue
+ if "Organization Name" in proposal:
+ logging.info(
+ f"{proposal['Organization Name']}"
+ f"{': ' if 'Project Title' in proposal else ''}"
+ f"{proposal.get('Project Title', '')}"
+ )
+
try:
llm_analysis, proposal_stats = generate_analysis_support(
- llm, proposal, brave, **vars(args)
+ llm,
+ prompts,
+ proposal,
+ brave,
+ evaluate=evaluate,
+ similarity=valid_config.SEMANTIC_SIMILARITY,
+ log_level=log_level,
+ proposals_completed=i,
+ total_proposals=len(proposals),
)
except Exception as e:
- logging.error(f"Error generating analysis support: {e}")
+ logging.exception(e)
continue
for section, section_stats in proposal_stats.items():
@@ -920,20 +857,25 @@ def cli():
summary_stats[section] += section_stats
- if not args.dry_run:
+ if not dry_run and isinstance(proposal, MutableProposal):
# Setting this variable on a torqueclient proposal saves the data back
# out to the server
proposal["LLM LFC Analysis"] = llm_analysis
- if args.evaluate:
+ if evaluate:
logging.info("")
- logging.info("Evaluation Summary (similar / total):")
+ logging.info("Evaluation Summary (sentences):")
+ table = Table("", "Similar", "Total", "Percent")
for section, stats in summary_stats.items():
- logging.info(f" * {section}: {stats}")
+ table.add_row(
+ section, str(stats.total), str(stats.similar), f"{stats.percent}%"
+ )
+
+ console.print(table)
logging.info("")
if __name__ == "__main__":
- cli()
+ app()
diff --git a/scripts/llm/config.py.tmpl b/scripts/llm/config.py.tmpl
index 6b54631301b621ad4329c33234d54a6352eef758..54ede3e3087497e0ba299ae7163a2f611955465c 100644
--- a/scripts/llm/config.py.tmpl
+++ b/scripts/llm/config.py.tmpl
@@ -1,6 +1,6 @@
TORQUE_PASSWORD = "__PASSWORD__"
TORQUE_USER = "__USERNAME__"
-TORQUE_URL = "https://torque.leverforchange.org/GlobalView"
+TORQUE_URL = "https://torque.leverforchange.org/"
LLM_API_KEY = "__LLM_API_KEY__"
LLM_ENDPOINT = "__LLM_ENDPOINT__"
diff --git a/scripts/llm/test_analysis_support.py b/scripts/llm/test_analysis_support.py
index 04f5fab719fccfbf0149456e18d64c1654fe1994..d040b6353d53766eac07c33f824bdf770a1be7cf 100644
--- a/scripts/llm/test_analysis_support.py
+++ b/scripts/llm/test_analysis_support.py
@@ -1,16 +1,15 @@
-from dataclasses import asdict
import subprocess
from unittest import mock
+
from analysis_support import (
- LLMProposal,
- MarkdownRenderer,
+ LLM,
AnalysisRequest,
AnalysisResponse,
- LLM,
+ MarkdownRenderer,
+ Proposal,
clean_text,
)
-
# Test convert_md_to_mwiki #
@@ -68,16 +67,7 @@ def test_without_intro_with_colon():
def test_without_intro_without_colon():
assert (
AnalysisResponse(id="1", value="Intro.\n\nSome content").without_intro().value
- == "Some content"
- )
-
-
-def test_without_intro_not_bullets():
- assert (
- AnalysisResponse(id="1", value="• Not an intro.\n\n• Some content")
- .without_intro()
- .value
- == "• Not an intro.\n\n• Some content"
+ == "Intro.\n\nSome content"
)
@@ -85,7 +75,7 @@ def test_without_intro_not_bullets():
def test_render_proposal_markdown():
- output = LLMProposal(
+ output = Proposal(
{"Organization Name": "Name", "Project Title": "Title"}
).render_markdown()
assert output.startswith("# Name\n\n## Title\n")
@@ -117,7 +107,7 @@ def test_make_llm_request_success(mocked_requests):
llm = LLM(endpoint="http://example.com", api_key="key")
- response = llm.make_request(path, data, **params)
+ response = llm.make_request(path, data=data, **params)
assert response == {"result": "success"}
assert mocked_requests.called