Enhance Tableau insights with custom scrapers

bioscancast/datasets/sources.yaml

@@ -217,6 +217,11 @@ specific_pathogen_sources:
         url: "https://www.paho.org/en/topics/oropouche-virus-disease"
         geography: "Americas"
 
+      - id: "paho_oropouche_portal"
+        name: "ARBO Portal - Oropouche"
+        url: "https://www.paho.org/en/arbo-portal/arbo-portal-oropouche"
+        geography: "Americas"
+
   enteric:
     cholera:
       - id: "who_cholera"

bioscancast/main.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import argparse
+import json
 import logging
 import os
 import sys
@@ -192,6 +193,11 @@ def _parse_args(argv: list[str]) -> argparse.Namespace:
         action="store_true",
         help="Disable the search-stage cache.",
     )
+    parser.add_argument(
+        "--no-history-context",
+        action="store_true",
+        help="Disable prior-run forecast/insight context injection into forecasting.",
+    )
     parser.add_argument(
         "--max-input-tokens",
         type=int,
@@ -326,6 +332,175 @@ def _estimate_total_cost(per_model: dict[str, dict[str, int]]) -> tuple[float, l
     return total, warnings
 
 
+def _build_forecast_history_context(
+    out_root: Path,
+    question_id: str,
+    current_run_id: str,
+    *,
+    max_runs: int = 5,
+) -> str:
+    """Summarize prior forecast outputs for prompt context.
+
+    Pulls previous ``forecast.json`` + ``question.json`` artifacts for this
+    question and emits a compact text block the forecasting prompt can use as
+    historical context. Failures are best-effort and silently skipped.
+    """
+    qdir = out_root / question_id
+    if not qdir.exists() or not qdir.is_dir():
+        return ""
+
+    run_dirs = [p for p in qdir.iterdir() if p.is_dir() and p.name != current_run_id]
+    run_dirs.sort(key=lambda p: p.name, reverse=True)
+
+    lines: list[str] = []
+    for run_dir in run_dirs:
+        if len(lines) >= max_runs:
+            break
+        forecast_path = run_dir / "forecast.json"
+        question_path = run_dir / "question.json"
+        if not forecast_path.exists() or not question_path.exists():
+            continue
+
+        try:
+            forecast_payload = json.loads(forecast_path.read_text(encoding="utf-8"))
+            question_payload = json.loads(question_path.read_text(encoding="utf-8"))
+        except Exception:
+            continue
+
+        as_of = question_payload.get("as_of_date") or "(none)"
+        distributions = forecast_payload.get("distributions") or []
+        chosen = None
+        for d in distributions:
+            if d.get("forecast_source") == "bioscancast":
+                chosen = d
+                break
+        if chosen is None and distributions:
+            chosen = distributions[0]
+        if not chosen:
+            continue
+
+        probs = chosen.get("probabilities") or {}
+        if not probs:
+            continue
+        top_option = max(probs, key=probs.get)
+        top_prob = float(probs[top_option])
+        prob_parts = [f"{k}={float(v):.3f}" for k, v in probs.items()]
+
+        rationale = ""
+        samples = forecast_payload.get("samples") or []
+        for s in samples:
+            r = (s or {}).get("rationale")
+            if r:
+                rationale = str(r).strip()
+                break
+        if not rationale:
+            r = forecast_payload.get("baseline_rationale")
+            if r:
+                rationale = str(r).strip()
+
+        if len(rationale) > 240:
+            rationale = rationale[:237].rstrip() + "..."
+
+        line = (
+            f"- as_of={as_of} run={run_dir.name}: "
+            f"top={top_option} ({top_prob:.3f}); probs: "
+            f"{', '.join(prob_parts)}"
+        )
+        if rationale:
+            line += f"; rationale: {rationale}"
+        lines.append(line)
+
+    return "\n".join(lines)
+
+
+def _build_insight_history_context(
+    out_root: Path,
+    question_id: str,
+    current_run_id: str,
+    *,
+    max_runs: int = 3,
+    max_records_per_run: int = 3,
+) -> str:
+    """Summarize prior insight extracts for prompt context.
+
+    Each emitted line explicitly includes the prior scrape ``as_of`` date so
+    the forecasting model can distinguish historical snapshots from current
+    evidence.
+    """
+    qdir = out_root / question_id
+    if not qdir.exists() or not qdir.is_dir():
+        return ""
+
+    run_dirs = [p for p in qdir.iterdir() if p.is_dir() and p.name != current_run_id]
+    run_dirs.sort(key=lambda p: p.name, reverse=True)
+
+    lines: list[str] = []
+    runs_added = 0
+    for run_dir in run_dirs:
+        if runs_added >= max_runs:
+            break
+        insight_path = run_dir / "insight.json"
+        question_path = run_dir / "question.json"
+        if not insight_path.exists() or not question_path.exists():
+            continue
+
+        try:
+            insight_payload = json.loads(insight_path.read_text(encoding="utf-8"))
+            question_payload = json.loads(question_path.read_text(encoding="utf-8"))
+        except Exception:
+            continue
+
+        as_of = question_payload.get("as_of_date") or "(none)"
+        records = insight_payload.get("records") or []
+        if not isinstance(records, list) or not records:
+            continue
+
+        kept = 0
+        for rec in records:
+            if kept >= max_records_per_run:
+                break
+            if not isinstance(rec, dict):
+                continue
+
+            metric_name = rec.get("metric_name") or "(none)"
+            metric_value = rec.get("metric_value")
+            if metric_value is None:
+                metric_value_text = "n/a"
+            else:
+                metric_value_text = str(metric_value)
+            event_type = rec.get("event_type") or "other"
+            location = rec.get("location") or "(unknown)"
+            summary = (rec.get("summary") or "").strip()
+            if len(summary) > 140:
+                summary = summary[:137].rstrip() + "..."
+
+            quote = ""
+            sources = rec.get("sources") or []
+            if isinstance(sources, list) and sources:
+                first = sources[0]
+                if isinstance(first, dict):
+                    quote = (first.get("quote") or "").strip()
+            if len(quote) > 120:
+                quote = quote[:117].rstrip() + "..."
+
+            line = (
+                f"- past_scrape_as_of={as_of} run={run_dir.name}: "
+                f"event_type={event_type}, location={location}, "
+                f"metric={metric_name}, value={metric_value_text}"
+            )
+            if summary:
+                line += f"; summary: {summary}"
+            if quote:
+                line += f"; quote: {quote}"
+            lines.append(line)
+            kept += 1
+
+        if kept > 0:
+            runs_added += 1
+
+    return "\n".join(lines)
+
+
 # ----------------------------------------------------------------------------
 # Main pipeline
 # ----------------------------------------------------------------------------
@@ -402,6 +577,7 @@ def run_pipeline(args: argparse.Namespace) -> "ForecastResult | InsightRunResult
             "extraction": asdict(ExtractionConfig()),
             "insight": asdict(insight_config),
             "forecast": asdict(forecast_config),
+            "no_history_context": bool(args.no_history_context),
         },
         "forecast_options": options,
         "forecast_options_source": options_source,
@@ -513,12 +689,30 @@ def run_pipeline(args: argparse.Namespace) -> "ForecastResult | InsightRunResult
 
         if not args.no_forecast:
             with _stage_timer(manifest, "forecast"):
+                if args.no_history_context:
+                    historical_context = ""
+                    historical_insight_context = ""
+                else:
+                    historical_context = _build_forecast_history_context(
+                        out_root,
+                        question.id,
+                        run_id,
+                    )
+                    historical_insight_context = _build_insight_history_context(
+                        out_root,
+                        question.id,
+                        run_id,
+                    )
                 forecast_pipeline = ForecastingPipeline(
                     llm_client=shared_llm_raw,
                     config=forecast_config,
                 )
                 forecast_result = forecast_pipeline.run(
-                    question, insight_result.records, options,
+                    question,
+                    insight_result.records,
+                    options,
+                    historical_context=historical_context,
+                    historical_insight_context=historical_insight_context,
                 )
                 persistence.save_forecast(run_dir, forecast_result)
             stage_usage["forecast"] = (