Extraction: live PAHO Oropouche fetch + USDA date-parse fix

README.md

@@ -539,15 +539,15 @@ Additional packages:
 pip install openai tavily-python python-dotenv
 ```
 
-The USDA APHIS livestock scraper drives a headless browser to export the
-Tableau crosstab. After installing `requirements.txt`, download the browser
-once:
+The USDA APHIS livestock and PAHO Oropouche scrapers drive a headless browser
+to export their Tableau data. After installing `requirements.txt`, download the
+browser once:
 
 ```bash
 playwright install chromium
 ```
 
-If the browser is unavailable, that scraper degrades gracefully (it returns
+If the browser is unavailable, those scrapers degrade gracefully (they return
 no data and the generic fetcher is used instead) — the rest of the pipeline
 is unaffected.
 

bioscancast/datasets/sources.yaml

@@ -217,6 +217,11 @@ specific_pathogen_sources:
         url: "https://www.paho.org/en/topics/oropouche-virus-disease"
         geography: "Americas"
 
+      - id: "paho_oropouche_documents"
+        name: "PAHO Oropouche epidemiological updates (documents)"
+        url: "https://www.paho.org/en/documents/topics/oropouche-virus-disease"
+        geography: "Americas"
+
       - id: "paho_oropouche_portal"
         name: "ARBO Portal - Oropouche"
         url: "https://www.paho.org/en/arbo-portal/arbo-portal-oropouche"

bioscancast/stages/extraction/custom_scrapers/paho_oropouche_portal.py

@@ -1,53 +1,158 @@
-"""Custom scraper for PAHO ARBO Portal Oropouche weekly CSV.
-
-The PAHO Oropouche portal is dashboard-backed. This scraper pulls the downloadable
-underlying CSV and renders computed weekly/country analytics as compact HTML so
-standard extraction + insight stages can consume the information.
+"""Custom scraper for PAHO ARBO Portal Oropouche weekly data.
+
+The PAHO Oropouche dashboard is an authenticated Tableau Server view exposed
+publicly through a trusted-ticket wrapper. This scraper drives a headless
+browser to redeem a fresh ticket, reads the underlying "Full Data" table via the
+Tableau vizql export endpoint, and renders computed weekly/country analytics as
+compact HTML so the standard extraction + insight stages can consume it.
+
+Access notes: the public embed disables the download toolbar, and the underlying
+table is labelled "PAHO internal use only"; this scraper reconstructs the export
+endpoint the UI would otherwise expose. The view/table identifiers are stable
+across sessions -- only the vizql session token is ephemeral (minted per
+render). Any failure returns None so extraction degrades to the generic fetcher.
 """
 
 from __future__ import annotations
 
 import html
 import io
+import logging
 import re
+import time
 import unicodedata
 from datetime import datetime, timezone
 from typing import Callable, Optional
 
 import numpy as np
 import pandas as pd
-from curl_cffi import requests as curl_requests
 
 from bioscancast.stages.extraction.config import ExtractionConfig
 from bioscancast.stages.extraction.fetcher import FetchResult
 
-CSV_URL = (
-    "https://phip.paho.org/vizql/w/AME_OROV_Cases/v/Oropouche/vudcsv/"
-    "sessions/21F758F6EAB9431B932EB51F61EBDE01-1:0/views/"
-    "10956961424773455462_12644916002446576356"
-    "?underlying_table_id=FZ_AME_OROV_Cases_SE_12E30DDB8F874DC6AE5888461B08AB74"
-    "&underlying_table_caption=Full%20Data"
-)
+logger = logging.getLogger(__name__)
+
+# Public trusted-ticket wrapper: loading this mints a single-use Tableau ticket
+# and establishes an authenticated vizql session in the browser context.
+WRAPPER_URL = "https://ais.paho.org/ha_viz/Oropouche/AME_OROV_Viz.asp?env=pri"
+
+# Tableau vizql workbook/view and the stable identifiers for the Oropouche
+# "Full Data" underlying table. Only the session token (below) is per-render; if
+# PAHO republishes the workbook these ids may change and the fetch will 404 ->
+# None -> generic-fetch fallback.
+VIZQL_BASE = "https://phip.paho.org/vizql/w/AME_OROV_Cases/v/Oropouche"
+VIEW_ID = "10956961424773455462_12644916002446576356"
+UNDERLYING_TABLE_ID = "FZ_AME_OROV_Cases_SE_12E30DDB8F874DC6AE5888461B08AB74"
+
+_SESSION_RE = re.compile(r"/sessions/([0-9A-F]+-\d+:\d+)")
+
+# The trusted-ticket + vizql flow occasionally misses (a slow bootstrap or a
+# transient throttle); retry the whole browser flow a couple of times so those
+# self-heal without failing the extraction.
+_FETCH_ATTEMPTS = 2
+_RETRY_BACKOFF_SECONDS = 2.0
 
 CsvFetcher = Callable[[str, ExtractionConfig], Optional[str]]
 
 
-def _fetch_csv_text(url: str, cfg: ExtractionConfig) -> Optional[str]:
+def _fetch_full_data_csv(url: str, cfg: ExtractionConfig) -> Optional[str]:
+    """Return the Oropouche "Full Data" underlying CSV as text, or None.
+
+    Loads the trusted-ticket wrapper (``url``) to establish a vizql session,
+    captures the fresh session token from the viz bootstrap request, then GETs
+    the vudcsv underlying-data export for the stable view/table ids. Retries the
+    whole flow a couple of times on a transient miss. Never raises: any failure
+    (Playwright missing, navigation, non-200) returns None.
+    """
     try:
-        resp = curl_requests.get(
-            url,
-            timeout=max(cfg.fetch_timeout_seconds, 30.0),
-            impersonate=cfg.impersonate,
-            allow_redirects=True,
-        )
-    except Exception:
+        import playwright.sync_api  # noqa: F401
+    except Exception as exc:  # pragma: no cover - env without playwright
+        logger.info("PAHO Oropouche: Playwright unavailable: %s", exc)
         return None
 
-    if resp.status_code != 200:
-        return None
+    for attempt in range(1, _FETCH_ATTEMPTS + 1):
+        text = _fetch_full_data_csv_once(url, cfg)
+        if text:
+            return text
+        if attempt < _FETCH_ATTEMPTS:
+            logger.info(
+                "PAHO Oropouche: attempt %s returned nothing; retrying", attempt
+            )
+            time.sleep(_RETRY_BACKOFF_SECONDS)
+    return None
+
+
+def _fetch_full_data_csv_once(url: str, cfg: ExtractionConfig) -> Optional[str]:
+    """One browser attempt at the Full Data export (see _fetch_full_data_csv)."""
+    from playwright.sync_api import sync_playwright
 
-    text = (resp.text or "").strip()
-    return text or None
+    timeout_ms = int(max(cfg.fetch_timeout_seconds, 45.0) * 1000)
+
+    try:
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=True)
+            # phip.paho.org serves an incomplete TLS chain; it is PAHO's own
+            # host, reached only for this data export.
+            context = browser.new_context(ignore_https_errors=True)
+            page = context.new_page()
+
+            sessions: list[str] = []
+
+            def _on_request(req) -> None:
+                if "vizql" in req.url:
+                    match = _SESSION_RE.search(req.url)
+                    if match:
+                        sessions.append(match.group(1))
+
+            page.on("request", _on_request)
+
+            logger.info("PAHO Oropouche: loading trusted-ticket wrapper")
+            page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
+
+            waited = 0
+            while waited < timeout_ms and not sessions:
+                page.wait_for_timeout(500)
+                waited += 500
+            if not sessions:
+                logger.info("PAHO Oropouche: no vizql session captured")
+                context.close()
+                browser.close()
+                return None
+
+            session = sessions[0]
+            # Let the session finish bootstrapping before exporting.
+            page.wait_for_timeout(3000)
+
+            vudcsv = (
+                f"{VIZQL_BASE}/vudcsv/sessions/{session}/views/{VIEW_ID}"
+                f"?underlying_table_id={UNDERLYING_TABLE_ID}"
+                "&underlying_table_caption=Full%20Data"
+            )
+
+            text: Optional[str] = None
+            for _ in range(2):
+                resp = context.request.get(vudcsv, timeout=timeout_ms)
+                if resp.status == 200:
+                    body = resp.body()
+                    if body:
+                        text = body.decode("utf-8-sig", errors="replace").strip()
+                        if text:
+                            break
+                logger.info("PAHO Oropouche: vudcsv retry (status=%s)", resp.status)
+                page.wait_for_timeout(2000)
+
+            context.close()
+            browser.close()
+            if not text:
+                logger.info("PAHO Oropouche: no CSV returned")
+                return None
+            logger.info(
+                "PAHO Oropouche: fetched Full Data CSV (%s bytes)", len(text)
+            )
+            return text
+    except Exception as exc:  # pragma: no cover - network/browser failure
+        logger.info("PAHO Oropouche: fetch failed: %s", exc)
+        return None
 
 
 def _normalize_col(name: str) -> str:
@@ -208,7 +313,7 @@ def fetch(
     cfg = config or ExtractionConfig()
     fetched_at = datetime.now(timezone.utc)
 
-    text = (csv_fetcher or _fetch_csv_text)(CSV_URL, cfg)
+    text = (csv_fetcher or _fetch_full_data_csv)(WRAPPER_URL, cfg)
     if not text:
         return None
 
@@ -294,7 +399,7 @@ def fetch(
         "</head><body>"
         "<h1>PAHO Oropouche weekly analytics snapshot</h1>"
         f"<p>Source portal URL: {html.escape(url)}</p>"
-        f"<p>CSV source URL: {html.escape(CSV_URL)}</p>"
+        f"<p>Data source: {html.escape(WRAPPER_URL)}</p>"
         f"<p>Retrieved at: {fetched_at.isoformat()} | latest year_week: {html.escape(latest_week)}.</p>"
         "<h2>Weekly counts from Año + Semanas Epi</h2>"
         f"<p>Summary statistics (weekly summed Week Lab Confirmed): {_fmt_stats(weekly_series)}. "
@@ -308,8 +413,8 @@ def fetch(
     ).encode("utf-8")
 
     return FetchResult(
-        url=CSV_URL,
-        final_url=CSV_URL,
+        url=WRAPPER_URL,
+        final_url=WRAPPER_URL,
         status_code=200,
         content_type="text/html",
         content_bytes=rendered,

bioscancast/stages/extraction/custom_scrapers/usda_aphis_livestock.py

@@ -349,8 +349,11 @@ def fetch(
     if state_col != "State":
         df = df.rename(columns={state_col: "State"})
 
+    # The crosstab renders dates like "14-Jul-26" (%d-%b-%y). Opt into
+    # per-element parsing explicitly (format="mixed") — the same inference the
+    # bare call already did, but without the "Could not infer format" warning.
     df["Confirmed Diagnosis"] = pd.to_datetime(
-        df["Confirmed Diagnosis"], errors="coerce"
+        df["Confirmed Diagnosis"], format="mixed", errors="coerce"
     )
     df = df.dropna(subset=["Confirmed Diagnosis"]).copy()
     if df.empty:

bioscancast/tests/test_paho_oropouche_portal_scraper.py

@@ -71,7 +71,7 @@ def test_as_of_date_filters_future_weeks():
 def test_dispatcher_uses_source_id_custom_scraper(monkeypatch):
     from bioscancast.stages.extraction import fetcher as fetcher_mod
 
-    monkeypatch.setattr(paho_oropouche_portal, "_fetch_csv_text", lambda _url, _cfg: _CSV)
+    monkeypatch.setattr(paho_oropouche_portal, "_fetch_full_data_csv", lambda _url, _cfg: _CSV)
 
     result = fetcher_mod.fetch(
         "https://www.paho.org/en/arbo-portal/arbo-portal-oropouche",
@@ -93,7 +93,7 @@ def test_returns_none_on_missing_required_columns():
 
 
 def test_oropouche_scraper_output_reaches_insight_llm(monkeypatch):
-    monkeypatch.setattr(paho_oropouche_portal, "_fetch_csv_text", lambda _url, _cfg: _CSV)
+    monkeypatch.setattr(paho_oropouche_portal, "_fetch_full_data_csv", lambda _url, _cfg: _CSV)
 
     fdoc = FilteredDocument(
         result_id="r-orov-1",
@@ -154,3 +154,27 @@ def test_oropouche_scraper_output_reaches_insight_llm(monkeypatch):
 
     assert insight.documents_processed == 1
     assert llm.call_count == 1
+
+
+# The live "Full Data" vudcsv export carries extra columns (SE, Iso3166,
+# Actualizado, Dominio, ...) and a different column order than the minimal
+# fixture above; the parser must resolve the four it needs and ignore the rest.
+_FULL_DATA_CSV = (
+    "Week Lab Confirmed,Semanas Epi,Pais,Año,SE,Iso3166,Actualizado,Dominio,Fuente\n"
+    "1,2,Brasil,2026,15,76,2026-04-28,PAHO internal use only,Boletim\n"
+    "3,5,Brasil,2026,15,76,2026-04-28,PAHO internal use only,Boletim\n"
+    "2,4,Panamá,2026,15,591,2026-04-28,PAHO internal use only,MINSA\n"
+)
+
+
+def test_parses_full_data_column_superset():
+    result = paho_oropouche_portal.fetch(
+        "https://www.paho.org/en/arbo-portal/arbo-portal-oropouche",
+        csv_fetcher=_fetcher(_FULL_DATA_CSV),
+    )
+    body = _html(result)
+    assert "Weekly counts from Año + Semanas Epi" in body
+    assert "Per-country summary" in body
+    # Both reporting countries surface in the per-country grouping.
+    assert "Brasil" in body
+    assert "Panamá" in body