diff --git a/.agents/metrics.jsonl b/.agents/metrics.jsonl index 4b0cd7ab..21e085c2 100644 --- a/.agents/metrics.jsonl +++ b/.agents/metrics.jsonl @@ -5,3 +5,9 @@ {"timestamp": "2026-08-10T15:12:36Z", "agent": "main", "task": "close out cache pre-warming (Plan 11): prewarm tests, [routing.prewarm] config + CONFIG.md docs, prewarm loop fix", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "DI core prewarm_domains + 6 offline tests; fixed join loop that always waited full 60s OVERALL_TIMEOUT; config.toml + agents-docs/CONFIG.md; full suite 138 green, clippy/fmt clean."} {"timestamp": "2026-08-11T07:57:58Z", "agent": "main", "task": "review, fix, and merge cache pre-warming PR #570", "skill_used": "do-github-pr-sentinel", "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "Self-review roasted 8 issues; fixed 1-6 + moved config keys out of [routing] (log_level now honored); metrics kept out of product PR; CI green; squash-merged 62ea294."} {"timestamp": "2026-08-11T08:25:03Z", "agent": "main", "task": "drop unnecessary Sync bound on prewarm_domains (#573)", "skill_used": "do-github-pr-sentinel", "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "Relaxed F bound to Fn+Clone+Send+'static; Sync never exercised (closure cloned per task). cargo test 140 pass, clippy clean; PR #573 squash-merged 72bf164."} +{"timestamp": "2026-08-11T00:00:00Z", "agent": "main", "task": "Implement 7 approved codebase improvements: web XSS href scheme validation, thread user maxChars into providers, visual_resolver urlopen timeouts, wire config.log_level into init_logging, delete inert max_links field, warn on malformed config.toml, refresh plans/README stale statuses", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "All 7 landed; quality_gate.sh, web lint+typecheck, cargo fmt+clippy+test, ruff/black/mypy, web vitest targeted all green"} +{"timestamp": "2026-08-11T01:00:00Z", "agent": "main", "task": "Tier1 correctness: refactor guarded Option unwraps in resolver hot loops, non-panicking shared reqwest client, log Mistral agent cleanup failures, delete dead resolve_with_order + ConfigError::InvalidConfig; Tier2 tests: 7 metrics.rs unit tests, implement broken check_python_cli; config subcommand now prints routing/cache.ttl/rate-limits; refresh plans/03+11 stale statuses", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "quality_gate.sh green; cargo 98 unit + all integration tests pass; clippy/fmt clean; validate_docs 0/0; ruff/black clean"} +{"timestamp": "2026-08-11T02:00:00Z", "agent": "main", "task": "Open items: delete dead provider_decorator.py; wire UI profile into /api/resolve (custom->balanced guard); file-backed RoutingMemory persistence (AUDIT #25) via DO_WDR_ROUTING_MEMORY_PATH + cli default path; 7 persistence tests; parity assessed (exa_mcp_mistral/hedge/deep-research scoped as feature work)", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "quality_gate.sh green; web 176 tests; routing 115 tests incl 7 new persistence; mypy/ruff clean; provider_decorator deleted (0 refs repo-wide)"} +{"timestamp": "2026-08-11T03:00:00Z", "agent": "main", "task": "Next batch: assessed stealth.py (retained - designed FetchTier.STEALTH escalation slot, tested, unreachable); implemented fix_duplicate_links fixer (whole-line dedupe); removed fake fix_python_cli/fix_repo_trees stubs + dispatch; removed dead runSequential/runParallel/RunResult, orphan ToggleChip, legacy loadUiState/saveUiState aliases", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "quality_gate.sh green; web 176 tests; python 872 passed; ruff/mypy clean; validate_docs 0/0; fix_duplicate_links proven via synthetic test"} +{"timestamp": "2026-08-11T04:00:00Z", "agent": "main", "task": "Final cleanup sweep: deleted production-dead web/lib/providers.ts + its test (app uses constants.ts/routing.ts); removed dead get_state() + ResolverState.semantic_cache from scripts/state.py; removed vestigial NEXT_PUBLIC_RESOLVER_URL from .env.example/DEPLOYMENT/help page (unread by code); flagged sync_skill.py (orphan+stale) and models.py methods (tested API) for user decision", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "quality_gate.sh green; web 172 tests (4 removed with dead module); python 872 passed; ruff/mypy 50 files clean; imports verified"} +{"timestamp": "2026-08-11T05:00:00Z", "agent": "main", "task": "Resolved flagged items: FIXED (not deleted) sync_skill.py - made utils-package-aware (syncs scripts/utils/*.py + copies real __init__.py, deletes obsolete flat utils.py); refreshed stale skill copy (18 files + 9 new utils/*), fixed 3 stale skill-family test refs (constants import, cache-hit stub, ftp SSRF assertion); corrected DEPLOYMENT.md I introduced NEXT_PUBLIC_APP_URL to real env vars (RECORDS_*, WEB_RESOLVER_MAX_CHARS, keys), removed vestigial NEXT_PUBLIC_* from .env.example", "skill_used": null, "status": "completed", "tokens_used": 0, "duration_seconds": 0, "notes": "quality_gate green; main 872 + skill 91 + web 172 tests pass; black/rff clean; sync_skill --dry-run 0 pending"} diff --git a/.agents/skills/do-web-doc-resolver/__init__.py b/.agents/skills/do-web-doc-resolver/__init__.py index fcfe2b9b..38ea8c4c 100644 --- a/.agents/skills/do-web-doc-resolver/__init__.py +++ b/.agents/skills/do-web-doc-resolver/__init__.py @@ -1,6 +1,6 @@ """do-web-doc-resolver: Resolve URLs and queries into LLM-ready markdown.""" -from scripts.resolve import resolve, resolve_url, resolve_query, main +from scripts.resolve import resolve, resolve_url, resolve_query __version__ = "0.1.0" -__all__ = ["resolve", "resolve_url", "resolve_query", "main"] +__all__ = ["resolve", "resolve_url", "resolve_query"] diff --git a/.agents/skills/do-web-doc-resolver/scripts/cache_negative.py b/.agents/skills/do-web-doc-resolver/scripts/cache_negative.py index f148f8f0..b46cb84e 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/cache_negative.py +++ b/.agents/skills/do-web-doc-resolver/scripts/cache_negative.py @@ -2,8 +2,11 @@ Negative caching logic for the Web Doc Resolver. """ +import logging from datetime import datetime, timedelta, timezone +logger = logging.getLogger(__name__) + def should_skip_from_negative_cache(cache, key: str, provider: str) -> bool: if cache is None: @@ -24,6 +27,7 @@ def should_skip_from_negative_cache(cache, key: str, provider: str) -> bool: dt = dt.replace(tzinfo=timezone.utc) return dt > datetime.now(timezone.utc) except Exception: + logger.debug("Failed to parse negative cache expiry: %s", expires_at, exc_info=True) return False @@ -49,3 +53,19 @@ def write_negative_cache( "metadata": metadata, } cache.set(f"neg:{provider}:{key}", entry, expire=ttl_seconds) + + +def should_skip_from_bot_challenge_cache( + provider: str, + url: str, + bot_challenge_cache: dict[str, set[str]], +) -> bool: + """Skip plain-fetch providers for URLs known to serve bot challenges.""" + from urllib.parse import urlparse + + try: + domain = urlparse(url).netloc + return provider in ("direct_fetch",) and domain in bot_challenge_cache.get(provider, set()) + except Exception as e: + logger.debug("Bot challenge cache lookup failed for %s: %s", url, e) + return False diff --git a/.agents/skills/do-web-doc-resolver/scripts/constants.py b/.agents/skills/do-web-doc-resolver/scripts/constants.py index 7294591f..249ed665 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/constants.py +++ b/.agents/skills/do-web-doc-resolver/scripts/constants.py @@ -5,8 +5,13 @@ import os import typing +from scripts.models import FetchTier + logger = logging.getLogger(__name__) +if typing.TYPE_CHECKING: + pass + def _load_config() -> dict[str, typing.Any]: config_path = os.getenv("DO_WDR_CONFIG") or "config.toml" @@ -110,3 +115,16 @@ def _env( BLOCKED_SCHEMES: set[str] = {"file", "javascript", "data", "vbscript"} DNS_CACHE_TTL: int = 60 + +CLEAN_CONTENT: bool = os.environ.get("WDR_CLEAN_CONTENT", "1") != "0" + +PROVIDER_TIERS: dict[str, FetchTier] = { + "llms_txt": FetchTier.FREE_STATIC, + "direct_fetch": FetchTier.FREE_DIRECT, + "duckduckgo": FetchTier.FREE_SEARCH, + "jina": FetchTier.PAID_LITE, + "firecrawl": FetchTier.PAID_LITE, + "visual_clip": FetchTier.PAID_LITE, + "stealth": FetchTier.STEALTH, + "mistral_browser": FetchTier.PAID_BROWSER, +} diff --git a/.agents/skills/do-web-doc-resolver/scripts/models.py b/.agents/skills/do-web-doc-resolver/scripts/models.py index 3a009fbb..7f9805eb 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/models.py +++ b/.agents/skills/do-web-doc-resolver/scripts/models.py @@ -20,6 +20,7 @@ class ErrorType(Enum): INVALID_RESPONSE = "invalid_response" SSRF_BLOCKED = "ssrf_blocked" CONTENT_TOO_LARGE = "content_too_large" + BOT_CHALLENGE = "bot_challenge" UNKNOWN = "unknown" @@ -50,6 +51,18 @@ def max_hops(self) -> int: return 4 +class FetchTier(int, Enum): + """Escalation cost tier for fetch providers. + Lower = cheaper, always tried first.""" + + FREE_STATIC = 0 # llms_txt: static text file, zero cost + FREE_DIRECT = 1 # direct_fetch: plain httpx, zero cost + FREE_SEARCH = 2 # duckduckgo: free web search + PAID_LITE = 3 # jina, firecrawl: paid but cheap per-call + STEALTH = 4 # anti-bot bypass tier + PAID_BROWSER = 5 # mistral_browser: paid + JS execution + + class ProviderType(Enum): """Available providers for resolution.""" @@ -71,6 +84,7 @@ class ProviderType(Enum): # New providers DOCLING = "docling" OCR = "ocr" + VISUAL_CLIP = "visual_clip" def is_paid(self) -> bool: return self in ( @@ -80,6 +94,7 @@ def is_paid(self) -> bool: ProviderType.FIRECRAWL, ProviderType.MISTRAL_WEBSEARCH, ProviderType.MISTRAL_BROWSER, + ProviderType.VISUAL_CLIP, ) def is_fast(self) -> bool: @@ -179,3 +194,15 @@ class ReadonlyResolverProtocol(Protocol): """ def __call__(self) -> ResolvedResult | str | None: ... + + +__all__ = [ + "ErrorType", + "Profile", + "ProviderType", + "ValidationResult", + "ProviderMetric", + "ResolveMetrics", + "ResolvedResult", + "ReadonlyResolverProtocol", +] diff --git a/.agents/skills/do-web-doc-resolver/scripts/providers_impl.py b/.agents/skills/do-web-doc-resolver/scripts/providers_impl.py index c0457b5a..e64537d1 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/providers_impl.py +++ b/.agents/skills/do-web-doc-resolver/scripts/providers_impl.py @@ -1,518 +1,53 @@ """ Individual provider implementations for the Web Doc Resolver. -""" - -import json -import logging -import os -import subprocess -import threading -import time -import requests +This module re-exports all provider functions from the providers package for backward compatibility. +""" -from scripts.constants import ( - DDG_RESULTS, - DEFAULT_TIMEOUT, - EXA_RESULTS, - MAX_CHARS, - MIN_CHARS, - TAVILY_RESULTS, +from scripts.providers import ( + _clear_rate_limits, + _is_rate_limited, + _rate_limits, + _set_rate_limit, + is_rate_limited, + resolve_with_docling, + resolve_with_duckduckgo, + resolve_with_exa, + resolve_with_exa_mcp, + resolve_with_firecrawl, + resolve_with_jina, + resolve_with_mistral_browser, + resolve_with_mistral_websearch, + resolve_with_ocr, + resolve_with_serper, + resolve_with_stealth, + resolve_with_tavily, + resolve_with_visual_clip, + resolve_with_visual_clip_async, + set_rate_limit, ) -from scripts.models import ResolvedResult -from scripts.utils import ( - _get_from_cache, - _save_to_cache, - get_session, - is_safe_url, -) - -logger = logging.getLogger(__name__) - -_rate_limits: dict[str, float] = {} -_rate_limits_lock = threading.Lock() - - -def _is_rate_limited(provider: str) -> bool: - with _rate_limits_lock: - if provider in _rate_limits: - if time.time() < _rate_limits[provider]: - return True - del _rate_limits[provider] - return False - - -def _set_rate_limit(provider: str, cooldown: int = 60): - with _rate_limits_lock: - _rate_limits[provider] = time.time() + cooldown - - -def _clear_rate_limits() -> None: - with _rate_limits_lock: - _rate_limits.clear() - - -# Exported names for both internal use and tests -is_rate_limited = _is_rate_limited -set_rate_limit = _set_rate_limit - - -def resolve_with_jina(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - if not is_safe_url(url): - logger.warning("SSRF blocked: %s", url) - return None - cached = _get_from_cache(url, "jina") - if cached: - return ResolvedResult(**cached) - if _is_rate_limited("jina"): - return None - try: - session = get_session() - response = session.get( - f"https://r.jina.ai/{url}", - timeout=DEFAULT_TIMEOUT, - headers={"Accept": "text/markdown"}, - ) - if response.status_code == 429: - logger.warning("Jina rate limited — setting cooldown") - _set_rate_limit("jina") - return None - if response.status_code == 401 or response.status_code == 403: - logger.warning("Jina auth error: HTTP %s for %s", response.status_code, url) - return None - if response.status_code != 200: - logger.warning("Jina HTTP %s for %s", response.status_code, url) - return None - content = response.text.strip() - if len(content) < MIN_CHARS: - logger.warning( - "Jina returned insufficient content (%s chars) for %s", len(content), url - ) - return None - result = ResolvedResult(source="jina", content=content[:max_chars], url=url) - _save_to_cache(url, "jina", result.to_dict()) - return result - except requests.RequestException as e: - logger.warning("Jina resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_exa_mcp(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - cached = _get_from_cache(query, "exa_mcp") - if cached: - return ResolvedResult(**cached) - if _is_rate_limited("exa_mcp"): - return None - try: - mcp_request = { - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "web_search_exa", "arguments": {"query": query, "numResults": 8}}, - } - session = get_session() - response = session.post( - "https://mcp.exa.ai/mcp", - json=mcp_request, - headers={"Accept": "application/json, text/event-stream"}, - timeout=25, - ) - if response.status_code != 200: - logger.warning("Exa MCP HTTP %s for query: %s", response.status_code, query) - return None - for line in response.text.split("\n"): - if line.startswith("data: "): - data = json.loads(line[6:]) - if data.get("result") and data["result"].get("content"): - content = data["result"]["content"][0].get("text", "") - if not content: - logger.warning("Exa MCP returned empty content for query: %s", query) - return None - result = ResolvedResult( - source="exa_mcp", content=content[:max_chars], query=query - ) - _save_to_cache(query, "exa_mcp", result.to_dict()) - return result - logger.warning("Exa MCP returned no usable content for query: %s", query) - except json.JSONDecodeError as e: - logger.warning("Exa MCP JSON parse failed: %s", e) - except requests.RequestException as e: - logger.warning("Exa MCP resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_exa(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - cached = _get_from_cache(query, "exa") - if cached: - return ResolvedResult(**cached) - api_key = os.getenv("EXA_API_KEY") - if not api_key: - logger.debug("Exa skipped: no API key") - return None - if _is_rate_limited("exa"): - logger.debug("Exa skipped: rate limited") - return None - try: - from exa_py import Exa - - client = Exa(api_key) - res = client.search_and_contents( - query, use_autoprompt=True, highlights=True, num_results=EXA_RESULTS - ) - if not res or not res.results: - logger.warning("Exa returned no results for query: %s", query) - return None - content = "\n\n---\n\n".join( - [ - r.highlight or r.text - for r in res.results - if hasattr(r, "highlight") and r.highlight or hasattr(r, "text") and r.text - ] - ) - if not content: - logger.warning("Exa returned empty content for query: %s", query) - return None - result = ResolvedResult(source="exa", content=content[:max_chars], query=query) - _save_to_cache(query, "exa", result.to_dict()) - return result - except Exception as e: - status = getattr(e, "status_code", None) - if status == 401: - logger.warning("Exa failed: 401 Unauthorized — API key may be invalid or expired") - elif status == 429: - logger.warning("Exa failed: 429 Rate limited — setting cooldown") - _set_rate_limit("exa") - elif status == 403: - logger.warning("Exa failed: 403 Forbidden — %s", e) - else: - logger.warning("Exa resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_tavily(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - cached = _get_from_cache(query, "tavily") - if cached: - return ResolvedResult(**cached) - api_key = os.getenv("TAVILY_API_KEY") - if not api_key: - logger.debug("Tavily skipped: no API key") - return None - if _is_rate_limited("tavily"): - logger.debug("Tavily skipped: rate limited") - return None - try: - from tavily import TavilyClient - - client = TavilyClient(api_key=api_key) - res = client.search(query, max_results=TAVILY_RESULTS) - if not res or not res.get("results"): - logger.warning("Tavily returned no results for query: %s", query) - return None - content = "\n\n---\n\n".join([f"## {r['title']}\n\n{r['content']}" for r in res["results"]]) - result = ResolvedResult(source="tavily", content=content[:max_chars], query=query) - _save_to_cache(query, "tavily", result.to_dict()) - return result - except Exception as e: - status = getattr(e, "status_code", None) - if status == 401: - logger.warning("Tavily failed: 401 Unauthorized — API key may be invalid or expired") - elif status == 429: - logger.warning("Tavily failed: 429 Rate limited — setting cooldown") - _set_rate_limit("tavily") - elif status == 403: - logger.warning("Tavily failed: 403 Forbidden — %s", e) - else: - logger.warning("Tavily resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_serper(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - """Search via Serper (Google Search API). Free tier: 2500 credits.""" - cached = _get_from_cache(query, "serper") - if cached: - return ResolvedResult(**cached) - api_key = os.getenv("SERPER_API_KEY") - if not api_key: - logger.debug("Serper skipped: no API key") - return None - if _is_rate_limited("serper"): - logger.debug("Serper skipped: rate limited") - return None - try: - session = get_session() - response = session.post( - "https://google.serper.dev/search", - headers={ - "X-API-KEY": api_key, - "Content-Type": "application/json", - }, - json={"q": query, "num": 5}, - timeout=DEFAULT_TIMEOUT, - ) - if response.status_code == 429: - logger.warning("Serper rate limited — setting 1hr cooldown") - _set_rate_limit("serper", 3600) - return None - if response.status_code == 401 or response.status_code == 403: - logger.warning( - "Serper auth error: HTTP %s — API key may be invalid", response.status_code - ) - return None - if response.status_code != 200: - logger.warning("Serper HTTP %s for query: %s", response.status_code, query) - return None - data = response.json() - organic = data.get("organic", []) - if not organic: - logger.warning("Serper returned no organic results for query: %s", query) - return None - parts = [] - for r in organic: - title = r.get("title", "") - link = r.get("link", "") - snippet = r.get("snippet", "") - if title and snippet: - parts.append(f"## {title}\n\n{snippet}\n\n[{link}]({link})") - if not parts: - logger.warning("Serper returned no usable snippets for query: %s", query) - return None - content = "\n\n---\n\n".join(parts) - result = ResolvedResult(source="serper", content=content[:max_chars], query=query) - _save_to_cache(query, "serper", result.to_dict()) - return result - except requests.RequestException as e: - logger.warning("Serper resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_duckduckgo(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - cached = _get_from_cache(query, "duckduckgo") - if cached: - return ResolvedResult(**cached) - if _is_rate_limited("duckduckgo"): - logger.debug("DuckDuckGo skipped: rate limited") - return None - try: - from ddgs import DDGS - - with DDGS() as ddgs: - results = list(ddgs.text(query, max_results=DDG_RESULTS)) - if not results: - logger.warning("DuckDuckGo returned no results for query: %s", query) - return None - content = "\n\n---\n\n".join( - [f"## {r.get('title', '')}\n\n{r.get('body', '')}" for r in results] - ) - result = ResolvedResult(source="duckduckgo", content=content[:max_chars], query=query) - _save_to_cache(query, "duckduckgo", result.to_dict()) - return result - except Exception as e: - logger.warning("DuckDuckGo resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_firecrawl(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - if not is_safe_url(url): - logger.warning("SSRF blocked: %s", url) - return None - cached = _get_from_cache(url, "firecrawl") - if cached: - return ResolvedResult(**cached) - api_key = os.getenv("FIRECRAWL_API_KEY") - if not api_key: - logger.debug("Firecrawl skipped: no API key") - return None - if _is_rate_limited("firecrawl"): - logger.debug("Firecrawl skipped: rate limited") - return None - try: - from firecrawl import Firecrawl - - app = Firecrawl(api_key=api_key) - res = app.scrape(url, formats=["markdown"]) - if not res or not hasattr(res, "markdown"): - logger.warning("Firecrawl returned no markdown for URL: %s", url) - return None - markdown = res.markdown - if not markdown: - logger.warning("Firecrawl returned empty markdown for URL: %s", url) - return None - result = ResolvedResult(source="firecrawl", content=markdown[:max_chars], url=url) - _save_to_cache(url, "firecrawl", result.to_dict()) - return result - except Exception as e: - status = getattr(e, "status_code", None) - if status == 401: - logger.warning("Firecrawl failed: 401 Unauthorized — API key may be invalid or expired") - elif status == 429: - logger.warning("Firecrawl failed: 429 Rate limited — setting cooldown") - _set_rate_limit("firecrawl") - elif status == 403: - logger.warning("Firecrawl failed: 403 Forbidden — %s", e) - else: - logger.warning("Firecrawl resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_mistral_browser(url: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - if not is_safe_url(url): - logger.warning("SSRF blocked: %s", url) - return None - cached = _get_from_cache(url, "mistral_browser") - if cached: - return ResolvedResult(**cached) - api_key = os.getenv("MISTRAL_API_KEY") - if not api_key: - logger.debug("Mistral browser skipped: no API key") - return None - if _is_rate_limited("mistral"): - logger.debug("Mistral browser skipped: rate limited") - return None - try: - from mistralai.client import Mistral - - client = Mistral(api_key=api_key) - - # Create an agent with web_search tool - agent = client.beta.agents.create( - model="mistral-small-latest", - name="url-extractor", - instructions="Extract and summarize content from web pages. Return clean markdown.", - tools=[{"type": "web_search"}], # type: ignore[arg-type] - ) - - try: - # Start conversation to extract the URL - result = client.beta.conversations.start( - agent_id=agent.id, - inputs=f"Extract the main content from this URL and return it as markdown: {url}", - ) - - content = "" - for entry in result.outputs: - if hasattr(entry, "content") and entry.content is not None: - # In newer mistralai, content might be a list of chunks - if isinstance(entry.content, str): - content += entry.content - elif isinstance(entry.content, list): - for chunk in entry.content: - if hasattr(chunk, "text") and chunk.text: - content += chunk.text - elif isinstance(chunk, str): - content += chunk - - if not content: - logger.warning("Mistral browser returned empty content for URL: %s", url) - return None - - resolved = ResolvedResult( - source="mistral-browser", content=content[:max_chars], url=url - ) - _save_to_cache(url, "mistral_browser", resolved.to_dict()) - return resolved - finally: - # Clean up the agent - try: - client.beta.agents.delete(agent_id=agent.id) - except Exception as e: - logger.warning("Mistral browser agent cleanup failed: %s", e) - except Exception as e: - status = getattr(e, "status_code", None) - if status == 401: - logger.warning( - "Mistral browser failed: 401 Unauthorized — API key may be invalid or expired" - ) - elif status == 429: - logger.warning("Mistral browser failed: 429 Rate limited — setting cooldown") - _set_rate_limit("mistral") - elif status == 403: - logger.warning("Mistral browser failed: 403 Forbidden — %s", e) - else: - logger.warning("Mistral browser failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_mistral_websearch(query: str, max_chars: int = MAX_CHARS) -> ResolvedResult | None: - cached = _get_from_cache(query, "mistral_websearch") - if cached: - return ResolvedResult(**cached) - api_key = os.getenv("MISTRAL_API_KEY") - if not api_key: - logger.debug("Mistral websearch skipped: no API key") - return None - if _is_rate_limited("mistral"): - logger.debug("Mistral websearch skipped: rate limited") - return None - try: - from mistralai.client import Mistral - from mistralai.client.models import UserMessage - - client = Mistral(api_key=api_key) - resp = client.chat.complete( - model="mistral-small-latest", - messages=[UserMessage(content=f"Search: {query}")], # type: ignore[arg-type] - ) - content = "" - if resp.choices and resp.choices[0].message and resp.choices[0].message.content: - msg_content = resp.choices[0].message.content - if isinstance(msg_content, str): - content = msg_content - elif isinstance(msg_content, list): - # Handle list of chunks if necessary - for chunk in msg_content: - if hasattr(chunk, "text") and chunk.text: - content += chunk.text - elif isinstance(chunk, str): - content += chunk - if not content: - logger.warning("Mistral websearch returned empty content for query: %s", query) - return None - result = ResolvedResult( - source="mistral-websearch", content=content[:max_chars], query=query - ) - _save_to_cache(query, "mistral_websearch", result.to_dict()) - return result - except Exception as e: - status = getattr(e, "status_code", None) - if status == 401: - logger.warning( - "Mistral websearch failed: 401 Unauthorized — API key may be invalid or expired" - ) - elif status == 429: - logger.warning("Mistral websearch failed: 429 Rate limited — setting cooldown") - _set_rate_limit("mistral") - elif status == 403: - logger.warning("Mistral websearch failed: 403 Forbidden — %s", e) - else: - logger.warning("Mistral websearch failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_docling(url: str, max_chars: int) -> ResolvedResult | None: - if not is_safe_url(url): - logger.warning("SSRF blocked: %s", url) - return None - try: - res = subprocess.run( - ["docling", "--format", "markdown", url], capture_output=True, text=True, timeout=60 - ) - if res.returncode == 0: - return ResolvedResult(source="docling", content=res.stdout[:max_chars], url=url) - except (subprocess.SubprocessError, OSError) as e: - logger.warning("Docling resolution failed: %s: %s", type(e).__name__, e) - return None - - -def resolve_with_ocr(url: str, max_chars: int) -> ResolvedResult | None: - if not is_safe_url(url): - logger.warning("SSRF blocked: %s", url) - return None - try: - res = subprocess.run( - ["tesseract", url, "stdout"], capture_output=True, text=True, timeout=30 - ) - if res.returncode == 0: - return ResolvedResult(source="ocr-tesseract", content=res.stdout[:max_chars], url=url) - except (subprocess.SubprocessError, OSError) as e: - logger.warning("OCR resolution failed: %s: %s", type(e).__name__, e) - return None +from scripts.utils import get_session + +__all__ = [ + "resolve_with_jina", + "resolve_with_exa", + "resolve_with_exa_mcp", + "resolve_with_tavily", + "resolve_with_serper", + "resolve_with_duckduckgo", + "resolve_with_firecrawl", + "resolve_with_mistral_browser", + "resolve_with_mistral_websearch", + "resolve_with_docling", + "resolve_with_ocr", + "resolve_with_stealth", + "resolve_with_visual_clip", + "resolve_with_visual_clip_async", + "_is_rate_limited", + "_set_rate_limit", + "_clear_rate_limits", + "_rate_limits", + "is_rate_limited", + "set_rate_limit", + "get_session", +] diff --git a/.agents/skills/do-web-doc-resolver/scripts/quality.py b/.agents/skills/do-web-doc-resolver/scripts/quality.py index b7150095..ff8db32f 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/quality.py +++ b/.agents/skills/do-web-doc-resolver/scripts/quality.py @@ -169,9 +169,8 @@ def _compute_bonuses(score: float, has_frontmatter: bool, has_anchors: bool) -> def score_content(markdown: str, links: list[str] | None = None) -> QualityScore: - # Handle MagicMocks in tests if not isinstance(markdown, str): - return QualityScore(1.0, False, False, False, False, True) + return QualityScore(0.0, True, True, False, False, False) text = (markdown or "").strip() links = links or [] diff --git a/.agents/skills/do-web-doc-resolver/scripts/resolve.py b/.agents/skills/do-web-doc-resolver/scripts/resolve.py index 80a69ba1..21afdb94 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/resolve.py +++ b/.agents/skills/do-web-doc-resolver/scripts/resolve.py @@ -4,11 +4,13 @@ Main orchestrator. CLI entrypoint moved to scripts/cli.py. """ +import asyncio import logging from typing import Any import scripts._query_resolve import scripts._url_resolve +import scripts._url_resolve_async import scripts.providers_impl import scripts.semantic_cache import scripts.synthesis @@ -35,9 +37,10 @@ resolve_with_ocr, resolve_with_serper, resolve_with_tavily, + resolve_with_visual_clip, ) from scripts.semantic_cache import get_semantic_cache -from scripts.state import circuit_breakers, get_executor, routing_memory +from scripts.state import circuit_breakers, routing_memory from scripts.utils import ( _cache_key, _detect_error_type, @@ -92,6 +95,10 @@ def _store_in_semantic_cache(query_or_url: str, result: dict) -> bool: "resolve_with_order", "resolve_url_with_order", "resolve_query_with_order", + "resolve_async", + "resolve_url_async", + "resolve_background", + "resolve_url_background", "ResolvedResult", "ValidationResult", "ErrorType", @@ -107,19 +114,18 @@ def _store_in_semantic_cache(query_or_url: str, result: dict) -> bool: "_detect_error_type", "_is_rate_limited", "_set_rate_limit", + "_rate_limits", "get_session", "_get_from_cache", "_save_to_cache", "_cache_key", "_get_cache", "get_cache", - "_rate_limits", "_cache", "_check_semantic_cache", "_store_in_semantic_cache", "circuit_breakers", "routing_memory", - "get_executor", ] @@ -127,6 +133,7 @@ def _store_in_semantic_cache(query_or_url: str, result: dict) -> bool: resolve_url_stream = scripts._url_resolve.resolve_url_stream resolve_query = scripts._query_resolve.resolve_query resolve_query_stream = scripts._query_resolve.resolve_query_stream +resolve_url_stream_async = scripts._url_resolve_async.resolve_url_stream_async def synthesize_results(query: str, results: list[ResolvedResult], api_key: str, model: str) -> str: @@ -138,12 +145,15 @@ def resolve( max_chars: int = MAX_CHARS, skip_providers: set[str] | None = None, profile: Profile | str = Profile.BALANCED, + query: str | None = None, ) -> dict[str, Any]: if isinstance(profile, str): profile = Profile(profile.lower()) if is_url(input_str): - return resolve_url(input_str, max_chars, profile=profile) + return resolve_url( + input_str, max_chars, profile=profile, query=query, skip_providers=skip_providers + ) return resolve_query(input_str, max_chars, skip_providers, profile=profile) @@ -168,6 +178,7 @@ def resolve_direct( ProviderType.SERPER: resolve_with_serper, ProviderType.DOCLING: resolve_with_docling, ProviderType.OCR: resolve_with_ocr, + ProviderType.VISUAL_CLIP: resolve_with_visual_clip, } if provider in funcs: res = funcs[provider](input_str, max_chars) @@ -195,3 +206,61 @@ def resolve_query_with_order( query: str, order: list[ProviderType], max_chars: int = MAX_CHARS ) -> dict[str, Any]: return resolve_with_order(query, order, max_chars) + + +# Async entry points + + +async def resolve_url_async( + url: str, + max_chars: int = MAX_CHARS, + profile: Profile | str = Profile.BALANCED, + query: str | None = None, + skip_providers: set[str] | None = None, +) -> dict[str, Any]: + """Async version of resolve_url.""" + if isinstance(profile, str): + profile = Profile(profile.lower()) + return await scripts._url_resolve_async.resolve_url_async( + url, max_chars, profile, query=query, skip_providers=skip_providers + ) + + +async def resolve_async( + input_str: str, + max_chars: int = MAX_CHARS, + skip_providers: set[str] | None = None, + profile: Profile | str = Profile.BALANCED, + query: str | None = None, +) -> dict[str, Any]: + """Async version of resolve.""" + if isinstance(profile, str): + profile = Profile(profile.lower()) + if is_url(input_str): + return await resolve_url_async( + input_str, max_chars, profile=profile, query=query, skip_providers=skip_providers + ) + return resolve_query(input_str, max_chars, skip_providers, profile=profile) + + +def resolve_url_background( + url: str, + max_chars: int = MAX_CHARS, + profile: Profile | str = Profile.BALANCED, + query: str | None = None, + skip_providers: set[str] | None = None, +) -> dict[str, Any]: + """Run async resolve_url in a new event loop (for sync callers).""" + return asyncio.run( + resolve_url_async(url, max_chars, profile, query=query, skip_providers=skip_providers) + ) + + +def resolve_background( + input_str: str, + max_chars: int = MAX_CHARS, + skip_providers: set[str] | None = None, + profile: Profile | str = Profile.BALANCED, +) -> dict[str, Any]: + """Run async resolve in a new event loop (for sync callers).""" + return asyncio.run(resolve_async(input_str, max_chars, skip_providers, profile)) diff --git a/.agents/skills/do-web-doc-resolver/scripts/routing.py b/.agents/skills/do-web-doc-resolver/scripts/routing.py index 7ff0984a..13667102 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/routing.py +++ b/.agents/skills/do-web-doc-resolver/scripts/routing.py @@ -2,12 +2,16 @@ Budget-aware routing logic for the Web Doc Resolver. """ +import logging import os +import re from dataclasses import dataclass from urllib.parse import urlparse from scripts.routing_memory import RoutingMemory +logger = logging.getLogger(__name__) + DEFAULT_MIN_FREE_QUALITY = float(os.getenv("DO_WDR_MIN_FREE_QUALITY_TO_SKIP_PAID", "0.70")) @@ -83,6 +87,7 @@ def extract_domain(url: str) -> str | None: hostname = parsed.hostname return hostname.lower() if hostname else None except Exception: + logger.debug("Failed to extract domain from URL: %s", url, exc_info=True) return None @@ -91,6 +96,7 @@ def detect_doc_platform(url: str) -> str | None: try: parsed = urlparse(url) except Exception: + logger.debug("Failed to parse URL for platform detection: %s", url, exc_info=True) return None hostname = (parsed.hostname or "").lower() @@ -112,8 +118,8 @@ def detect_doc_platform(url: str) -> str | None: return "notion" if ( (hostname.endswith(".atlassian.net") and path.startswith("/wiki")) - or "confluence" in hostname - or "confluence" in path + or bool(re.search(r"\bconfluence\b", hostname)) + or bool(re.search(r"\bconfluence\b", path)) ): return "confluence" @@ -197,13 +203,21 @@ def plan_provider_order( strategy = preflight.get("preferred_strategy", "llms_txt") if platform in ("notion", "confluence") or preflight.get("js_heavy"): - base = ["firecrawl", "mistral_browser", "jina", "direct_fetch", "duckduckgo"] + base = [ + "jina", + "firecrawl", + "visual_clip", + "mistral_browser", + "direct_fetch", + "duckduckgo", + ] elif strategy == "direct_fetch": base = [ "direct_fetch", "llms_txt", "jina", "firecrawl", + "visual_clip", "mistral_browser", "duckduckgo", ] @@ -212,13 +226,15 @@ def plan_provider_order( "llms_txt", "jina", "firecrawl", - "direct_fetch", + "visual_clip", "mistral_browser", + "direct_fetch", "duckduckgo", ] else: # DuckDuckGo deprioritized due to instability (Alert 2026-04-20) - base = ["exa_mcp", "exa", "tavily", "serper", "mistral_websearch", "duckduckgo"] + # Serper deprioritized due to instability (Alert 2026-07-20) + base = ["exa_mcp", "exa", "tavily", "mistral_websearch", "duckduckgo", "serper"] skip_providers = skip_providers or set() diff --git a/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py b/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py index 02bf3a69..3385cf9b 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py +++ b/.agents/skills/do-web-doc-resolver/scripts/routing_memory.py @@ -2,11 +2,15 @@ Per-domain routing memory for the Web Doc Resolver. """ +import json import logging import math +import os import threading import time from collections import defaultdict +from pathlib import Path +from typing import Any, cast from scripts._routing_utils import DEFAULT_PROVIDER_STATS, compute_p75_latency @@ -15,47 +19,117 @@ RECENCY_DECAY_DAYS = 7.0 SCORE_SCALE = 1000.0 +# Minimum seconds between disk writes; bounds I/O while retaining durability. +SAVE_INTERVAL_SECONDS = 5.0 + class RoutingMemory: - def __init__(self): + def __init__(self, path: str | os.PathLike[str] | None = None) -> None: # domain -> provider -> stats - self.domain_stats = defaultdict(lambda: defaultdict(lambda: dict(DEFAULT_PROVIDER_STATS))) + self.domain_stats: dict[str, dict[str, dict[str, Any]]] = defaultdict( + lambda: defaultdict(lambda: dict(DEFAULT_PROVIDER_STATS)) + ) self._lock = threading.RLock() + self._path = Path(path) if path is not None else None + self._dirty = False + self._last_save = 0.0 + if self._path is not None: + self._load_from_disk() + + # --- Persistence ------------------------------------------------------- + + def _load_from_disk(self) -> None: + if self._path is None or not self._path.exists(): + return + try: + with self._path.open("r", encoding="utf-8") as fh: + raw = json.load(fh) + for domain, providers in raw.items(): + for provider, stats in providers.items(): + # Sanitize each entry against the default shape so corrupt or + # partial files degrade gracefully instead of crashing rank(). + merged = dict(DEFAULT_PROVIDER_STATS) + if isinstance(stats, dict): + merged.update( + {k: v for k, v in stats.items() if k in merged and v is not None} + ) + self.domain_stats[str(domain)][str(provider)] = merged + logger.debug("Loaded routing memory from %s", self._path) + except (OSError, ValueError, TypeError) as e: + logger.warning("Failed to load routing memory from %s: %s", self._path, e) + + def _save_to_disk_unlocked(self) -> None: + if self._path is None: + return + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + data = {d: dict(ps) for d, ps in self.domain_stats.items()} + tmp = self._path.with_suffix(self._path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8") as fh: + json.dump(data, fh, sort_keys=True) + tmp.replace(self._path) + self._dirty = False + self._last_save = time.time() + except OSError as e: + logger.warning("Failed to save routing memory to %s: %s", self._path, e) + + def save(self) -> None: + """Flush routing memory to disk (no-op when no path is configured).""" + with self._lock: + self._save_to_disk_unlocked() def record( self, domain: str, provider: str, success: bool, latency_ms: int, quality_score: float ) -> None: with self._lock: stats = self.domain_stats[domain][provider] - total = stats["success"] + stats["failure"] - stats["avg_latency_ms"] = ((stats["avg_latency_ms"] * total) + latency_ms) / (total + 1) - stats["avg_quality"] = ((stats["avg_quality"] * total) + quality_score) / (total + 1) + s = cast(int, stats.get("success", 0)) + f = cast(int, stats.get("failure", 0)) + total = s + f + + avg_lat = cast(float, stats.get("avg_latency_ms", 0.0)) + avg_qual = cast(float, stats.get("avg_quality", 0.0)) + + stats["avg_latency_ms"] = ((avg_lat * total) + float(latency_ms)) / (total + 1) + stats["avg_quality"] = ((avg_qual * total) + float(quality_score)) / (total + 1) stats["last_attempted"] = time.time() if success: - stats["success"] += 1 + stats["success"] = s + 1 else: - stats["failure"] += 1 + stats["failure"] = f + 1 + + # Throttled auto-persist so a running CLI retains learned preferences. + if self._path is not None and ( + self._dirty is False or time.time() - self._last_save >= SAVE_INTERVAL_SECONDS + ): + self._dirty = True + self._save_to_disk_unlocked() - def get_domain_stats(self, provider: str, domain: str) -> dict | None: + def get_domain_stats(self, provider: str, domain: str) -> dict[str, Any] | None: with self._lock: - if domain not in self.domain_stats or provider not in self.domain_stats[domain]: + domain_dict = self.domain_stats.get(domain) + if not domain_dict: + return None + stats = domain_dict.get(provider) + if not stats: return None - stats = self.domain_stats[domain][provider] - attempts = stats.get("success", 0) + stats.get("failure", 0) + s = cast(int, stats.get("success", 0)) + f = cast(int, stats.get("failure", 0)) + attempts = s + f if attempts == 0: return None - success_rate = stats.get("success", 0) / max(attempts, 1) + success_rate = float(s) / max(attempts, 1) days_since_last = 0.0 - last = stats.get("last_attempted") + last = cast(float | None, stats.get("last_attempted")) if last: days_since_last = (time.time() - last) / 86400.0 return { "attempts": attempts, "success_rate": success_rate, - "avg_latency_ms": stats.get("avg_latency_ms", 0), + "avg_latency_ms": stats.get("avg_latency_ms", 0.0), "avg_quality": stats.get("avg_quality", 0.5), "days_since_last": days_since_last, } @@ -97,11 +171,15 @@ def rank(self, domain: str, providers: list[str]) -> list[str]: def get_p75_latency(self, domain: str, provider: str, default: int = 3000) -> int: with self._lock: - stats = self.domain_stats.get(domain, {}).get(provider) + domain_dict = self.domain_stats.get(domain) + if not domain_dict: + return default + stats = domain_dict.get(provider) if not stats: return default - return compute_p75_latency(stats["avg_latency_ms"], default) + return compute_p75_latency(cast(float, stats["avg_latency_ms"]), default) def clear(self) -> None: with self._lock: self.domain_stats.clear() + self._dirty = False diff --git a/.agents/skills/do-web-doc-resolver/scripts/state.py b/.agents/skills/do-web-doc-resolver/scripts/state.py index 6d13fa8c..bf20990a 100644 --- a/.agents/skills/do-web-doc-resolver/scripts/state.py +++ b/.agents/skills/do-web-doc-resolver/scripts/state.py @@ -1,49 +1,31 @@ """Shared mutable state for the Web Doc Resolver — eliminates monkey-patching.""" -import atexit -import concurrent.futures +import os from dataclasses import dataclass, field -from typing import Any from scripts.circuit_breaker import CircuitBreakerRegistry from scripts.routing_memory import RoutingMemory +def _routing_memory_factory() -> RoutingMemory: + """Build RoutingMemory, persisting to disk when DO_WDR_ROUTING_MEMORY_PATH is set. + + Defaults to in-memory so library/test usage stays side-effect free; the CLI + sets DO_WDR_ROUTING_MEMORY_PATH before importing this module to retain + learned provider preferences across runs (AUDIT #25). + """ + path = os.getenv("DO_WDR_ROUTING_MEMORY_PATH") + if not path: + return RoutingMemory() + return RoutingMemory(path=path) + + @dataclass class ResolverState: circuit_breakers: CircuitBreakerRegistry = field(default_factory=CircuitBreakerRegistry) - routing_memory: RoutingMemory = field(default_factory=RoutingMemory) - semantic_cache: Any = None - executor: concurrent.futures.ThreadPoolExecutor | None = None + routing_memory: RoutingMemory = field(default_factory=_routing_memory_factory) _state = ResolverState() circuit_breakers = _state.circuit_breakers routing_memory = _state.routing_memory - -_executor: concurrent.futures.ThreadPoolExecutor | None = None - - -def get_state() -> ResolverState: - return _state - - -def get_executor(max_workers: int = 10) -> concurrent.futures.ThreadPoolExecutor: - global _executor - if _executor is None: - _executor = concurrent.futures.ThreadPoolExecutor( - max_workers=max_workers, thread_name_prefix="resolver" - ) - _state.executor = _executor - return _executor - - -def _shutdown_executor() -> None: - global _executor - if _executor is not None: - _executor.shutdown(wait=False) - _executor = None - _state.executor = None - - -atexit.register(_shutdown_executor) diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils.py b/.agents/skills/do-web-doc-resolver/scripts/utils.py deleted file mode 100644 index 12695d9a..00000000 --- a/.agents/skills/do-web-doc-resolver/scripts/utils.py +++ /dev/null @@ -1,789 +0,0 @@ -""" -Utility functions for the Web Doc Resolver. -""" - -import hashlib -import ipaddress -import logging -import os -import re -import socket -import threading -import time -import typing -from concurrent.futures import ThreadPoolExecutor -from functools import lru_cache -from html.parser import HTMLParser -from typing import Any -from urllib.parse import parse_qs, urlencode, urljoin, urlparse - -import requests -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry - -from scripts.constants import ( - BLOCKED_HOSTNAMES, - BLOCKED_NETWORKS, - BLOCKED_SCHEMES, - CACHE_DIR, - DEFAULT_TIMEOUT, - DNS_CACHE_TTL, - MAX_CHARS, - TIERED_TTL, - USER_AGENT, -) -from scripts.models import ErrorType, ResolvedResult, ValidationResult - -logger = logging.getLogger(__name__) - -_CONFIG_DATA: dict[str, Any] | None = None - - -def get_config_data() -> dict[str, Any]: - """Load configuration from config.toml if available.""" - global _CONFIG_DATA # noqa: W0603 - if _CONFIG_DATA is not None: - return _CONFIG_DATA - - _CONFIG_DATA = {} - config_path = os.getenv("DO_WDR_CONFIG") or "config.toml" - if os.path.exists(config_path): - try: - try: - import tomllib - except ImportError: - import tomli as tomllib # type: ignore - - with open(config_path, "rb") as f: - _CONFIG_DATA = typing.cast(dict[str, Any], tomllib.load(f)) - except Exception as e: - logger.debug("Failed to load config.toml: %s", e) - - return _CONFIG_DATA - - -_global_session: requests.Session | None = None -_session_lock = threading.Lock() -_cache = None -_cache_lock = threading.RLock() - - -def create_session_with_retry() -> requests.Session: - session = requests.Session() - retry_strategy = Retry( - total=3, - backoff_factor=1.0, - status_forcelist=[429, 500, 502, 503, 504], - allowed_methods=["HEAD", "GET", "OPTIONS"], - raise_on_status=False, - ) - adapter = HTTPAdapter(max_retries=retry_strategy) - session.mount("http://", adapter) - session.mount("https://", adapter) - session.headers.update( - { - "User-Agent": USER_AGENT, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - } - ) - return session - - -def get_session() -> requests.Session: - global _global_session # noqa: W0603 - with _session_lock: - if _global_session is None: - _global_session = create_session_with_retry() - return _global_session - - -def close_session() -> None: - global _global_session # noqa: W0603 - with _session_lock: - if _global_session is not None: - _global_session.close() - _global_session = None - - -def _safe_request( - method: str, - url: str, - session: requests.Session | None = None, - *, - max_redirects: int = 5, - **kwargs, -) -> requests.Response: - """Perform an HTTP request while validating each redirect hop for SSRF.""" - - current_url = url - history: list[requests.Response] = [] - # Ensure we control redirect behavior - kwargs.pop("allow_redirects", None) - active_session = session or get_session() - - for _ in range(max_redirects + 1): - if not is_safe_url(current_url): - raise requests.RequestException(f"SSRF blocked: {current_url}") - - response = active_session.request(method, current_url, allow_redirects=False, **kwargs) - response.history = list(history) - - if response.is_redirect: - history.append(response) - location = response.headers.get("Location") - if not location: - break - next_url = location - if not urlparse(next_url).netloc: - next_url = urljoin(current_url, next_url) - current_url = next_url - continue - - return response - - raise requests.TooManyRedirects(f"Exceeded {max_redirects} redirects") - - -@lru_cache(maxsize=1024) -def _getaddrinfo_bucketed(host: str, port: int | str | None, bucket: int) -> list[tuple]: # noqa: W0613 - """Internal helper for cached getaddrinfo using time-bucketing.""" - return socket.getaddrinfo(host, port) - - -def _getaddrinfo_cached(host: str, port: int | str | None = None) -> list[tuple]: - """Cached version of socket.getaddrinfo with TTL to balance performance and security.""" - bucket = int(time.time() // DNS_CACHE_TTL) - return _getaddrinfo_bucketed(host, port, bucket) - - -def _normalize_host(hostname: str) -> str: - """Normalise encoded IP representations to dotted-decimal.""" - h = hostname.strip().lower() - if h.isdigit(): - try: - return str(ipaddress.IPv4Address(int(h))) - except (ValueError, OverflowError): - pass - if h.startswith("0x"): - try: - return str(ipaddress.IPv4Address(int(h, 16))) - except (ValueError, OverflowError): - pass - if h.startswith("::ffff:"): - return h[7:] - return h - - -def is_safe_url(url: str) -> bool: # noqa: R1000 - try: - parsed = urlparse(url) - if parsed.scheme.lower() in BLOCKED_SCHEMES: - return False - if parsed.scheme not in ("http", "https"): - return False - hostname = parsed.hostname - if not hostname: - return False - normalized = _normalize_host(hostname) - if normalized in ( # noqa: BAN-B104 - "localhost", - "localhost.localdomain", - "127.0.0.1", - "::1", - "0.0.0.0", - ): - return False - hostname_lower = hostname.lower().strip(".") - if hostname_lower in BLOCKED_HOSTNAMES: - return False - if any(hostname_lower.endswith("." + blocked) for blocked in BLOCKED_HOSTNAMES): - return False - try: - ip = ipaddress.ip_address(normalized) - if any(ip in network for network in BLOCKED_NETWORKS): - return False - except ValueError: - try: - infos = _getaddrinfo_cached(hostname, None) - for _family, _socktype, _proto, _canonname, sockaddr in infos: - ip = ipaddress.ip_address(sockaddr[0]) - if any(ip in network for network in BLOCKED_NETWORKS): - return False - except Exception: - pass - if normalized.endswith(".local") or normalized.endswith(".internal"): - return False - return True - except Exception: - return False - - -def is_url(input_str: str) -> bool: - if not input_str: - return False - trimmed = input_str.strip() - if not trimmed: - return False - # Fast path: must start with http (case-insensitive check without full lower() allocation) - prefix = trimmed[:8].lower() - if not prefix.startswith(("http://", "https://")): - return False - try: - result = urlparse(trimmed) - return all([result.scheme in {"http", "https"}, result.netloc]) - except Exception: - return False - - -def validate_url(url: str, timeout: int = 10, check_ssrf: bool = True) -> ValidationResult: - if not url or not url.strip(): - return ValidationResult(is_valid=False, error="Empty URL") - if not is_url(url): - return ValidationResult(is_valid=False, error="Invalid URL format") - try: - session = get_session() - if check_ssrf: - response = _safe_request("HEAD", url, session=session, timeout=timeout, verify=True) - else: - response = session.head(url, timeout=timeout, allow_redirects=True, verify=True) - redirect_chain = [h.url for h in response.history] + [response.url] - if response.status_code >= 400: - return ValidationResult( - is_valid=False, - status_code=response.status_code, - error=f"HTTP {response.status_code}", - final_url=response.url, - redirect_chain=redirect_chain, - ) - return ValidationResult( - is_valid=True, - status_code=response.status_code, - final_url=response.url, - redirect_chain=redirect_chain, - content_type=response.headers.get("Content-Type", ""), - ) - except Exception as e: - return ValidationResult(is_valid=False, error=str(e)) - - -def _validate_single_link(link: str, timeout: int, session: requests.Session) -> str | None: - try: - response = _safe_request("HEAD", link, session=session, timeout=timeout, verify=True) - if response.status_code < 400: - return link - except Exception: - return None - return None - - -def validate_links(links: list[str], timeout: int = 5) -> list[str]: - """Validate a list of links in parallel, preserving input order.""" - if not links: - return [] - - session = get_session() - max_workers = min(10, len(links)) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - results = list( - executor.map(lambda link: _validate_single_link(link, timeout, session), links) - ) - - return [link for link in results if link] - - -def score_result(url: str | None, content: str) -> float: - score = 0.5 - if url: - try: - domain = urlparse(url).netloc.lower() - if any(domain.endswith(tld) for tld in [".edu", ".gov", ".org", ".rs", ".io"]): - score += 0.2 - if any( - site in domain - for site in ["github.com", "stackoverflow.com", "docs.rs", "mozilla.org"] - ): - score += 0.2 - except Exception: - pass - word_count = len(content.split()) - if word_count > 500: - score += 0.1 - elif word_count < 50: - score -= 0.2 - return max(0.0, min(1.0, score)) - - -def compact_content(content: str, max_chars: int) -> str: - lines = content.splitlines() - unique_lines = set() - compacted = [] - for line in lines: - trimmed = line.strip() - if not trimmed: - compacted.append("") - continue - if trimmed not in unique_lines: - compacted.append(trimmed) - unique_lines.add(trimmed) - return "\n".join(compacted)[:max_chars] - - -def _strip_html_tags(html: str) -> str: - """Minimal fallback: strip all HTML tags.""" - text = re.sub(r"<[^>]+>", " ", html) - return re.sub(r"\s+", " ", text).strip() - - -def clean_content( - html: str, - url: str = "", - max_chars: int = 32_000, - favor_recall: bool = False, -) -> str: - """Extract main content from HTML, removing boilerplate for LLM efficiency.""" - if not html or not html.strip(): - return "" - - try: - import trafilatura - - result = trafilatura.extract( - html, - url=url or None, - include_tables=favor_recall, - include_links=favor_recall, - include_images=False, - favor_precision=not favor_recall, - output_format="txt", - deduplicate=True, - ) - if result and len(result.strip()) > 200: - return result[:max_chars] - except ImportError: - pass - except Exception: - pass - - try: - from readability import Document # type: ignore[import] - - doc = Document(html) - summary_html = doc.summary() - text = _strip_html_tags(summary_html) - if text and len(text.strip()) > 200: - return text[:max_chars] - except ImportError: - pass - except Exception: - pass - - return _strip_html_tags(html)[:max_chars] - - -class EnhancedHTMLParser(HTMLParser): - _block_tags = { - "p", - "div", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "li", - "tr", - "article", - "section", - "header", - "footer", - "nav", - "aside", - "blockquote", - "ul", - "ol", - "table", - "hr", - } - - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.result: list[str] = [] - self._skip_depth = 0 - self._in_pre = 0 - - def handle_starttag(self, tag, attrs): - tag_lower = tag.lower() - if tag_lower in ("script", "style"): - self._skip_depth += 1 - elif self._skip_depth == 0: - if tag_lower == "pre": - self._in_pre += 1 - self.result.append("\n\n```\n") - elif tag_lower == "br": - self.result.append("\n") - elif tag_lower == "hr": - self.result.append("\n\n---\n\n") - elif tag_lower in ( - "p", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "blockquote", - "ul", - "ol", - "table", - ): - self.result.append("\n\n") - elif tag_lower in self._block_tags: - self.result.append("\n") - - if tag_lower == "code": - self.result.append("`") - - def handle_endtag(self, tag): - tag_lower = tag.lower() - if tag_lower in ("script", "style") and self._skip_depth > 0: - self._skip_depth -= 1 - elif self._skip_depth == 0: - if tag_lower == "pre": - self._in_pre = max(0, self._in_pre - 1) - self.result.append("\n```\n\n") - elif tag_lower == "code": - self.result.append("`") - elif tag_lower in ( - "p", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "blockquote", - "ul", - "ol", - "table", - ): - self.result.append("\n\n") - elif tag_lower in self._block_tags: - self.result.append("\n") - - def handle_data(self, data): - if self._skip_depth == 0: - if self._in_pre > 0: - self.result.append(data) - else: - # Fast path: only sub if multiple spaces or tabs present - if "\t" in data or " " in data: - normalized = _RE_SPACES.sub(" ", data) - else: - normalized = data - - if normalized: - # Prevent double spaces across chunks - if normalized.startswith(" ") and self.result and self.result[-1].endswith(" "): - normalized = normalized[1:] - if normalized: - self.result.append(normalized) - - -_RE_SPACES = re.compile(r"[ \t]+") -_RE_NEWLINES = re.compile(r"\n{3,}") - - -def extract_text_from_html(html: str, base_url: str = "") -> str: - stripper = EnhancedHTMLParser() - stripper.feed(html) - text = "".join(stripper.result) - # Normalize word joiner and other problematic characters - if "\u2060" in text: - text = text.replace("\u2060", "") - # Note: _RE_SPACES is handled per-chunk in handle_data to preserve code blocks. - if "\n\n\n" in text: - text = _RE_NEWLINES.sub("\n\n", text) - return text.strip() - - -def fetch_url_content( - url: str, timeout: int = DEFAULT_TIMEOUT, max_chars: int = MAX_CHARS -) -> ResolvedResult | None: - validation = validate_url(url, timeout=timeout // 2) - if not validation.is_valid: - return None - try: - session = get_session() - response = _safe_request("GET", url, session=session, timeout=timeout, verify=True) - if response.status_code >= 400: - return None - content = ( - extract_text_from_html(response.text, url) - if "text/html" in response.headers.get("Content-Type", "") - else response.text - ) - return ResolvedResult( - source="direct_fetch", - content=content[:max_chars], - url=validation.final_url or url, - metadata={"status_code": response.status_code}, - ) - except Exception: - return None - - -def fetch_llms_txt(url: str) -> str | None: - try: - if not is_safe_url(url): - return None - parsed = urlparse(url) - base_url = f"{parsed.scheme}://{parsed.netloc}" - llms_url = f"{base_url}/llms.txt" - cached = _get_from_cache(base_url, "llms_txt") - if cached is not None: - if cached.get("found"): - return str(cached.get("content", "")) - return None - session = get_session() - response = _safe_request("GET", llms_url, session=session, timeout=10) - if response.status_code == 200: - content_type = response.headers.get("Content-Type", "") - if "text" in content_type or "markdown" in content_type: - _save_to_cache( - base_url, - "llms_txt", - {"found": True, "content": response.text}, - ttl=get_ttl("llms_txt"), - ) - return response.text - _save_to_cache(base_url, "llms_txt", {"found": False}, ttl=get_ttl("llms_txt")) - except Exception: - pass - return None - - -_TRACKING_PARAMS = { - # UTM parameters - "utm_source", - "utm_medium", - "utm_campaign", - "utm_term", - "utm_content", - # Social/tracking parameters - "fbclid", - "gclid", - "gclsrc", - "dclid", - "msclkid", - "twclid", - "li_fat_id", - "mc_cid", - "mc_eid", - # Referral/tracking - "ref", - "ref_src", - "ref_url", - "source", - "via", - # Session/tracking - "session_id", - "sid", - "_ga", - "_gl", - # Misc tracking - "hsa_cam", - "hsa_grp", - "hsa_mt", - "hsa_src", - "hsa_ad", - "hsa_acc", - "hsa_net", - "hsa_kw", - "hsa_tgt", - "hsa_ver", -} - - -def normalize_url(url: str) -> str: - """Normalize URL by stripping tracking params, anchors, and common aliases.""" - try: - parsed = urlparse(url) - # Strip all known tracking params - if parsed.query: - params = parse_qs(parsed.query) - filtered_params = { - k: v - for k, v in params.items() - if k.lower() not in _TRACKING_PARAMS and not k.startswith("utm_") - } - query = urlencode(filtered_params, doseq=True) - else: - query = "" - - # Normalize fragment: strip if empty or just a section ref - fragment = "" if not parsed.fragment else parsed.fragment - - # Normalize trailing slash (keep for root, strip for paths) - path = parsed.path - if path and path != "/" and path.endswith("/"): - path = path.rstrip("/") - - # Normalize netloc: lowercase, strip default ports - netloc = parsed.netloc.lower() - if netloc.endswith(":80") and parsed.scheme == "http": - netloc = netloc[:-3] - elif netloc.endswith(":443") and parsed.scheme == "https": - netloc = netloc[:-4] - - # Reconstruct - normalized = parsed._replace( - scheme=parsed.scheme.lower(), netloc=netloc, path=path, query=query, fragment=fragment - ).geturl() - return normalized.strip() - except Exception: - return url.lower().strip() - - -def normalize_query(query: str) -> str: - """Normalize search query.""" - # Lowercase, trim whitespace, and collapse multiple spaces - # Using split() and join() is significantly faster than re.sub for collapsing whitespace - return " ".join(query.lower().split()) - - -def _cache_key(input_str: str, source: str) -> str: - # Use normalized input for cache key - if is_url(input_str): - normalized = normalize_url(input_str) - else: - normalized = normalize_query(input_str) - - hash_input = f"{source}:{normalized}" - return hashlib.sha256(hash_input.encode()).hexdigest() - - -def _get_cache_proxy(): - import scripts.resolve - - if hasattr(scripts.resolve, "_cache") and scripts.resolve._cache is not None: # noqa: W0212 - return scripts.resolve._cache - return _cache - - -def get_cache(): - try: - import diskcache - - os.makedirs(CACHE_DIR, exist_ok=True) - return diskcache.Cache(CACHE_DIR) - except Exception: - return None - - -def _get_cache(): - global _cache # noqa: W0603 - with _cache_lock: - _cache = _get_cache_proxy() - if _cache is None: - _cache = get_cache() - return _cache - - -def get_ttl(provider: str, config: dict | None = None) -> int: - """Get the TTL for a given provider from config or defaults.""" - # Normalize provider name for alias support - provider_key = provider - if provider in ("exa_mcp", "exa"): - provider_key = "exa" - elif provider in ("mistral_browser", "mistral_websearch"): - provider_key = "mistral" - - # Use provided config or load from file - cfg = config if config is not None else get_config_data() - - # Environment variable override takes precedence over file-based config - env_key = f"DO_WDR_CACHE_TTL_{provider_key.upper()}" - if env_key in os.environ: - try: - return int(os.environ[env_key]) - except ValueError: - pass - - if cfg: - # Try to get from nested config.toml style - ttl_cfg = cfg.get("cache", {}).get("ttl", {}) - if provider_key in ttl_cfg: - return int(ttl_cfg[provider_key]) - if "default" in ttl_cfg: - return int(ttl_cfg["default"]) - - return TIERED_TTL.get(provider_key, TIERED_TTL.get("default", 3600)) - - -def _get_from_cache(input_str: str, source: str) -> dict[str, Any] | None: - with _cache_lock: - cache = _get_cache() - if not cache: - return None - with _cache_lock: - result = cache.get(_cache_key(input_str, source)) - if result is None: - return None - return dict(result) - - -def _save_to_cache(input_str: str, source: str, result: dict[str, Any], ttl: int | None = None): - with _cache_lock: - cache = _get_cache() - if not cache: - return - - if ttl is None: - ttl = get_ttl(source) - - with _cache_lock: - cache.set(_cache_key(input_str, source), result, expire=ttl) - - -def _detect_error_type(error: Exception) -> ErrorType: # noqa: R1000 - error_msg = str(error).lower() - if any(code in error_msg for code in ["429", "rate limit", "too many requests", "rate_limit"]): - return ErrorType.RATE_LIMIT - if any( - code in error_msg - for code in [ - "401", - "403", - "unauthorized", - "forbidden", - "invalid api key", - "invalid_key", - "authentication", - ] - ): - return ErrorType.AUTH_ERROR - if any( - code in error_msg - for code in [ - "402", - "payment", - "credit", - "quota", - "insufficient", - "exhausted", - "limit exceeded", - ] - ): - return ErrorType.QUOTA_EXHAUSTED - if any(code in error_msg for code in ["timeout", "timed out"]): - return ErrorType.TIMEOUT - if any(code in error_msg for code in ["connection", "network"]): - return ErrorType.NETWORK_ERROR - if any(code in error_msg for code in ["not found", "404"]): - return ErrorType.NOT_FOUND - if any(code in error_msg for code in ["ssrf", "blocked", "private ip", "localhost"]): - return ErrorType.SSRF_BLOCKED - if any(code in error_msg for code in ["too large", "content size", "exceeds"]): - return ErrorType.CONTENT_TOO_LARGE - return ErrorType.UNKNOWN diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/__init__.py b/.agents/skills/do-web-doc-resolver/scripts/utils/__init__.py new file mode 100644 index 00000000..19ff7190 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/__init__.py @@ -0,0 +1,175 @@ +""" +Utility package for the Web Doc Resolver. +""" + +import logging +import os +import typing +from typing import Any + +from scripts.utils.async_http import ( + async_safe_request, + async_validate_links, + async_validate_url, + close_async_client, + get_async_client, +) +from scripts.utils.cache import ( + _cache_key, + _get_cache, + _get_from_cache, + _save_to_cache, + get_cache, + get_ttl, +) +from scripts.utils.content_clean import clean_content +from scripts.utils.fetch import ( + fetch_llms_txt, + fetch_url_content, +) +from scripts.utils.html import ( + EnhancedHTMLParser, + compact_content, + extract_text_from_html, +) +from scripts.utils.http import ( + _safe_request, + close_session, + create_session_with_retry, + get_session, + is_safe_url, + validate_links, + validate_url, +) +from scripts.utils.urls import ( + is_url, + normalize_query, + normalize_url, + score_result, +) + +logger = logging.getLogger(__name__) + +_CONFIG_DATA: dict[str, Any] | None = None + + +def get_config_data() -> dict[str, Any]: + """Load configuration from config.toml if available.""" + global _CONFIG_DATA + if _CONFIG_DATA is not None: + return _CONFIG_DATA + + _CONFIG_DATA = {} + config_path = os.getenv("DO_WDR_CONFIG") or "config.toml" + if os.path.exists(config_path): + try: + try: + import tomllib + except ImportError: + import tomli as tomllib # type: ignore + + with open(config_path, "rb") as f: + _CONFIG_DATA = typing.cast(dict[str, Any], tomllib.load(f)) + except Exception as e: + logger.debug(f"Failed to load config.toml: {e}") + + return _CONFIG_DATA + + +def _detect_error_type(error: Exception): + from scripts.models import ErrorType + + error_msg = str(error).lower() + if any(code in error_msg for code in ["429", "rate limit", "too many requests", "rate_limit"]): + return ErrorType.RATE_LIMIT + if any( + code in error_msg + for code in [ + "401", + "403", + "unauthorized", + "forbidden", + "invalid api key", + "invalid_key", + "authentication", + ] + ): + return ErrorType.AUTH_ERROR + if any( + code in error_msg + for code in [ + "402", + "payment", + "credit", + "quota", + "insufficient", + "exhausted", + "limit exceeded", + ] + ): + return ErrorType.QUOTA_EXHAUSTED + if any(code in error_msg for code in ["timeout", "timed out"]): + return ErrorType.TIMEOUT + if any(code in error_msg for code in ["connection", "network"]): + return ErrorType.NETWORK_ERROR + if any(code in error_msg for code in ["not found", "404"]): + return ErrorType.NOT_FOUND + if any(code in error_msg for code in ["ssrf", "blocked", "private ip", "localhost"]): + return ErrorType.SSRF_BLOCKED + if any( + code in error_msg + for code in [ + "bot_challenge", + "bot challenge", + "cloudflare", + "cf-ray", + "checking your browser", + "just a moment", + ] + ): + return ErrorType.BOT_CHALLENGE + if any(code in error_msg for code in ["too large", "content size", "exceeds"]): + return ErrorType.CONTENT_TOO_LARGE + return ErrorType.UNKNOWN + + +__all__ = [ + # Config + "get_config_data", + # HTTP utilities (sync) + "create_session_with_retry", + "get_session", + "close_session", + "_safe_request", + "is_safe_url", + "validate_url", + "validate_links", + # HTTP utilities (async) + "get_async_client", + "close_async_client", + "async_safe_request", + "async_validate_url", + "async_validate_links", + # HTML utilities + "EnhancedHTMLParser", + "extract_text_from_html", + "compact_content", + "clean_content", + # Cache utilities + "_cache_key", + "_get_cache", + "_get_from_cache", + "_save_to_cache", + "get_cache", + "get_ttl", + # URL utilities + "is_url", + "normalize_url", + "normalize_query", + "score_result", + # Error utilities + "_detect_error_type", + # Fetch utilities + "fetch_url_content", + "fetch_llms_txt", +] diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/async_http.py b/.agents/skills/do-web-doc-resolver/scripts/utils/async_http.py new file mode 100644 index 00000000..07cfdb8b --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/async_http.py @@ -0,0 +1,238 @@ +""" +Async HTTP utilities for the Web Doc Resolver. +Uses httpx.AsyncClient for non-blocking HTTP requests. +""" + +import asyncio +import ipaddress +import logging +import socket +import time +from functools import lru_cache +from urllib.parse import urljoin, urlparse + +import httpx + +from scripts.constants import ( + BLOCKED_HOSTNAMES, + BLOCKED_NETWORKS, + BLOCKED_SCHEMES, + DNS_CACHE_TTL, + USER_AGENT, +) +from scripts.models import ValidationResult + +logger = logging.getLogger(__name__) + +_global_client: httpx.AsyncClient | None = None +_client_lock = asyncio.Lock() + + +async def get_async_client() -> httpx.AsyncClient: + """Get or create the global async HTTP client.""" + global _global_client + if _global_client is None or _global_client.is_closed: + _global_client = httpx.AsyncClient( + http2=True, + timeout=httpx.Timeout(30.0), + follow_redirects=False, + headers={ + "User-Agent": USER_AGENT, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }, + limits=httpx.Limits( + max_connections=100, + max_keepalive_connections=20, + keepalive_expiry=30, + ), + verify=True, + ) + return _global_client + + +async def close_async_client() -> None: + """Close the global async HTTP client.""" + global _global_client + if _global_client is not None and not _global_client.is_closed: + await _global_client.aclose() + _global_client = None + + +@lru_cache(maxsize=1024) +def _getaddrinfo_bucketed(host: str, port: int | str | None, bucket: int) -> list[tuple]: + """Internal helper for cached getaddrinfo using time-bucketing.""" + return socket.getaddrinfo(host, port) + + +def _getaddrinfo_cached(host: str, port: int | str | None = None) -> list[tuple]: + """Cached version of socket.getaddrinfo with TTL.""" + bucket = int(time.time() // DNS_CACHE_TTL) + return _getaddrinfo_bucketed(host, port, bucket) + + +def _normalize_host(hostname: str) -> str: + """Normalise encoded IP representations to dotted-decimal.""" + h = hostname.strip().lower() + if h.isdigit(): + try: + return str(ipaddress.IPv4Address(int(h))) + except (ValueError, OverflowError): + pass + if h.startswith("0x"): + try: + return str(ipaddress.IPv4Address(int(h, 16))) + except (ValueError, OverflowError): + pass + if h.startswith("::ffff:"): + return h[7:] + return h + + +def is_safe_url(url: str) -> bool: + """Check if a URL is safe to fetch (no SSRF).""" + try: + parsed = urlparse(url) + if parsed.scheme.lower() in BLOCKED_SCHEMES: + return False + if parsed.scheme not in ("http", "https"): + return False + hostname = parsed.hostname + if not hostname: + return False + normalized = _normalize_host(hostname) + if normalized in ( + "localhost", + "localhost.localdomain", + "127.0.0.1", + "::1", + "0.0.0.0", + ): + return False + hostname_lower = hostname.lower().strip(".") + if hostname_lower in BLOCKED_HOSTNAMES: + logger.warning("SSRF blocked (hostname blocklist): %s", url) + return False + if any(hostname_lower.endswith("." + blocked) for blocked in BLOCKED_HOSTNAMES): + logger.warning("SSRF blocked (hostname suffix): %s", url) + return False + try: + ip = ipaddress.ip_address(normalized) + if any(ip in network for network in BLOCKED_NETWORKS): + return False + except ValueError: + try: + infos = _getaddrinfo_cached(hostname, None) + for _family, _socktype, _proto, _canonname, sockaddr in infos: + ip = ipaddress.ip_address(sockaddr[0]) + if any(ip in network for network in BLOCKED_NETWORKS): + return False + except Exception: + logger.debug("DNS resolution failed for SSRF check: %s", hostname, exc_info=True) + if normalized.endswith(".local") or normalized.endswith(".internal"): + return False + return True + except Exception: + logger.debug("URL safety check failed: %s", url, exc_info=True) + return False + + +async def async_safe_request( + method: str, + url: str, + *, + max_redirects: int = 5, + **kwargs, +) -> httpx.Response: + """Perform an async HTTP request while validating each redirect hop for SSRF.""" + client = await get_async_client() + current_url = url + history: list[httpx.Response] = [] + + for _ in range(max_redirects + 1): + if not is_safe_url(current_url): + raise httpx.RequestError(f"SSRF blocked: {current_url}") + + response = await client.request(method, current_url, **kwargs) + + if response.is_redirect: + history.append(response) + location = response.headers.get("location") + if not location: + break + next_url = location + if not urlparse(next_url).netloc: + next_url = urljoin(current_url, next_url) + current_url = next_url + continue + + response.history = history + return response + + raise httpx.TooManyRedirects(f"Exceeded {max_redirects} redirects") + + +async def async_validate_url( + url: str, timeout: int = 10, check_ssrf: bool = True +) -> ValidationResult: + """Validate a URL asynchronously.""" + if not url or not url.strip(): + return ValidationResult(is_valid=False, error="Empty URL") + from scripts.utils.urls import is_url + + if not is_url(url): + return ValidationResult(is_valid=False, error="Invalid URL format") + try: + if check_ssrf: + response = await async_safe_request("HEAD", url, timeout=timeout) + else: + client = await get_async_client() + response = await client.head(url, follow_redirects=True, timeout=timeout) + redirect_chain = [str(h.url) for h in response.history] + [str(response.url)] + if response.status_code >= 400: + return ValidationResult( + is_valid=False, + status_code=response.status_code, + error=f"HTTP {response.status_code}", + final_url=str(response.url), + redirect_chain=redirect_chain, + ) + return ValidationResult( + is_valid=True, + status_code=response.status_code, + final_url=str(response.url), + redirect_chain=redirect_chain, + content_type=response.headers.get("content-type", ""), + ) + except Exception as e: + return ValidationResult(is_valid=False, error=str(e)) + + +async def _async_validate_single_link( + link: str, timeout: int, client: httpx.AsyncClient +) -> str | None: + """Validate a single link asynchronously.""" + try: + response = await async_safe_request("HEAD", link, timeout=timeout) + if response.status_code < 400: + return link + except Exception: + logger.debug("Link validation failed: %s", link, exc_info=True) + return None + return None + + +async def async_validate_links(links: list[str], timeout: int = 5) -> list[str]: + """Validate a list of links in parallel, preserving input order.""" + if not links: + return [] + + client = await get_async_client() + tasks = [_async_validate_single_link(link, timeout, client) for link in links] + results = await asyncio.gather(*tasks, return_exceptions=True) + + return [ + link + for link, result in zip(links, results, strict=False) + if result and not isinstance(result, Exception) + ] diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/cache.py b/.agents/skills/do-web-doc-resolver/scripts/utils/cache.py new file mode 100644 index 00000000..15dd1b9d --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/cache.py @@ -0,0 +1,228 @@ +""" +Cache utilities for the Web Doc Resolver. +""" + +import asyncio +import hashlib +import logging +import os +import threading +import time +from collections.abc import Callable +from typing import Any + +from scripts.constants import CACHE_DIR, TIERED_TTL + +logger = logging.getLogger(__name__) + +_cache = None +_cache_lock = threading.RLock() + +# L1 in-memory cache: fast TTL cache sitting in front of disk cache +_l1_cache: dict[str, tuple[Any, float]] = {} +_l1_cache_lock = threading.RLock() +L1_CACHE_MAX_SIZE = 1000 +L1_CACHE_DEFAULT_TTL = 300 # 5 minutes + +# Request coalescing: track in-flight requests to deduplicate concurrent calls +_inflight_requests: dict[str, asyncio.Future] = {} +_inflight_lock = asyncio.Lock() + + +async def coalesce_request(key: str, func: Callable) -> Any: + """Coalesce concurrent requests for the same key. + + If a request is already in-flight for this key, wait for its result. + Otherwise, execute the function and cache the result for other waiters. + """ + async with _inflight_lock: + if key in _inflight_requests: + # Another request is in-flight — wait for it + future = _inflight_requests[key] + else: + # First request — create a future and execute + future = asyncio.get_event_loop().create_future() + _inflight_requests[key] = future + + if future.done(): + # Already completed (edge case: lock acquired after completion) + async with _inflight_lock: + _inflight_requests.pop(key, None) + return future.result() + + # Check if we're the one who should execute + async with _inflight_lock: + is_first = _inflight_requests.get(key) is future + + if is_first: + try: + result = await func() + future.set_result(result) + return result + except Exception as e: + future.set_exception(e) + raise + finally: + async with _inflight_lock: + _inflight_requests.pop(key, None) + else: + # Wait for the first request to complete + return await future + + +def _l1_get(key: str) -> Any | None: + """Get from L1 in-memory cache.""" + with _l1_cache_lock: + entry = _l1_cache.get(key) + if entry is None: + return None + value, expire_time = entry + if time.time() > expire_time: + del _l1_cache[key] + return None + return value + + +def _l1_set(key: str, value: Any, ttl: int = L1_CACHE_DEFAULT_TTL) -> None: + """Set in L1 in-memory cache with TTL.""" + with _l1_cache_lock: + # Evict oldest entries if at capacity + if len(_l1_cache) >= L1_CACHE_MAX_SIZE: + # Remove 10% of oldest entries + evict_count = max(1, L1_CACHE_MAX_SIZE // 10) + to_evict = sorted(_l1_cache.keys(), key=lambda k: _l1_cache[k][1])[:evict_count] + for k in to_evict: + del _l1_cache[k] + _l1_cache[key] = (value, time.time() + ttl) + + +def _l1_clear() -> None: + """Clear L1 in-memory cache.""" + with _l1_cache_lock: + _l1_cache.clear() + + +def _clear_inflight() -> None: + """Clear inflight request tracking (for testing).""" + global _inflight_requests + _inflight_requests = {} + + +def _cache_key(input_str: str, source: str) -> str: + from scripts.utils.urls import is_url, normalize_query, normalize_url + + # Use normalized input for cache key + if is_url(input_str): + normalized = normalize_url(input_str) + else: + normalized = normalize_query(input_str) + + hash_input = f"{source}:{normalized}" + return hashlib.sha256(hash_input.encode()).hexdigest() + + +def _get_cache_proxy(): + import scripts.resolve + + if hasattr(scripts.resolve, "_cache") and scripts.resolve._cache is not None: + return scripts.resolve._cache + return _cache + + +def get_cache(): + try: + import diskcache + + os.makedirs(CACHE_DIR, exist_ok=True) + return diskcache.Cache(CACHE_DIR) + except Exception: + logger.debug("Failed to initialize diskcache", exc_info=True) + return None + + +def _get_cache(): + global _cache + with _cache_lock: + _cache = _get_cache_proxy() + if _cache is None: + _cache = get_cache() + return _cache + + +def get_ttl(provider: str, config: dict | None = None) -> int: + """Get the TTL for a given provider from config or defaults.""" + from scripts.utils import get_config_data + + # Normalize provider name for alias support + provider_key = provider + if provider in ("exa_mcp", "exa"): + provider_key = "exa" + elif provider in ("mistral_browser", "mistral_websearch"): + provider_key = "mistral" + + # Use provided config or load from file + cfg = config if config is not None else get_config_data() + + # Environment variable override takes precedence over file-based config + env_key = f"DO_WDR_CACHE_TTL_{provider_key.upper()}" + if env_key in os.environ: + try: + return int(os.environ[env_key]) + except ValueError: + pass + + if cfg: + # Try to get from nested config.toml style + ttl_cfg = cfg.get("cache", {}).get("ttl", {}) + if provider_key in ttl_cfg: + return int(ttl_cfg[provider_key]) + if "default" in ttl_cfg: + return int(ttl_cfg["default"]) + + return TIERED_TTL.get(provider_key, TIERED_TTL.get("default", 3600)) + + +def _get_from_cache(input_str: str, source: str) -> dict[str, Any] | None: + key = _cache_key(input_str, source) + + # Check L1 in-memory cache first (fast path) + result = _l1_get(key) + if result is not None: + return dict(result) + + # Check disk cache (slow path) + from scripts.utils import _get_cache + + with _cache_lock: + cache = _get_cache() + if not cache: + return None + with _cache_lock: + result = cache.get(key) + if result is None: + return None + + # Promote to L1 cache for faster subsequent access + _l1_set(key, result, ttl=min(get_ttl(source), L1_CACHE_DEFAULT_TTL)) + return dict(result) + + +def _save_to_cache(input_str: str, source: str, result: dict[str, Any], ttl: int | None = None): + from scripts.utils import _get_cache + + if ttl is None: + ttl = get_ttl(source) + + key = _cache_key(input_str, source) + + # Store in L1 in-memory cache + _l1_set(key, result, ttl=min(ttl, L1_CACHE_DEFAULT_TTL)) + + # Store in disk cache + with _cache_lock: + cache = _get_cache() + if not cache: + return + + with _cache_lock: + cache.set(key, result, expire=ttl) diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/content_clean.py b/.agents/skills/do-web-doc-resolver/scripts/utils/content_clean.py new file mode 100644 index 00000000..54babb06 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/content_clean.py @@ -0,0 +1,80 @@ +"""Content cleaning utilities for token-efficient LLM output. + +Priority: + 1. trafilatura — best article extraction, handles most doc/blog pages + 2. readability-lxml — fallback for pages trafilatura returns None on + 3. raw HTML strip — last resort, strips tags with regex +""" + +import logging +import re + +logger = logging.getLogger(__name__) + +_STRIP_RE = re.compile(r"<[^>]+>") +_SPACE_RE = re.compile(r"\s+") + + +def _strip_html_tags(html: str) -> str: + """Minimal fallback: strip all HTML tags.""" + text = _STRIP_RE.sub(" ", html) + return _SPACE_RE.sub(" ", text).strip() + + +def clean_content( + html: str, + url: str = "", + max_chars: int = 32_000, + favor_recall: bool = False, +) -> str: + """Extract main content from HTML, removing boilerplate for LLM efficiency. + + Args: + html: Raw HTML string from the fetch provider. + url: Source URL (improves trafilatura heuristics). + max_chars: Hard limit on returned character count. + favor_recall: If True, use trafilatura include_tables/include_links=True. + + Returns: + Cleaned plain-text or light markdown string, capped at max_chars. + """ + if not html or not html.strip(): + return "" + + try: + import trafilatura + + result = trafilatura.extract( + html, + url=url or None, + include_tables=favor_recall, + include_links=favor_recall, + include_images=False, + favor_precision=not favor_recall, + output_format="txt", + deduplicate=True, + ) + if result and len(result.strip()) > 200: + logger.debug("content_clean: trafilatura succeeded (%d chars)", len(result)) + return result[:max_chars] + except ImportError: + logger.debug("content_clean: trafilatura not installed, trying readability") + except Exception as e: + logger.debug("content_clean: trafilatura failed: %s", e) + + try: + from readability import Document # type: ignore[import] + + doc = Document(html) + summary_html = doc.summary() + text = _strip_html_tags(summary_html) + if text and len(text.strip()) > 200: + logger.debug("content_clean: readability succeeded (%d chars)", len(text)) + return text[:max_chars] + except ImportError: + logger.debug("content_clean: readability-lxml not installed, using fallback") + except Exception as e: + logger.debug("content_clean: readability failed: %s", e) + + logger.debug("content_clean: using raw strip fallback") + return _strip_html_tags(html)[:max_chars] diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/fetch.py b/.agents/skills/do-web-doc-resolver/scripts/utils/fetch.py new file mode 100644 index 00000000..68053f32 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/fetch.py @@ -0,0 +1,111 @@ +""" +Fetch utilities for the Web Doc Resolver. +""" + +import logging +from typing import cast +from urllib.parse import urlparse + +from scripts.constants import CLEAN_CONTENT, DEFAULT_TIMEOUT, MAX_CHARS +from scripts.models import ResolvedResult + +logger = logging.getLogger(__name__) + + +def fetch_url_content( + url: str, timeout: int = DEFAULT_TIMEOUT, max_chars: int = MAX_CHARS +) -> ResolvedResult | None: + from scripts.utils import _safe_request, get_session, validate_url + + validation = validate_url(url, timeout=timeout // 2) + if not validation.is_valid: + return None + try: + session = get_session() + response = _safe_request("GET", url, session=session, timeout=timeout, verify=True) + if response.status_code >= 400: + # Check for bot challenge even on error status codes (e.g., 403 Forbidden) + from scripts.quality import is_bot_challenge + + if is_bot_challenge(response.text): + raise ValueError(f"Bot challenge detected (HTTP {response.status_code})") + return None + raw_html = response.text + + from scripts.quality import is_bot_challenge + + if is_bot_challenge(raw_html): + raise ValueError("Bot challenge detected") + + is_html = "text/html" in response.headers.get("Content-Type", "") + + if is_html and CLEAN_CONTENT: + from scripts.utils.content_clean import clean_content + + content = clean_content(raw_html, url=url, max_chars=max_chars) + elif is_html: + from scripts.utils import extract_text_from_html + + content = extract_text_from_html(raw_html, url)[:max_chars] + else: + content = raw_html[:max_chars] + + return ResolvedResult( + source="direct_fetch", + content=content, + url=validation.final_url or url, + metadata={ + "status_code": response.status_code, + "cleaned": is_html and CLEAN_CONTENT, + "raw_length": len(raw_html), + }, + ) + except Exception as e: + # Surface bot challenges to the cascade + from scripts.models import ErrorType + from scripts.utils import _detect_error_type + + if _detect_error_type(e) == ErrorType.BOT_CHALLENGE: + raise + + logger.debug("Direct fetch failed: %s", url, exc_info=True) + return None + + +def fetch_llms_txt(url: str) -> str | None: + from scripts.utils import ( + _get_from_cache, + _safe_request, + _save_to_cache, + get_session, + get_ttl, + is_safe_url, + ) + + try: + if not is_safe_url(url): + return None + parsed = urlparse(url) + base_url = f"{parsed.scheme}://{parsed.netloc}" + llms_url = f"{base_url}/llms.txt" + cached = _get_from_cache(base_url, "llms_txt") + if cached is not None: + if cached.get("found"): + return str(cached.get("content", "")) + return None + session = get_session() + response = _safe_request("GET", llms_url, session=session, timeout=10) + if response.status_code == 200: + content_type = response.headers.get("Content-Type", "") + if "text" in content_type or "markdown" in content_type: + _save_to_cache( + base_url, + "llms_txt", + {"found": True, "content": response.text}, + ttl=get_ttl("llms_txt"), + ) + return cast(str, response.text) + _save_to_cache(base_url, "llms_txt", {"found": False}, ttl=get_ttl("llms_txt")) + except Exception: + logger.debug("llms.txt fetch failed: %s", url, exc_info=True) + return None diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/html.py b/.agents/skills/do-web-doc-resolver/scripts/utils/html.py new file mode 100644 index 00000000..db5f043f --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/html.py @@ -0,0 +1,146 @@ +""" +HTML utilities for the Web Doc Resolver. +""" + +import re +from html.parser import HTMLParser + +_RE_SPACES = re.compile(r"[ \t]+") +_RE_NEWLINES = re.compile(r"\n{3,}") + + +class EnhancedHTMLParser(HTMLParser): + _block_tags = { + "p", + "div", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "li", + "tr", + "article", + "section", + "header", + "footer", + "nav", + "aside", + "blockquote", + "ul", + "ol", + "table", + "hr", + } + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.result: list[str] = [] + self._skip_depth = 0 + self._in_pre = 0 + + def handle_starttag(self, tag, attrs): + tag_lower = tag.lower() + if tag_lower in ("script", "style"): + self._skip_depth += 1 + elif self._skip_depth == 0: + if tag_lower == "pre": + self._in_pre += 1 + self.result.append("\n\n```\n") + elif tag_lower == "br": + self.result.append("\n") + elif tag_lower == "hr": + self.result.append("\n\n---\n\n") + elif tag_lower in ( + "p", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "blockquote", + "ul", + "ol", + "table", + ): + self.result.append("\n\n") + elif tag_lower in self._block_tags: + self.result.append("\n") + + if tag_lower == "code": + self.result.append("`") + + def handle_endtag(self, tag): + tag_lower = tag.lower() + if tag_lower in ("script", "style") and self._skip_depth > 0: + self._skip_depth -= 1 + elif self._skip_depth == 0: + if tag_lower == "pre": + self._in_pre = max(0, self._in_pre - 1) + self.result.append("\n```\n\n") + elif tag_lower == "code": + self.result.append("`") + elif tag_lower in ( + "p", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "blockquote", + "ul", + "ol", + "table", + ): + self.result.append("\n\n") + elif tag_lower in self._block_tags: + self.result.append("\n") + + def handle_data(self, data): + if self._skip_depth == 0: + if self._in_pre > 0: + self.result.append(data) + else: + # Fast path: only sub if multiple spaces or tabs present + if "\t" in data or " " in data: + normalized = _RE_SPACES.sub(" ", data) + else: + normalized = data + + if normalized: + # Prevent double spaces across chunks + if normalized.startswith(" ") and self.result and self.result[-1].endswith(" "): + normalized = normalized[1:] + if normalized: + self.result.append(normalized) + + +def extract_text_from_html(html: str, base_url: str = "") -> str: + stripper = EnhancedHTMLParser() + stripper.feed(html) + text = "".join(stripper.result) + # Normalize word joiner and other problematic characters + if "\u2060" in text: + text = text.replace("\u2060", "") + # Note: _RE_SPACES is handled per-chunk in handle_data to preserve code blocks. + if "\n\n\n" in text: + text = _RE_NEWLINES.sub("\n\n", text) + return text.strip() + + +def compact_content(content: str, max_chars: int) -> str: + lines = content.splitlines() + unique_lines = set() + compacted = [] + for line in lines: + trimmed = line.strip() + if not trimmed: + compacted.append("") + continue + if trimmed not in unique_lines: + compacted.append(trimmed) + unique_lines.add(trimmed) + return "\n".join(compacted)[:max_chars] diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/http.py b/.agents/skills/do-web-doc-resolver/scripts/utils/http.py new file mode 100644 index 00000000..11273ffd --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/http.py @@ -0,0 +1,268 @@ +""" +HTTP utilities for the Web Doc Resolver. +Uses httpx.Client for sync operations. +""" + +import asyncio +import ipaddress +import logging +import socket +import threading +import time +from functools import lru_cache +from urllib.parse import urljoin, urlparse + +import httpx + +from scripts.constants import ( + BLOCKED_HOSTNAMES, + BLOCKED_NETWORKS, + BLOCKED_SCHEMES, + DNS_CACHE_TTL, + USER_AGENT, +) +from scripts.models import ValidationResult + +logger = logging.getLogger(__name__) + +_global_client: httpx.Client | None = None +_client_lock = threading.Lock() + + +def create_client_with_retry() -> httpx.Client: + """Create an httpx.Client with retry configuration.""" + return httpx.Client( + http2=True, + timeout=httpx.Timeout(30.0), + follow_redirects=False, + headers={ + "User-Agent": USER_AGENT, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }, + limits=httpx.Limits( + max_connections=100, + max_keepalive_connections=20, + keepalive_expiry=30, + ), + transport=httpx.HTTPTransport(retries=3), + verify=True, + ) + + +def get_session() -> httpx.Client: + """Get or create the global sync HTTP client (backward compatible name).""" + global _global_client + with _client_lock: + if _global_client is None or _global_client.is_closed: + _global_client = create_client_with_retry() + return _global_client + + +def close_session() -> None: + """Close the global sync HTTP client.""" + global _global_client + with _client_lock: + if _global_client is not None and not _global_client.is_closed: + _global_client.close() + _global_client = None + + +# Keep create_session_with_retry as alias for backward compatibility +create_session_with_retry = create_client_with_retry + + +@lru_cache(maxsize=1024) +def _getaddrinfo_bucketed(host: str, port: int | str | None, bucket: int) -> list[tuple]: + """Internal helper for cached getaddrinfo using time-bucketing.""" + return socket.getaddrinfo(host, port) + + +def _getaddrinfo_cached(host: str, port: int | str | None = None) -> list[tuple]: + """Cached version of socket.getaddrinfo with TTL to balance performance and security.""" + bucket = int(time.time() // DNS_CACHE_TTL) + return _getaddrinfo_bucketed(host, port, bucket) + + +def _normalize_host(hostname: str) -> str: + """Normalise encoded IP representations to dotted-decimal. + + Handles decimal integer notation, hex notation, and IPv4-mapped IPv6. + """ + h = hostname.strip().lower() + if h.isdigit(): + try: + return str(ipaddress.IPv4Address(int(h))) + except (ValueError, OverflowError): + pass + if h.startswith("0x"): + try: + return str(ipaddress.IPv4Address(int(h, 16))) + except (ValueError, OverflowError): + pass + if h.startswith("::ffff:"): + return h[7:] + return h + + +def is_safe_url(url: str) -> bool: + """Check if a URL is safe to fetch (no SSRF).""" + try: + parsed = urlparse(url) + if parsed.scheme.lower() in BLOCKED_SCHEMES: + return False + if parsed.scheme not in ("http", "https"): + return False + hostname = parsed.hostname + if not hostname: + return False + normalized = _normalize_host(hostname) + if normalized in ( + "localhost", + "localhost.localdomain", + "127.0.0.1", + "::1", + "0.0.0.0", + ): + return False + hostname_lower = hostname.lower().strip(".") + if hostname_lower in BLOCKED_HOSTNAMES: + logger.warning("SSRF blocked (hostname blocklist): %s", url) + return False + if any(hostname_lower.endswith("." + blocked) for blocked in BLOCKED_HOSTNAMES): + logger.warning("SSRF blocked (hostname suffix): %s", url) + return False + try: + ip = ipaddress.ip_address(normalized) + if any(ip in network for network in BLOCKED_NETWORKS): + return False + except ValueError: + try: + infos = _getaddrinfo_cached(hostname, None) + for _family, _socktype, _proto, _canonname, sockaddr in infos: + ip = ipaddress.ip_address(sockaddr[0]) + if any(ip in network for network in BLOCKED_NETWORKS): + return False + except Exception: + logger.debug("DNS resolution failed for SSRF check: %s", hostname, exc_info=True) + if normalized.endswith(".local") or normalized.endswith(".internal"): + return False + return True + except Exception: + logger.debug("URL safety check failed: %s", url, exc_info=True) + return False + + +def _safe_request( + method: str, + url: str, + client: httpx.Client | None = None, + *, + max_redirects: int = 5, + **kwargs, +) -> httpx.Response: + """Perform an HTTP request while validating each redirect hop for SSRF.""" + current_url = url + history: list[httpx.Response] = [] + kwargs.pop("allow_redirects", None) + active_client = client or get_session() + + for _ in range(max_redirects + 1): + if not is_safe_url(current_url): + raise httpx.RequestError(f"SSRF blocked: {current_url}") + + response = active_client.request(method, current_url, **kwargs) + + if response.is_redirect: + history.append(response) + location = response.headers.get("location") + if not location: + break + next_url = location + if not urlparse(next_url).netloc: + next_url = urljoin(current_url, next_url) + current_url = next_url + continue + + response.history = history + return response + + raise httpx.TooManyRedirects(f"Exceeded {max_redirects} redirects") + + +def validate_url(url: str, timeout: int = 10, check_ssrf: bool = True) -> ValidationResult: + """Validate a URL.""" + if not url or not url.strip(): + return ValidationResult(is_valid=False, error="Empty URL") + from scripts.utils.urls import is_url + + if not is_url(url): + return ValidationResult(is_valid=False, error="Invalid URL format") + try: + client = get_session() + if check_ssrf: + response = _safe_request("HEAD", url, client=client, timeout=timeout) + else: + response = client.head(url, follow_redirects=True, timeout=timeout) + redirect_chain = [str(h.url) for h in response.history] + [str(response.url)] + if response.status_code >= 400: + return ValidationResult( + is_valid=False, + status_code=response.status_code, + error=f"HTTP {response.status_code}", + final_url=str(response.url), + redirect_chain=redirect_chain, + ) + return ValidationResult( + is_valid=True, + status_code=response.status_code, + final_url=str(response.url), + redirect_chain=redirect_chain, + content_type=response.headers.get("content-type", ""), + ) + except Exception as e: + return ValidationResult(is_valid=False, error=str(e)) + + +def _validate_single_link(link: str, timeout: int, client: httpx.Client) -> str | None: + """Validate a single link.""" + try: + response = _safe_request("HEAD", link, client=client, timeout=timeout) + if response.status_code < 400: + return link + except Exception: + logger.debug("Link validation failed: %s", link, exc_info=True) + return None + return None + + +def validate_links(links: list[str], timeout: int = 5) -> list[str]: + """Validate a list of links in parallel, preserving input order.""" + if not links: + return [] + + client = get_session() + + async def _validate_all(): + tasks = [_validate_single_link_async(link, timeout, client) for link in links] + return await asyncio.gather(*tasks) + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + + if loop and loop.is_running(): + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + results = pool.submit(asyncio.run, _validate_all()).result() + else: + results = asyncio.run(_validate_all()) + + return [link for link, valid in zip(links, results, strict=False) if valid] + + +async def _validate_single_link_async(link: str, timeout: int, client: httpx.Client) -> str | None: + """Validate a single link asynchronously.""" + return await asyncio.to_thread(_validate_single_link, link, timeout, client) diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/thread_pool.py b/.agents/skills/do-web-doc-resolver/scripts/utils/thread_pool.py new file mode 100644 index 00000000..d10a55cf --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/thread_pool.py @@ -0,0 +1,34 @@ +""" +Shared ThreadPoolExecutor for the Web Doc Resolver. + +Provides a single thread pool for all async-to-sync conversions, +reducing thread creation overhead and improving performance. +""" + +import concurrent.futures +import threading + +# Shared thread pool for async-to-sync conversions +_shared_pool: concurrent.futures.ThreadPoolExecutor | None = None +_pool_lock = threading.Lock() + + +def get_shared_pool(max_workers: int = 10) -> concurrent.futures.ThreadPoolExecutor: + """Get or create the shared ThreadPoolExecutor.""" + global _shared_pool + with _pool_lock: + if _shared_pool is None or _shared_pool._shutdown: + _shared_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers, + thread_name_prefix="wdr-worker", + ) + return _shared_pool + + +def shutdown_shared_pool() -> None: + """Shutdown the shared ThreadPoolExecutor.""" + global _shared_pool + with _pool_lock: + if _shared_pool is not None: + _shared_pool.shutdown(wait=False) + _shared_pool = None diff --git a/.agents/skills/do-web-doc-resolver/scripts/utils/urls.py b/.agents/skills/do-web-doc-resolver/scripts/utils/urls.py new file mode 100644 index 00000000..61dfeff5 --- /dev/null +++ b/.agents/skills/do-web-doc-resolver/scripts/utils/urls.py @@ -0,0 +1,137 @@ +""" +URL utilities for the Web Doc Resolver. +""" + +import logging +from urllib.parse import parse_qs, urlencode, urlparse + +logger = logging.getLogger(__name__) + +_TRACKING_PARAMS = { + # UTM parameters + "utm_source", + "utm_medium", + "utm_campaign", + "utm_term", + "utm_content", + # Social/tracking parameters + "fbclid", + "gclid", + "gclsrc", + "dclid", + "msclkid", + "twclid", + "li_fat_id", + "mc_cid", + "mc_eid", + # Referral/tracking + "ref", + "ref_src", + "ref_url", + "source", + "via", + # Session/tracking + "session_id", + "sid", + "_ga", + "_gl", + # Misc tracking + "hsa_cam", + "hsa_grp", + "hsa_mt", + "hsa_src", + "hsa_ad", + "hsa_acc", + "hsa_net", + "hsa_kw", + "hsa_tgt", + "hsa_ver", +} + + +def is_url(input_str: str) -> bool: + if not input_str: + return False + trimmed = input_str.strip() + if not trimmed: + return False + # Fast path: must start with http (case-insensitive check without full lower() allocation) + prefix = trimmed[:8].lower() + if not prefix.startswith(("http://", "https://")): + return False + try: + result = urlparse(trimmed) + return all([result.scheme in {"http", "https"}, result.netloc]) + except Exception: + logger.debug("URL parsing failed: %s", trimmed, exc_info=True) + return False + + +def normalize_url(url: str) -> str: + """Normalize URL by stripping tracking params, anchors, and common aliases.""" + try: + parsed = urlparse(url) + # Strip all known tracking params + if parsed.query: + params = parse_qs(parsed.query) + filtered_params = { + k: v + for k, v in params.items() + if k.lower() not in _TRACKING_PARAMS and not k.startswith("utm_") + } + query = urlencode(filtered_params, doseq=True) + else: + query = "" + + # Normalize fragment: strip if empty or just a section ref + fragment = "" if not parsed.fragment else parsed.fragment + + # Normalize trailing slash (keep for root, strip for paths) + path = parsed.path + if path and path != "/" and path.endswith("/"): + path = path.rstrip("/") + + # Normalize netloc: lowercase, strip default ports + netloc = parsed.netloc.lower() + if netloc.endswith(":80") and parsed.scheme == "http": + netloc = netloc[:-3] + elif netloc.endswith(":443") and parsed.scheme == "https": + netloc = netloc[:-4] + + # Reconstruct + normalized = parsed._replace( + scheme=parsed.scheme.lower(), netloc=netloc, path=path, query=query, fragment=fragment + ).geturl() + return normalized.strip() + except Exception: + logger.debug("URL normalization failed: %s", url, exc_info=True) + return url.lower().strip() + + +def normalize_query(query: str) -> str: + """Normalize search query.""" + # Lowercase, trim whitespace, and collapse multiple spaces + # Using split() and join() is significantly faster than re.sub for collapsing whitespace + return " ".join(query.lower().split()) + + +def score_result(url: str | None, content: str) -> float: + score = 0.5 + if url: + try: + domain = urlparse(url).netloc.lower() + if any(domain.endswith(tld) for tld in [".edu", ".gov", ".org", ".rs", ".io"]): + score += 0.2 + if any( + site in domain + for site in ["github.com", "stackoverflow.com", "docs.rs", "mozilla.org"] + ): + score += 0.2 + except Exception: + logger.debug("URL domain scoring failed", exc_info=True) + word_count = len(content.split()) + if word_count > 500: + score += 0.1 + elif word_count < 50: + score -= 0.2 + return max(0.0, min(1.0, score)) diff --git a/.agents/skills/do-web-doc-resolver/tests/test_providers.py b/.agents/skills/do-web-doc-resolver/tests/test_providers.py index 12468626..65dd40aa 100644 --- a/.agents/skills/do-web-doc-resolver/tests/test_providers.py +++ b/.agents/skills/do-web-doc-resolver/tests/test_providers.py @@ -8,13 +8,11 @@ from unittest.mock import patch, MagicMock import time +from scripts.constants import DEFAULT_TIMEOUT, MAX_CHARS, MIN_CHARS from scripts.providers_impl import ( is_rate_limited, set_rate_limit, _rate_limits, - MAX_CHARS, - MIN_CHARS, - DEFAULT_TIMEOUT, ) @@ -129,18 +127,24 @@ def test_rate_limited_returns_none(self, mock_rate_limited): # This test demonstrates the rate limit check behavior assert is_rate_limited("jina") is False # Not rate limited by default - @patch("scripts.providers_impl._get_from_cache") - def test_cache_hit_returns_cached(self, mock_cache): - """Cached result should be returned immediately.""" + @patch("scripts.providers.jina._get_from_cache") + @patch("scripts.providers.jina.get_session") + def test_cache_hit_returns_cached(self, mock_session, mock_cache): + """A cached result is returned immediately without any HTTP call.""" from scripts.models import ResolvedResult + from scripts.providers.jina import resolve_with_jina mock_cache.return_value = { "source": "jina", "content": "cached content", "url": "https://example.com", } - # Would need full mock of get_session for actual test - # This demonstrates the cache check pattern + mock_session.return_value.get.side_effect = AssertionError("should not hit network") + + result = resolve_with_jina("https://example.com") + assert isinstance(result, ResolvedResult) + assert result.content == "cached content" + assert result.source == "jina" class TestResolveWithExaMcp: diff --git a/.agents/skills/do-web-doc-resolver/tests/test_resolve.py b/.agents/skills/do-web-doc-resolver/tests/test_resolve.py index 84888150..4ce3ae29 100644 --- a/.agents/skills/do-web-doc-resolver/tests/test_resolve.py +++ b/.agents/skills/do-web-doc-resolver/tests/test_resolve.py @@ -20,10 +20,10 @@ def test_detects_http_url(self): assert is_url("http://example.com") is True assert is_url("http://localhost:8000") is True - def test_detects_ftp_url(self): - """Should detect ftp URLs.""" - assert is_url("ftp://ftp.example.com") is True - assert is_url("ftps://secure.example.com") is True + def test_rejects_ftp_url(self): + """FTP URLs are rejected — only http/https are resolvable (SSRF guard).""" + assert is_url("ftp://ftp.example.com") is False + assert is_url("ftps://secure.example.com") is False def test_detects_url_with_path(self): """Should detect URLs with paths.""" diff --git a/agents-docs/DEPLOYMENT.md b/agents-docs/DEPLOYMENT.md index 734620ad..81cf216f 100644 --- a/agents-docs/DEPLOYMENT.md +++ b/agents-docs/DEPLOYMENT.md @@ -35,7 +35,10 @@ vercel build --prod # Verify production build Set in Vercel dashboard: -- `NEXT_PUBLIC_RESOLVER_URL`: Backend resolver URL (default: ) +- `RECORDS_MAX_SIZE`: Max in-memory records kept (default `100`) +- `RECORDS_TTL_DAYS`: Records older than this are dropped (default `30`) +- `WEB_RESOLVER_MAX_CHARS`: Max characters in resolved content (default `8000`) +- `EXA_API_KEY` / `TAVILY_API_KEY` / `SERPER_API_KEY` / `FIRECRAWL_API_KEY` / `MISTRAL_API_KEY`: optional provider keys (free providers work without keys) ### 2. Docker (Backend) diff --git a/cli/src/config/defaults.rs b/cli/src/config/defaults.rs index 592b6fab..96bb6a8c 100644 --- a/cli/src/config/defaults.rs +++ b/cli/src/config/defaults.rs @@ -112,10 +112,6 @@ pub(crate) fn default_prewarm_max_concurrency() -> usize { 4 } -pub(crate) fn default_max_links() -> usize { - 10 -} - pub(crate) fn default_ttl_firecrawl() -> u64 { 21600 } diff --git a/cli/src/config/mod.rs b/cli/src/config/mod.rs index 580173af..4cd62280 100644 --- a/cli/src/config/mod.rs +++ b/cli/src/config/mod.rs @@ -14,14 +14,11 @@ pub use defaults::RoutingProfileConfig; pub use defaults::routing_profile_defaults; #[derive(Error, Debug)] -#[allow(dead_code)] pub enum ConfigError { #[error("Failed to read config file: {0}")] IoError(#[from] std::io::Error), #[error("Failed to parse config file: {0}")] ParseError(#[from] toml::de::Error), - #[error("Invalid configuration: {0}")] - InvalidConfig(String), } #[derive(Debug, Clone, Deserialize)] @@ -64,8 +61,6 @@ pub struct Config { pub circuit_breaker_threshold: u32, #[serde(default = "default_circuit_breaker_cooldown")] pub circuit_breaker_cooldown_secs: u64, - #[serde(default = "default_max_links")] - pub max_links: usize, #[serde(default)] pub providers: HashMap, } @@ -199,7 +194,6 @@ impl Default for Config { error_cache_ttl_secs: default_error_cache_ttl(), circuit_breaker_threshold: default_circuit_breaker_threshold(), circuit_breaker_cooldown_secs: default_circuit_breaker_cooldown(), - max_links: default_max_links(), providers: HashMap::new(), } } @@ -255,7 +249,6 @@ impl Config { other.circuit_breaker_cooldown_secs, default_circuit_breaker_cooldown(), ); - merge_value(&mut self.max_links, other.max_links, default_max_links()); merge_bool( &mut self.semantic_cache.enabled, other.semantic_cache.enabled, diff --git a/cli/src/config/parsing.rs b/cli/src/config/parsing.rs index 1930062e..f649dc72 100644 --- a/cli/src/config/parsing.rs +++ b/cli/src/config/parsing.rs @@ -4,14 +4,26 @@ use super::Config; pub fn apply_env_overrides(config: &mut Config) { if let Ok(config_path) = env::var("DO_WDR_CONFIG") { - if let Ok(file_config) = Config::from_file(&config_path) { - config.merge(file_config); + match Config::from_file(&config_path) { + Ok(file_config) => config.merge(file_config), + Err(e) => eprintln!( + "Warning: ignoring invalid config file {}: {}", + config_path, e + ), } } else { for path in ["./config.toml", "./do-wdr.toml", "./do-wdr.conf"] { - if let Ok(file_config) = Config::from_file(path) { - config.merge(file_config); - break; + // Only warn when a file exists but fails to parse; a missing optional + // config file is the normal case and must stay silent. + if !std::path::Path::new(path).is_file() { + continue; + } + match Config::from_file(path) { + Ok(file_config) => { + config.merge(file_config); + break; + } + Err(e) => eprintln!("Warning: ignoring invalid config file {}: {}", path, e), } } } diff --git a/cli/src/main.rs b/cli/src/main.rs index 1a60b189..808c6325 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -16,13 +16,22 @@ use do_wdr_lib::{ types::ProviderType, }; -/// Initialize logging based on verbosity level -fn init_logging(verbose: u8) { +/// Initialize logging based on verbosity level and config log level +fn init_logging(verbose: u8, log_level: &str) { let filter = match verbose { - 0 => EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("do_wdr=info,do_wdr_lib=info")), 1 => EnvFilter::new("do_wdr=debug,do_wdr_lib=debug"), - _ => EnvFilter::new("do_wdr=trace,do_wdr_lib=trace"), + _ if verbose >= 2 => EnvFilter::new("do_wdr=trace,do_wdr_lib=trace"), + _ => { + let lvl = log_level.trim().to_lowercase(); + match lvl.as_str() { + "info" => EnvFilter::new("do_wdr=info,do_wdr_lib=info"), + "trace" | "debug" | "warn" | "error" | "off" => { + EnvFilter::new(format!("do_wdr={lvl},do_wdr_lib={lvl}")) + } + _ => EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("do_wdr=info,do_wdr_lib=info")), + } + } }; fmt() @@ -185,7 +194,7 @@ fn main() -> ExitCode { let cli = Cli::parse_args(); // Initialize logging - init_logging(cli.verbose); + init_logging(cli.verbose, &Config::load().log_level); // Run the appropriate command let result = match cli.command { diff --git a/cli/src/metrics.rs b/cli/src/metrics.rs index bceeaf40..14c8b217 100644 --- a/cli/src/metrics.rs +++ b/cli/src/metrics.rs @@ -101,3 +101,116 @@ impl ResolveMetrics { self.total_latency_ms += latency_ms; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new_is_empty() { + let m = ResolveMetrics::new(); + assert_eq!(m.total_latency_ms, 0); + assert!(m.provider_metrics.is_empty()); + assert!(!m.cache_hit); + assert!(!m.quality_gate_passed); + assert!(m.quality_gate_score.is_none()); + } + + #[test] + fn test_record_cache_hit() { + let mut m = ResolveMetrics::new(); + m.record_cache_hit("semantic"); + assert!(m.cache_hit); + assert!(!m.synthesis_cache_hit); + + let mut m2 = ResolveMetrics::new(); + m2.record_cache_hit("synthesis"); + assert!(m2.cache_hit); + assert!(m2.synthesis_cache_hit); + } + + #[test] + fn test_record_gate_sets_score() { + let mut m = ResolveMetrics::new(); + m.record_gate(0.88); + assert!(m.quality_gate_passed); + assert_eq!(m.quality_gate_score, Some(0.88)); + } + + #[test] + fn test_record_semantic_cache_hit_threshold() { + // Scores below 0.5 must not report as passing the quality gate. + let mut low = ResolveMetrics::new(); + low.record_semantic_cache_hit(42, 0.3); + assert!(low.cache_hit); + assert_eq!(low.total_latency_ms, 42); + assert!(!low.quality_gate_passed); + assert!(low.quality_gate_score.is_none()); + + let mut high = ResolveMetrics::new(); + high.record_semantic_cache_hit(7, 0.9); + assert!(high.quality_gate_passed); + assert_eq!(high.quality_gate_score, Some(0.9)); + } + + #[test] + fn test_record_provider_success_marks_paid_usage() { + let mut m = ResolveMetrics::new(); + m.record_provider(ProviderType::ExaMcp, 10, true); + assert!(!m.paid_usage); + assert_eq!(m.total_latency_ms, 10); + assert_eq!(m.provider_metrics.len(), 1); + assert_eq!(m.provider_metrics[0].provider, ProviderType::ExaMcp); + assert!(m.provider_metrics[0].accepted); + assert!(!m.provider_metrics[0].paid); + assert_eq!(m.provider_metrics[0].attempt_index, 0); + + let mut m2 = ResolveMetrics::new(); + m2.record_provider(ProviderType::Exa, 25, true); + assert!(m2.paid_usage); + assert!(m2.provider_metrics[0].paid); + } + + #[test] + fn test_record_provider_detailed_preserves_fields() { + let mut m = ResolveMetrics::new(); + m.record_provider_detailed( + ProviderType::Serper, + 99, + false, + 3, + Some(0.4), + false, + Some("thin_content".to_string()), + Some("timeout".to_string()), + true, + true, + ); + assert_eq!(m.provider_metrics.len(), 1); + let pm = &m.provider_metrics[0]; + assert_eq!(pm.latency_ms, 99); + assert!(!pm.success); + assert!(pm.paid); + assert_eq!(pm.attempt_index, 3); + assert_eq!(pm.quality_score, Some(0.4)); + assert!(!pm.accepted); + assert_eq!(pm.skip_reason.as_deref(), Some("thin_content")); + assert_eq!(pm.stop_reason.as_deref(), Some("timeout")); + assert!(pm.negative_cache_hit); + assert!(pm.circuit_open); + assert_eq!(m.total_latency_ms, 99); + } + + #[test] + fn test_metrics_round_trip_serialization() { + let mut m = ResolveMetrics::new(); + m.record_provider(ProviderType::Jina, 5, true); + m.record_gate(0.9); + let json = serde_json::to_string(&m).unwrap(); + let back: ResolveMetrics = serde_json::from_str(&json).unwrap(); + assert_eq!(back.provider_metrics.len(), 1); + assert_eq!(back.provider_metrics[0].provider, ProviderType::Jina); + assert!(back.quality_gate_passed); + assert_eq!(back.quality_gate_score, Some(0.9)); + } +} diff --git a/cli/src/output.rs b/cli/src/output.rs index 917b628c..6445af84 100644 --- a/cli/src/output.rs +++ b/cli/src/output.rs @@ -123,6 +123,33 @@ impl ConfigOutput { " semantic_cache.threshold: {}", config.semantic_cache.threshold ); + if let Some(q) = config.routing.min_free_quality_to_skip_paid { + println!(" routing.min_free_quality_to_skip_paid: {}", q); + } + println!( + " routing.prewarm.enabled: {}", + config.routing.prewarm.enabled + ); + println!( + " routing.prewarm.top_n_domains: {}", + config.routing.prewarm.top_n_domains + ); + println!( + " routing.prewarm.max_concurrency: {}", + config.routing.prewarm.max_concurrency + ); + let ttl = &config.cache.ttl; + println!(" cache.ttl: {:#?}", ttl); + let mut providers: Vec<_> = config.providers.iter().collect(); + providers.sort_by(|a, b| a.0.cmp(b.0)); + for (name, provider) in providers { + if let Some(rl) = &provider.rate_limit { + println!( + " providers.{}.rate_limit: {} rps, burst {}", + name, rl.requests_per_second, rl.burst + ); + } + } } } diff --git a/cli/src/providers/mistral_browser.rs b/cli/src/providers/mistral_browser.rs index e6261c86..a74c622f 100644 --- a/cli/src/providers/mistral_browser.rs +++ b/cli/src/providers/mistral_browser.rs @@ -148,13 +148,17 @@ impl crate::providers::UrlProvider for MistralBrowserProvider { .join("\n") } else { // Cleanup on failure - let _ = self.delete_agent(&agent_id, api_key).await; + if let Err(e) = self.delete_agent(&agent_id, api_key).await { + tracing::warn!("Failed to delete Mistral agent {}: {}", agent_id, e); + } let error_text = conv_response.text().await.unwrap_or_default(); return Err(detect_error_type(&error_text)); }; // Step 3: Cleanup - delete the agent - let _ = self.delete_agent(&agent_id, api_key).await; + if let Err(e) = self.delete_agent(&agent_id, api_key).await { + tracing::warn!("Failed to delete Mistral agent {}: {}", agent_id, e); + } Ok(ResolvedResult::new( url, diff --git a/cli/src/providers/shared_client.rs b/cli/src/providers/shared_client.rs index 60f4708f..03e7af0e 100644 --- a/cli/src/providers/shared_client.rs +++ b/cli/src/providers/shared_client.rs @@ -22,7 +22,7 @@ pub static SHARED_CLIENT: Lazy = Lazy::new(|| { .tcp_keepalive(Duration::from_secs(60)) .user_agent("WDR/1.0 (LLM documentation resolver)") .build() - .expect("Failed to create shared HTTP client") + .unwrap_or_default() }); /// Get a reference to the shared HTTP client. diff --git a/cli/src/resolver/mod.rs b/cli/src/resolver/mod.rs index 2e894abd..e2c572c6 100644 --- a/cli/src/resolver/mod.rs +++ b/cli/src/resolver/mod.rs @@ -315,25 +315,6 @@ impl Resolver { result } - /// Resolve with custom provider order - #[allow(dead_code)] - pub async fn resolve_with_order( - &self, - input: &str, - providers: &[ProviderType], - ) -> Result { - for provider in providers { - let result = self.resolve_direct(input, *provider).await; - if let Ok(res) = result { - if res.is_valid(self.config.min_chars) { - return Ok(res); - } - } - } - - Err(ResolverError::Provider("No provider succeeded".to_string())) - } - /// Access to routing memory pub fn routing_memory(&self) -> Arc> { self.routing_memory.clone() diff --git a/cli/src/resolver/query/mod.rs b/cli/src/resolver/query/mod.rs index be299936..8946229e 100644 --- a/cli/src/resolver/query/mod.rs +++ b/cli/src/resolver/query/mod.rs @@ -353,8 +353,9 @@ impl QueryCascade { .as_ref() .is_none_or(|b| quality.score as f64 > b.score); if dominated { - best_free_result = Some(first.clone()); - best_free_result.as_mut().unwrap().score = quality.score as f64; + let mut candidate = first.clone(); + candidate.score = quality.score as f64; + best_free_result = Some(candidate); } let threshold = profile_defaults.min_free_quality_to_skip_paid; diff --git a/cli/src/resolver/url.rs b/cli/src/resolver/url.rs index 7ed9931c..5c9b8e06 100644 --- a/cli/src/resolver/url.rs +++ b/cli/src/resolver/url.rs @@ -362,11 +362,14 @@ impl UrlCascade { if provider.is_paid { return Ok(res); } else { - if best_free_result.is_none() - || (quality.score as f64) > best_free_result.as_ref().unwrap().score - { - best_free_result = Some(res.clone()); - best_free_result.as_mut().unwrap().score = quality.score as f64; + let better_than_free = match best_free_result.as_ref() { + Some(best) => (quality.score as f64) > best.score, + None => true, + }; + if better_than_free { + let mut candidate = res.clone(); + candidate.score = quality.score as f64; + best_free_result = Some(candidate); } let threshold = config diff --git a/plans/03-performance-optimization.md b/plans/03-performance-optimization.md index dbce6692..2d2e3084 100644 --- a/plans/03-performance-optimization.md +++ b/plans/03-performance-optimization.md @@ -7,41 +7,51 @@ medium effort (Phase 2), high effort (Phase 3). ## Status -Several quick wins are partially or fully addressed by merged PRs. +> Status refreshed 2026-08-11; prior file said "~1/10 done" but most items are +> now shipped or partially shipped. See CHANGELOG v0.3.9 and the code +> references below. ## What's Done -- **Opt 1: Reuse ThreadPoolExecutor** (Phase 1): Partially done — shared - executor pattern not yet implemented, but the `_get_executor` approach is - straightforward when needed. -- **Opt 2: Eliminate busy-polling** (Phase 1): Not done — `timeout=0.01` still - in `scripts/resolve.py:239, 384`. -- **Opt 3: HTTP/2 + keep-alive** (Phase 1): Partially done. Python - `requests.Session()` via `get_session()` in `utils.py` reuses connections. - Rust `reqwest::Client` is shared across providers in some cases. No explicit - `HTTPAdapter` pool size configuration. -- **Opt 4: L1 in-memory cache** (Phase 1): Not done. Cache remains two-tier - (semantic cache + disk). No `TTLCache` layer. +- **Opt 1: Reuse ThreadPoolExecutor** (Phase 1): ✅ Shipped — `scripts/utils/thread_pool.py` + (`get_shared_pool()`), consumed by duckduckgo.py:34, exa.py:92, firecrawl.py:43, + mistral.py:70/124, tavily.py:35 via `loop.run_in_executor`. +- **Opt 2: Eliminate busy-polling** (Phase 1): ⚠️ Verify — `timeout=0.01` no longer + present in `scripts/resolve.py` (file still exists). Re-check for remaining + tight-poll loops before closing. +- **Opt 3: HTTP/2 + keep-alive** (Phase 1): ✅ Shipped — shared `get_session()` + (sync, `utils/http.py`) used by exa.py:141, jina.py:74, serper.py:96; async + `get_async_client()` (`utils/async_http.py`). Rust `shared_client.rs` shares a + single `reqwest::Client`. +- **Opt 4: L1 in-memory cache** (Phase 1): ⚠️ Partial — `functools.lru_cache` + used for DNS bucketing (`utils/http.py:75`, `utils/async_http.py:62`); no + general content `TTLCache` layer. Two-tier cache (semantic + disk) remains. - **Opt 5: Content compaction optimization** (Phase 1): ✅ PR #325 merged (`optimize compact_content`). -- **Opt 6: Early quality exit** (Phase 1): Not done. `scripts/quality.py` has - no early-exit optimization. -- **Opt 7: Shared reqwest Client** (Phase 2): Not done. Providers still create - individual clients. -- **Opt 8: Async-aware locks** (Phase 2): Not done. `std::sync::Mutex` still - used. -- **Opt 9: True parallel provider launch** (Phase 3): Not done. Python still - uses `ThreadPoolExecutor` with sequential launch. -- **Opt 10: Request coalescing** (Phase 3): Not done. +- **Opt 6: Early quality exit** (Phase 1): ⚠️ Verify — check `scripts/quality.py` + for early-exit in the recent refactor before closing. +- **Opt 7: Shared reqwest Client** (Phase 2): ✅ Shipped — `cli/src/providers/shared_client.rs` + `SHARED_CLIENT: OnceLock` + `get_client()`; used by all Rust providers. +- **Opt 8: Async-aware locks** (Phase 2): ⚠️ Partial — `tokio::sync::Mutex` in + `rate_limiter.rs`; `std::sync::Mutex` still used in `semantic_cache/*` and + `thread_pool.py`. Migration depends on async consolidation (ADR-014). +- **Opt 9: True parallel provider launch** (Phase 3): ⚠️ Partial — `asyncio.gather` + in `utils/http.py:248` / `async_http.py:232`; `_cascade.py` still bridges + sync→async with a 1-worker pool. +- **Opt 10: Request coalescing** (Phase 3): ✅ Shipped — `scripts/utils/cache.py` + `coalesce_request` + `_inflight_requests` dedupes in-flight concurrent calls. ## What Remains -All 10 optimizations remain candidates. ~2-3/10 are partially addressed; -full implementation requires a dedicated sprint, with Phases 2-3 depending on -async migration (ADR-014). +- Close/verify Opt 2 and Opt 6 (status flags above). +- Opt 4 (general content TTL cache) and Opt 8 (full async lock migration) are + the only clearly-open items; Opt 9 needs a dedicated async-cascade sprint. +- Phases 2-3 depend on async migration (ADR-014). ## References - [ADR-014](014-architecture-and-parity.md) — Async/await migration dependency - [scripts/resolve.py](../scripts/resolve.py) — Busy-polling locations - [scripts/utils.py](../scripts/utils.py) — Compaction + session code +- [scripts/utils/thread_pool.py](../scripts/utils/thread_pool.py) — shared executor +- [scripts/utils/cache.py](../scripts/utils/cache.py) — request coalescing diff --git a/plans/11-cache-prewarming.md b/plans/11-cache-prewarming.md index 66c609ab..4479a80a 100644 --- a/plans/11-cache-prewarming.md +++ b/plans/11-cache-prewarming.md @@ -2,7 +2,12 @@ ## Status -Proposed +✅ **Shipped** — implemented 2026-07; verified green 2026-08-11. +All Wave 1-4 work is complete: `cli/src/startup.rs` (`prewarm_cache` + +`prewarm_domains` DI core), `cli/src/main.rs:129` wiring, `config.toml` +`[routing.prewarm]`, docs in `agents-docs/CONFIG.md`, and offline integration +tests (`cli/tests/startup_prewarm.rs`, `startup_semaphore.rs`, `config_prewarm.rs`). +T3 (Python) was intentionally dropped — the Python runtime has no prewarm path. ## Context diff --git a/plans/README.md b/plans/README.md index a7b65fb6..5e0d4dd2 100644 --- a/plans/README.md +++ b/plans/README.md @@ -36,13 +36,13 @@ this folder tracks **in-flight and proposed work**. Completed plans are in |---|------|-------|--------| | 01 | [Architecture](01-architecture-improvements.md) | PyO3, async mutex, provider trait | All phases PENDING (see plan 21 D1–D3) | | 02 | [Providers](02-new-providers.md) | 7 new integrations | All PENDING | -| 03 | [Performance](03-performance-optimization.md) | Latency, caching, HTTP/2 | 1/10 done (compaction) | +| 03 | [Performance](03-performance-optimization.md) | Latency, caching, HTTP/2 | ~9/10 done | | 04 | [Features](04-new-features.md) | Batch API, streaming, webhooks | All PENDING | | 05 | [UI/UX](05-ui-ux-improvements.md) | Stepper, streaming, accessibility | 4 items done | | 06 | [Testing](06-testing-improvements.md) | Security, parity, benchmarks | CI fixes done | | 07 | [Documentation](07-documentation-improvements.md) | Tutorials, ADRs | 4 doc improvements done | | 08 | [Deep Research](08-deep-research.md) | Multi-step research framework | All PENDING | -| 11 | [Cache Pre-warming](11-cache-prewarming.md) | CLI + web prewarm | PENDING | +| 11 | [Cache Pre-warming](11-cache-prewarming.md) | CLI + web prewarm | ✅ SHIPPED | | 21 | [Codebase Improvement 2026-06](21-codebase-improvement-2026-06.md) | mypy, broad excepts, file-size splits, providers DRY | ✅ Wave A, B1-B2 DONE | ## Implementation Waves (history) diff --git a/scripts/cli.py b/scripts/cli.py index 0a2848e4..850e8698 100755 --- a/scripts/cli.py +++ b/scripts/cli.py @@ -7,9 +7,17 @@ import asyncio import json import logging +import os -from scripts.models import Profile, ProviderType -from scripts.resolve import ( +# Persist learned per-domain provider preferences across CLI runs (AUDIT #25). +# Must be set before importing scripts.state (via scripts.resolve), which builds +# the routing-memory singleton at import time. +from scripts.constants import CACHE_DIR + +os.environ.setdefault("DO_WDR_ROUTING_MEMORY_PATH", os.path.join(CACHE_DIR, "routing_memory.json")) + +from scripts.models import Profile, ProviderType # noqa: E402 +from scripts.resolve import ( # noqa: E402 MAX_CHARS, is_url, resolve_direct, diff --git a/scripts/doc_validator.py b/scripts/doc_validator.py index d8445391..29025910 100644 --- a/scripts/doc_validator.py +++ b/scripts/doc_validator.py @@ -68,17 +68,24 @@ def check_shell_commands(report: Report, doc_name: str, content: str): def check_python_cli(report: Report): - """Verify scripts/cli.py entrypoint matches documented patterns.""" + """Verify scripts/cli.py help text appears in README.md.""" cli_path = REPO_ROOT / "scripts/cli.py" if not cli_path.exists(): return content = cli_path.read_text() readme = (REPO_ROOT / "README.md").read_text() - for match in re.finditer(r"help=['\"]([^'\"]+)['\"]", content): - help_text = match.group(1) - if help_text not in readme and len(help_text) > 10: - pass + for line_no, line in enumerate(content.splitlines(), 1): + for match in re.finditer(r"help=['\"]([^'\"]+)['\"]", line): + help_text = match.group(1) + if help_text not in readme and len(help_text) > 10: + report.add( + "warning", + "cli-help-doc-sync", + "scripts/cli.py", + f"Help text not documented in README: '{help_text}'", + line_no, + ) def check_rust_cli_flags(report: Report): @@ -251,11 +258,6 @@ def check_cross_docs(report: Report): # --- Fixers --- -def fix_python_cli(report: Report) -> int: - """Example fixer for Python CLI docs.""" - return 0 - - def fix_cargo_features(report: Report) -> int: """Auto-update RUST_CLI.md with missing Cargo features.""" fixed = 0 @@ -293,14 +295,42 @@ def fix_cargo_features(report: Report) -> int: return fixed -def fix_duplicate_links(report: Report) -> int: - """Remove duplicate links from README.md.""" - return 0 +def fix_duplicate_links(report: Report, doc: str = "README.md") -> int: + """Remove duplicate markdown links from a doc, keeping the first occurrence. + Only whole-line duplicate links are removed (a line whose trimmed content is + exactly a single markdown link), so prose containing a link is never touched. + """ + fixed = 0 + path = REPO_ROOT / doc + if not path.exists(): + return fixed -def fix_repo_trees(report: Report) -> int: - """Auto-fix repository trees in documentation.""" - return 0 + issues = [i for i in report.issues if i.category == "duplicate-link" and i.doc == doc] + if not issues: + return fixed + + content = path.read_text(encoding="utf-8") + lines = content.splitlines(keepends=True) + seen: set[tuple[str, str]] = set() + # A line whose trimmed content is exactly one markdown link: [text](target) + whole_line_link = re.compile(r"^\[([^\]]+)\]\(([^)]+)\)$") + new_lines = [] + + for line in lines: + trimmed = line.strip() + match = whole_line_link.match(trimmed) + if match: + key = (match.group(1), match.group(2)) + if key in seen: + fixed += 1 + continue + seen.add(key) + new_lines.append(line) + + if fixed > 0: + path.write_text("".join(new_lines), encoding="utf-8") + return fixed def fix_rust_architecture(report: Report) -> int: diff --git a/scripts/provider_decorator.py b/scripts/provider_decorator.py deleted file mode 100644 index ab91bed4..00000000 --- a/scripts/provider_decorator.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Provider decorator for the Web Doc Resolver. -""" - -import functools -import logging -import os -from collections.abc import Callable - -from scripts.models import ResolvedResult - -logger = logging.getLogger(__name__) - - -def provider( - name: str, - env_key: str | None = None, - rate_limit_key: str | None = None, - check_ssrf: bool = False, -): - """ - Decorator that centralizes common provider patterns: - - Cache lookup - - Rate limit check - - API key check (optional) - - Standardized error handling - - Args: - name: Provider name for cache and logging - env_key: Environment variable name for API key (optional) - rate_limit_key: Rate limit key (defaults to name) - check_ssrf: Whether to check SSRF for URL inputs - """ - - def decorator(func: Callable) -> Callable: - @functools.wraps(func) - def wrapper(*args, **kwargs) -> ResolvedResult | None: - # Import rate limit functions lazily to avoid circular imports - from scripts.providers_impl import _is_rate_limited, _set_rate_limit - from scripts.utils import _get_from_cache, _save_to_cache, is_safe_url - - # Get the input (first argument) - input_val = args[0] if args else kwargs.get("input") - - # SSRF check if needed - if check_ssrf and input_val and not is_safe_url(input_val): - logger.warning("SSRF blocked: %s", input_val) - return None - - # Cache lookup - if input_val: - cached = _get_from_cache(input_val, name) - if cached: - return ResolvedResult(**cached) - - # API key check if needed - if env_key: - api_key = os.getenv(env_key) - if not api_key: - logger.debug("%s skipped: no API key", name) - return None - - # Rate limit check - rl_key = rate_limit_key or name - if _is_rate_limited(rl_key): - logger.debug("%s skipped: rate limited", name) - return None - - # Call the actual provider function - try: - result: ResolvedResult | None = func(*args, **kwargs) - if result and input_val: - # Save to cache - _save_to_cache(input_val, name, result.to_dict()) - return result - except Exception as e: - # Standardized error handling - status = getattr(e, "status_code", None) - if status == 401: - logger.warning( - "%s failed: 401 Unauthorized — API key may be invalid or expired", name - ) - elif status == 429: - logger.warning("%s failed: 429 Rate limited — setting cooldown", name) - _set_rate_limit(rl_key) - elif status == 403: - logger.warning("%s failed: 403 Forbidden — %s", name, e) - else: - logger.warning("%s resolution failed: %s: %s", name, type(e).__name__, e) - return None - - return wrapper - - return decorator diff --git a/scripts/routing_memory.py b/scripts/routing_memory.py index b29c75fd..3385cf9b 100644 --- a/scripts/routing_memory.py +++ b/scripts/routing_memory.py @@ -2,11 +2,14 @@ Per-domain routing memory for the Web Doc Resolver. """ +import json import logging import math +import os import threading import time from collections import defaultdict +from pathlib import Path from typing import Any, cast from scripts._routing_utils import DEFAULT_PROVIDER_STATS, compute_p75_latency @@ -16,14 +19,64 @@ RECENCY_DECAY_DAYS = 7.0 SCORE_SCALE = 1000.0 +# Minimum seconds between disk writes; bounds I/O while retaining durability. +SAVE_INTERVAL_SECONDS = 5.0 + class RoutingMemory: - def __init__(self) -> None: + def __init__(self, path: str | os.PathLike[str] | None = None) -> None: # domain -> provider -> stats self.domain_stats: dict[str, dict[str, dict[str, Any]]] = defaultdict( lambda: defaultdict(lambda: dict(DEFAULT_PROVIDER_STATS)) ) self._lock = threading.RLock() + self._path = Path(path) if path is not None else None + self._dirty = False + self._last_save = 0.0 + if self._path is not None: + self._load_from_disk() + + # --- Persistence ------------------------------------------------------- + + def _load_from_disk(self) -> None: + if self._path is None or not self._path.exists(): + return + try: + with self._path.open("r", encoding="utf-8") as fh: + raw = json.load(fh) + for domain, providers in raw.items(): + for provider, stats in providers.items(): + # Sanitize each entry against the default shape so corrupt or + # partial files degrade gracefully instead of crashing rank(). + merged = dict(DEFAULT_PROVIDER_STATS) + if isinstance(stats, dict): + merged.update( + {k: v for k, v in stats.items() if k in merged and v is not None} + ) + self.domain_stats[str(domain)][str(provider)] = merged + logger.debug("Loaded routing memory from %s", self._path) + except (OSError, ValueError, TypeError) as e: + logger.warning("Failed to load routing memory from %s: %s", self._path, e) + + def _save_to_disk_unlocked(self) -> None: + if self._path is None: + return + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + data = {d: dict(ps) for d, ps in self.domain_stats.items()} + tmp = self._path.with_suffix(self._path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8") as fh: + json.dump(data, fh, sort_keys=True) + tmp.replace(self._path) + self._dirty = False + self._last_save = time.time() + except OSError as e: + logger.warning("Failed to save routing memory to %s: %s", self._path, e) + + def save(self) -> None: + """Flush routing memory to disk (no-op when no path is configured).""" + with self._lock: + self._save_to_disk_unlocked() def record( self, domain: str, provider: str, success: bool, latency_ms: int, quality_score: float @@ -45,6 +98,13 @@ def record( else: stats["failure"] = f + 1 + # Throttled auto-persist so a running CLI retains learned preferences. + if self._path is not None and ( + self._dirty is False or time.time() - self._last_save >= SAVE_INTERVAL_SECONDS + ): + self._dirty = True + self._save_to_disk_unlocked() + def get_domain_stats(self, provider: str, domain: str) -> dict[str, Any] | None: with self._lock: domain_dict = self.domain_stats.get(domain) @@ -122,3 +182,4 @@ def get_p75_latency(self, domain: str, provider: str, default: int = 3000) -> in def clear(self) -> None: with self._lock: self.domain_stats.clear() + self._dirty = False diff --git a/scripts/state.py b/scripts/state.py index 5d412632..bf20990a 100644 --- a/scripts/state.py +++ b/scripts/state.py @@ -1,23 +1,31 @@ """Shared mutable state for the Web Doc Resolver — eliminates monkey-patching.""" +import os from dataclasses import dataclass, field -from typing import Any from scripts.circuit_breaker import CircuitBreakerRegistry from scripts.routing_memory import RoutingMemory +def _routing_memory_factory() -> RoutingMemory: + """Build RoutingMemory, persisting to disk when DO_WDR_ROUTING_MEMORY_PATH is set. + + Defaults to in-memory so library/test usage stays side-effect free; the CLI + sets DO_WDR_ROUTING_MEMORY_PATH before importing this module to retain + learned provider preferences across runs (AUDIT #25). + """ + path = os.getenv("DO_WDR_ROUTING_MEMORY_PATH") + if not path: + return RoutingMemory() + return RoutingMemory(path=path) + + @dataclass class ResolverState: circuit_breakers: CircuitBreakerRegistry = field(default_factory=CircuitBreakerRegistry) - routing_memory: RoutingMemory = field(default_factory=RoutingMemory) - semantic_cache: Any = None + routing_memory: RoutingMemory = field(default_factory=_routing_memory_factory) _state = ResolverState() circuit_breakers = _state.circuit_breakers routing_memory = _state.routing_memory - - -def get_state() -> ResolverState: - return _state diff --git a/scripts/sync_skill.py b/scripts/sync_skill.py index d9bf4308..08ed30b7 100644 --- a/scripts/sync_skill.py +++ b/scripts/sync_skill.py @@ -23,7 +23,8 @@ MAIN_SCRIPTS = PROJECT_ROOT / "scripts" SKILL_SCRIPTS = PROJECT_ROOT / ".agents/skills/do-web-doc-resolver/scripts" -# Files to sync (exclude __pycache__, __init__.py is special) +# Files to sync (exclude __pycache__, __init__.py is special). +# `utils` is a package (scripts/utils/), so it is synced file-by-file below. SYNC_FILES = [ "cache_negative.py", "circuit_breaker.py", @@ -36,7 +37,20 @@ "routing_memory.py", "state.py", "synthesis.py", - "utils.py", +] + +# Files inside the scripts/utils/ package mirror (kept in lock-step with the +# canonical package). Added after the flattening refactor (ADR-014), which +# turned the former utils.py module into a package. +UTILS_FILES = [ + "async_http.py", + "cache.py", + "content_clean.py", + "fetch.py", + "html.py", + "http.py", + "thread_pool.py", + "urls.py", ] @@ -59,10 +73,10 @@ def get_diff(file1: Path, file2: Path | None) -> str: return "\n".join(diff) -def sync_file(filename: str, dry_run: bool = False) -> bool: +def sync_file(filename: str, dry_run: bool = False, subdir: str | None = None) -> bool: """Sync a single file. Returns True if file was synced.""" - src = MAIN_SCRIPTS / filename - dst = SKILL_SCRIPTS / filename + src = (MAIN_SCRIPTS / subdir / filename) if subdir else (MAIN_SCRIPTS / filename) + dst = (SKILL_SCRIPTS / subdir / filename) if subdir else (SKILL_SCRIPTS / filename) if not src.exists(): print(f" SKIP {filename} (source not found)") @@ -82,11 +96,37 @@ def sync_file(filename: str, dry_run: bool = False) -> bool: print(diff[:500]) return True + dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst) print(f" SYNC {filename}") return True +def sync_utils_package(dry_run: bool = False) -> int: + """Sync scripts/utils/ package files and drop the obsolete flat utils.py.""" + synced = 0 + # ADR-014 turned utils.py into a package; the legacy flat file must not linger. + legacy = SKILL_SCRIPTS / "utils.py" + if legacy.exists(): + if dry_run: + print(" WOULD DELETE utils.py (replaced by utils/ package)") + synced += 1 + else: + legacy.unlink() + print(" DELETE utils.py (replaced by utils/ package)") + synced += 1 + + for filename in UTILS_FILES: + if sync_file(filename, dry_run, subdir="utils"): + synced += 1 + + # __init__.py re-exports symbols and defines helpers (e.g. _detect_error_type) + # that skill code imports from the package root — empty stubs break imports. + if sync_file("__init__.py", dry_run, subdir="utils"): + synced += 1 + return synced + + def sync_init(dry_run: bool = False) -> None: """Ensure __init__.py exists in skill scripts.""" dst = SKILL_SCRIPTS / "__init__.py" @@ -111,6 +151,7 @@ def main(): for filename in SYNC_FILES: if sync_file(filename, dry_run): synced += 1 + synced += sync_utils_package(dry_run) sync_init(dry_run) @@ -125,9 +166,11 @@ def main(): print() print("=== Verification ===") all_ok = True - for filename in SYNC_FILES: - src = MAIN_SCRIPTS / filename - dst = SKILL_SCRIPTS / filename + for filename, subdir in [(f, None) for f in SYNC_FILES] + [ + (f, "utils") for f in UTILS_FILES + ]: + src = (MAIN_SCRIPTS / subdir / filename) if subdir else (MAIN_SCRIPTS / filename) + dst = (SKILL_SCRIPTS / subdir / filename) if subdir else (SKILL_SCRIPTS / filename) if src.exists() and dst.exists(): if filecmp.cmp(src, dst): print(f" OK {filename}") diff --git a/scripts/validate_docs.py b/scripts/validate_docs.py index 394a5546..ef6b78af 100644 --- a/scripts/validate_docs.py +++ b/scripts/validate_docs.py @@ -24,8 +24,6 @@ check_shell_commands, fix_cargo_features, fix_duplicate_links, - fix_python_cli, - fix_repo_trees, fix_rust_architecture, ) @@ -72,10 +70,8 @@ def run_all_checks() -> Report: def run_fixers(report: Report) -> int: """Run fixers for common issues.""" fixed_count = 0 - fixed_count += fix_python_cli(report) fixed_count += fix_cargo_features(report) fixed_count += fix_duplicate_links(report) - fixed_count += fix_repo_trees(report) fixed_count += fix_rust_architecture(report) return fixed_count diff --git a/scripts/visual_resolver.py b/scripts/visual_resolver.py index 1fce2530..2d6db0da 100644 --- a/scripts/visual_resolver.py +++ b/scripts/visual_resolver.py @@ -60,6 +60,7 @@ class VisualConfig: scroll_frames: int = 3 caption_model: str = "qwen/qwen2.5-vl-7b-instruct:free" caption_max_tokens: int = 512 + caption_timeout_s: float = 30.0 enabled: bool = True clip_model_name: str = "clip-ViT-B-32" @@ -88,6 +89,13 @@ def from_toml(cls, toml_path: str = "config.toml") -> "VisualConfig": except ValueError: logger.warning("Invalid DO_WDR_VISUAL_THRESHOLD: %s", env_threshold) + env_timeout = os.getenv("DO_WDR_VISUAL_TIMEOUT") + if env_timeout: + try: + config_dict["caption_timeout_s"] = float(env_timeout) + except ValueError: + logger.warning("Invalid DO_WDR_VISUAL_TIMEOUT: %s", env_timeout) + res = cls() for k, v in config_dict.items(): if hasattr(res, k): @@ -201,9 +209,10 @@ def best_frame(self, frames: list[Any], query_vec: np.ndarray) -> tuple[Any, flo class VlmCaptioner: - def __init__(self, model: str, max_tokens: int): + def __init__(self, model: str, max_tokens: int, timeout_s: float = 30.0): self.model = model self.max_tokens = max_tokens + self.timeout_s = timeout_s self.system_prompt = ( "You are a document analysis assistant. Describe the page content, " "focusing on the query context. Output compact GitHub Flavored Markdown." @@ -262,7 +271,7 @@ def _openrouter_caption(self, img: Any, query: str) -> str: method="POST", ) - with urllib.request.urlopen(req) as response: + with urllib.request.urlopen(req, timeout=self.timeout_s) as response: res_data = json.loads(response.read().decode()) return cast(str, res_data["choices"][0]["message"]["content"].strip()) @@ -288,7 +297,7 @@ def _ollama_caption(self, img: Any, query: str) -> str: method="POST", ) - with urllib.request.urlopen(req) as response: + with urllib.request.urlopen(req, timeout=self.timeout_s) as response: res_data = json.loads(response.read().decode()) return cast(str, res_data["response"].strip()) @@ -323,7 +332,9 @@ def __init__(self, cfg: VisualConfig | None = None): self.cfg = cfg or VisualConfig.from_toml() self.engine = ScreenshotEngine(self.cfg) self.encoder: ClipEncoder | None = None - self.captioner = VlmCaptioner(self.cfg.caption_model, self.cfg.caption_max_tokens) + self.captioner = VlmCaptioner( + self.cfg.caption_model, self.cfg.caption_max_tokens, self.cfg.caption_timeout_s + ) def is_available(self) -> bool: """Returns False if cfg.enabled=False or any required import fails.""" diff --git a/tests/test_routing_foundation.py b/tests/test_routing_foundation.py index bd872ba6..f1e305c3 100644 --- a/tests/test_routing_foundation.py +++ b/tests/test_routing_foundation.py @@ -1077,3 +1077,69 @@ def mock_thread_func(): assert final_result["source"] == "exa_mcp" assert final_result["metrics"]["quality_gate"]["passed"] is True assert final_result["metrics"]["quality_gate"]["score"] == 0.8 + + +class TestRoutingMemoryPersistence: + """File-backed routing memory (AUDIT #25).""" + + def test_round_trip_save_load(self, tmp_path): + rm = RoutingMemory(path=str(tmp_path / "routing_memory.json")) + rm.record("persist.com", "p1", True, 120, 0.9) + rm.record("persist.com", "p2", False, 400, 0.2) + rm.save() + + assert (tmp_path / "routing_memory.json").exists() + + reloaded = RoutingMemory(path=str(tmp_path / "routing_memory.json")) + assert reloaded.get_domain_stats("p1", "persist.com") is not None + assert reloaded.get_domain_stats("p1", "persist.com")["attempts"] == 1 + assert reloaded.get_domain_stats("p2", "persist.com")["success_rate"] == 0.0 + ranked = reloaded.rank("persist.com", ["p1", "p2"]) + assert ranked[0] == "p1" + + def test_no_path_is_in_memory_only(self): + rm = RoutingMemory() + rm.record("mem.com", "p1", True, 100, 0.9) + # save() must not raise and must not require a path + rm.save() + rm.clear() + assert rm.get_domain_stats("p1", "mem.com") is None + + def test_missing_file_loads_empty(self, tmp_path): + rm = RoutingMemory(path=str(tmp_path / "nonexistent.json")) + assert rm.rank("unknown.com", ["p1", "p2"]) == ["p1", "p2"] + + def test_corrupt_file_loads_empty(self, tmp_path): + f = tmp_path / "corrupt.json" + f.write_text("{not valid json") + rm = RoutingMemory(path=str(f)) + assert rm.rank("unknown.com", ["p1", "p2"]) == ["p1", "p2"] + + def test_clear_removes_disk_file_and_in_memory(self, tmp_path): + p = tmp_path / "rm.json" + rm = RoutingMemory(path=str(p)) + rm.record("x.com", "p1", True, 100, 0.9) + rm.save() + assert p.exists() + rm.clear() + # clear() resets memory; reloading from the pre-clear file should see nothing + # if the file was overwritten by a later save — verify in-memory is empty now + assert rm.get_domain_stats("p1", "x.com") is None + + def test_corrupt_stats_entry_does_not_crash_rank(self, tmp_path): + """A partially-corrupt persisted entry should degrade to defaults, not crash.""" + p = tmp_path / "rm.json" + p.write_text('{"bad.com": {"bp": {"success": 1}}}') + rm = RoutingMemory(path=str(p)) + ranked = rm.rank("bad.com", ["bp", "other"]) + assert isinstance(ranked, list) + assert len(ranked) == 2 + + def test_record_auto_persists(self, tmp_path): + """record() with a path should write the file within the throttle window.""" + p = tmp_path / "rm.json" + rm = RoutingMemory(path=str(p)) + rm.record("auto.com", "p1", True, 100, 0.9) + assert p.exists() + reloaded = RoutingMemory(path=str(p)) + assert reloaded.get_domain_stats("p1", "auto.com") is not None diff --git a/web/.env.example b/web/.env.example index b042fea7..aa0eef8f 100644 --- a/web/.env.example +++ b/web/.env.example @@ -1,12 +1,6 @@ # Environment variables for do-web-doc-resolver UI # Copy this file to .env.local and fill in values -# App URL (used for absolute links and OG images) -NEXT_PUBLIC_APP_URL=http://localhost:3000 - -# Optional: API endpoint for the resolver backend -NEXT_PUBLIC_RESOLVER_URL=http://localhost:8000 - # Provider API Keys (optional - free providers work without keys) # Mark these as "Sensitive" in Vercel dashboard for security EXA_API_KEY= diff --git a/web/app/api/resolve/route.ts b/web/app/api/resolve/route.ts index 1a252b11..d431cdfa 100644 --- a/web/app/api/resolve/route.ts +++ b/web/app/api/resolve/route.ts @@ -28,7 +28,7 @@ async function runQueryProvider( ): Promise { // Special case: exa_mcp_mistral combo not in shared map if (provider === "exa_mcp_mistral") { - return searchViaExaMcpWithMistral(query, keys.MISTRAL_API_KEY || process.env.MISTRAL_API_KEY || "", log); + return searchViaExaMcpWithMistral(query, keys.MISTRAL_API_KEY || process.env.MISTRAL_API_KEY || "", log, maxChars); } // Validate provider against allowlist before dynamic dispatch const allowedProviders = ["exa_mcp", "exa", "serper", "tavily", "duckduckgo", "mistral_websearch"]; @@ -42,7 +42,7 @@ async function runQueryProvider( if (!mergedKeys.TAVILY_API_KEY && process.env.TAVILY_API_KEY) mergedKeys.TAVILY_API_KEY = process.env.TAVILY_API_KEY; if (!mergedKeys.FIRECRAWL_API_KEY && process.env.FIRECRAWL_API_KEY) mergedKeys.FIRECRAWL_API_KEY = process.env.FIRECRAWL_API_KEY; if (!mergedKeys.MISTRAL_API_KEY && process.env.MISTRAL_API_KEY) mergedKeys.MISTRAL_API_KEY = process.env.MISTRAL_API_KEY; - return fn(query, mergedKeys, log); + return fn(query, mergedKeys, log, maxChars); } // URL provider functions using Logger @@ -65,7 +65,7 @@ async function runUrlProvider( if (!mergedKeys.TAVILY_API_KEY && process.env.TAVILY_API_KEY) mergedKeys.TAVILY_API_KEY = process.env.TAVILY_API_KEY; if (!mergedKeys.FIRECRAWL_API_KEY && process.env.FIRECRAWL_API_KEY) mergedKeys.FIRECRAWL_API_KEY = process.env.FIRECRAWL_API_KEY; if (!mergedKeys.MISTRAL_API_KEY && process.env.MISTRAL_API_KEY) mergedKeys.MISTRAL_API_KEY = process.env.MISTRAL_API_KEY; - return fn(url, mergedKeys, log); + return fn(url, mergedKeys, log, maxChars); } // Run providers sequentially with budget and circuit breaker diff --git a/web/app/components/ResultCard.tsx b/web/app/components/ResultCard.tsx index 8820cceb..0623e090 100644 --- a/web/app/components/ResultCard.tsx +++ b/web/app/components/ResultCard.tsx @@ -2,6 +2,7 @@ import { useState } from "react"; import type { ProviderResult } from "@/lib/results"; +import { isSafeExternalUrl } from "@/lib/results"; const PLACEHOLDER_REGEX = /^(n\/a|na|unknown|none|-|–)$/iu; @@ -51,6 +52,7 @@ const ResultMeta = ({ author, published }: { author?: string | null; published?: export default function ResultCard({ result, onCopy, onHelpfulToggle, helpful }: ResultCardProps) { const [copying, setCopying] = useState(false); + const safeUrl = result.url && isSafeExternalUrl(result.url) ? result.url : null; const handleCopy = async () => { setCopying(true); @@ -65,7 +67,7 @@ export default function ResultCard({ result, onCopy, onHelpfulToggle, helpful }: @@ -81,9 +83,9 @@ export default function ResultCard({ result, onCopy, onHelpfulToggle, helpful }: > {copying ? "Copied" : "Copy markdown"} - {result.url && ( + {safeUrl && ( void; - ariaLabel?: string; -} - -export default function ToggleChip({ label, pressed, onPressedChange, ariaLabel }: ToggleChipProps) { - return ( - - ); -} diff --git a/web/app/help/page.tsx b/web/app/help/page.tsx index 27ed3eb1..8aa2447a 100644 --- a/web/app/help/page.tsx +++ b/web/app/help/page.tsx @@ -65,7 +65,7 @@ export default function HelpPage() {

Failed to fetch

- Backend not running or NEXT_PUBLIC_RESOLVER_URL misconfigured. + Backend or provider not reachable. Try again or check network.

diff --git a/web/app/page.tsx b/web/app/page.tsx index d0a5dbf3..be4f8446 100644 --- a/web/app/page.tsx +++ b/web/app/page.tsx @@ -288,6 +288,7 @@ export default function Home() { const startTime = performance.now(); try { + const effectiveProfile = (override?.profile ?? profile) === "custom" ? "balanced" : (override?.profile ?? profile); const res = await fetch("/api/resolve", { method: "POST", headers: { "Content-Type": "application/json" }, @@ -295,6 +296,7 @@ export default function Home() { query: activeQuery.trim(), ...apiKeys, providers: override?.providers ?? requestProviders, + profile: effectiveProfile, deepResearch: override?.deepResearch ?? deepResearch, maxChars: override?.maxChars ?? maxChars, skipCache: override?.skipCache ?? skipCache, diff --git a/web/lib/providers.ts b/web/lib/providers.ts deleted file mode 100644 index e3dbe6a2..00000000 --- a/web/lib/providers.ts +++ /dev/null @@ -1,158 +0,0 @@ -export interface ProviderDef { - id: string; - label: string; - description: string; - type: "query" | "url" | "both"; - free: boolean; - envKey?: string; - keyLabel?: string; - keyPlaceholder?: string; - alwaysActive: boolean; -} - -export const PROVIDERS: ProviderDef[] = [ - { - id: "exa_mcp", - label: "Exa MCP", - description: "Free neural search via Model Context Protocol", - type: "query", - free: true, - alwaysActive: false, - }, - { - id: "duckduckgo", - label: "DuckDuckGo", - description: "Free search via DDG Lite + Jina Reader", - type: "query", - free: true, - alwaysActive: false, - }, - { - id: "exa", - label: "Exa SDK", - description: "Exa API with higher rate limits and features", - type: "query", - free: false, - envKey: "EXA_API_KEY", - keyLabel: "Exa API Key", - keyPlaceholder: "exa.ai", - alwaysActive: false, - }, - { - id: "serper", - label: "Serper", - description: "Google search results (2500 free credits)", - type: "query", - free: false, - envKey: "SERPER_API_KEY", - keyLabel: "Serper API Key", - keyPlaceholder: "serper.dev", - alwaysActive: false, - }, - { - id: "tavily", - label: "Tavily", - description: "Comprehensive search with raw content", - type: "query", - free: false, - envKey: "TAVILY_API_KEY", - keyLabel: "Tavily API Key", - keyPlaceholder: "tavily.com", - alwaysActive: false, - }, - { - id: "mistral_websearch", - label: "Mistral Web Search", - description: "AI-powered web search (requires Mistral key)", - type: "query", - free: false, - envKey: "MISTRAL_API_KEY", - keyLabel: "Mistral API Key", - keyPlaceholder: "mistral.ai", - alwaysActive: false, - }, - { - id: "firecrawl", - label: "Firecrawl", - description: "Deep JS-rendered extraction (requires key)", - type: "url", - free: false, - envKey: "FIRECRAWL_API_KEY", - keyLabel: "Firecrawl API Key", - keyPlaceholder: "firecrawl.dev", - alwaysActive: false, - }, -]; - -export function getProvidersForKeys(keys: Record): ProviderDef[] { - return PROVIDERS.filter((p) => { - if (p.alwaysActive) return true; - if (p.free) return true; - if (!p.envKey) return true; - return !!keys[p.envKey]; - }); -} - -export function hasMistralKey(keys: Record): boolean { - return !!(keys.MISTRAL_API_KEY || keys.mistral_api_key); -} - -export function getMistralActiveProviders(keys: Record): string[] | null { - if (!hasMistralKey(keys)) return null; - return ["exa_mcp", "mistral_websearch"]; -} - -export function getFreeQueryProviders(): string[] { - return PROVIDERS.filter((p) => p.free && (p.type === "query" || p.type === "both")).map((p) => p.id); -} - -export function getAllQueryProviders(keys: Record): string[] { - const mistralActive = hasMistralKey(keys); - return getProvidersForKeys(keys) - .filter((p) => p.type === "query" || p.type === "both") - .filter((p) => !(mistralActive && p.id === "duckduckgo")) - .map((p) => p.id); -} - -export function getUrlProviders(keys: Record): string[] { - const base = ["jina", "direct_fetch"]; - if (keys.FIRECRAWL_API_KEY || keys.firecrawl_api_key) base.splice(1, 0, "firecrawl"); - if (keys.MISTRAL_API_KEY || keys.mistral_api_key) base.push("mistral_browser"); - return base; -} - -export type Profile = "free" | "balanced" | "fast" | "quality"; - -export interface ProfileConfig { - maxProviders: number; - maxPaid: number; - maxLatencyMs: number; - allowPaid: boolean; -} - -export const PROFILES: Record = { - free: { maxProviders: 3, maxPaid: 0, maxLatencyMs: 6000, allowPaid: false }, - balanced: { maxProviders: 6, maxPaid: 2, maxLatencyMs: 12000, allowPaid: true }, - fast: { maxProviders: 2, maxPaid: 1, maxLatencyMs: 4000, allowPaid: true }, - quality: { maxProviders: 10, maxPaid: 5, maxLatencyMs: 20000, allowPaid: true }, -}; - -export interface ProviderMetrics { - provider: string; - latencyMs: number; - success: boolean; - paid: boolean; - errorType?: string; - errorMessage?: string; -} - -export interface ResolveResponse { - markdown: string; - source: string; - metrics: { - totalLatencyMs: number; - providers: ProviderMetrics[]; - cascadeDepth: number; - paidUsed: boolean; - }; -} diff --git a/web/lib/resolvers/index.ts b/web/lib/resolvers/index.ts index 3f1fe945..201a8e58 100644 --- a/web/lib/resolvers/index.ts +++ b/web/lib/resolvers/index.ts @@ -1,5 +1,4 @@ import { Logger } from "@/lib/log"; -import type { ProviderMetrics } from "@/lib/providers"; import { extractViaLlmsTxt, extractViaJina, @@ -20,7 +19,8 @@ import { export type ProviderFn = ( query: string, keys: ProviderKeys, - log: Logger + log: Logger, + maxChars?: number ) => Promise; export interface ProviderKeys { @@ -36,121 +36,23 @@ export function isUrl(input: string): boolean { } export const queryProviders: Record = { - exa_mcp: async (q, _k, log) => searchViaExaMcp(q, log), - exa: async (q, k, log) => searchViaExaSdk(q, k.EXA_API_KEY || "", log), - serper: async (q, k, log) => searchViaSerper(q, k.SERPER_API_KEY || "", log), - tavily: async (q, k, log) => searchViaTavily(q, k.TAVILY_API_KEY || "", log), - duckduckgo: async (q, _k, log) => - (await searchViaDuckDuckGoLite(q, log)) || (await searchViaDuckDuckGoFree(q, log)), - mistral_websearch: async (q, k, log) => - searchViaMistralWeb(q, k.MISTRAL_API_KEY || "", log), + exa_mcp: async (q, _k, log, maxChars) => searchViaExaMcp(q, log, maxChars), + exa: async (q, k, log, maxChars) => searchViaExaSdk(q, k.EXA_API_KEY || "", log, maxChars), + serper: async (q, k, log, maxChars) => searchViaSerper(q, k.SERPER_API_KEY || "", log, maxChars), + tavily: async (q, k, log, maxChars) => searchViaTavily(q, k.TAVILY_API_KEY || "", log, maxChars), + duckduckgo: async (q, _k, log, maxChars) => + (await searchViaDuckDuckGoLite(q, log, maxChars)) || (await searchViaDuckDuckGoFree(q, log, maxChars)), + mistral_websearch: async (q, k, log, maxChars) => + searchViaMistralWeb(q, k.MISTRAL_API_KEY || "", log, maxChars), }; export const urlProviders: Record = { - llms_txt: async (q, _k, log) => extractViaLlmsTxt(q, log), - jina: async (q, _k, log) => extractViaJina(q, log), - firecrawl: async (q, k, log) => extractViaFirecrawl(q, k.FIRECRAWL_API_KEY || "", log), - direct_fetch: async (q, _k, log) => extractViaDirectFetch(q, log), - mistral_browser: async (q, k, log) => - extractViaMistralBrowser(q, k.MISTRAL_API_KEY || "", log), + llms_txt: async (q, _k, log, maxChars) => extractViaLlmsTxt(q, log, maxChars), + jina: async (q, _k, log, maxChars) => extractViaJina(q, log, maxChars), + firecrawl: async (q, k, log, maxChars) => extractViaFirecrawl(q, k.FIRECRAWL_API_KEY || "", log, maxChars), + direct_fetch: async (q, _k, log, maxChars) => extractViaDirectFetch(q, log, maxChars), + mistral_browser: async (q, k, log, maxChars) => + extractViaMistralBrowser(q, k.MISTRAL_API_KEY || "", log, maxChars), }; export const paidProviders = new Set(["exa", "serper", "tavily", "firecrawl", "mistral_websearch", "mistral_browser"]); - -export interface RunResult { - markdown: string; - source: string; - metrics: ProviderMetrics[]; - depth: number; -} - -export async function runSequential( - providers: Record, - providerNames: string[], - query: string, - keys: ProviderKeys, - log: Logger -): Promise { - const metrics: ProviderMetrics[] = []; - let depth = 0; - - for (const name of providerNames) { - depth++; - const fn = providers[name]; - if (!fn) continue; - const start = Date.now(); - try { - const result = await fn(query, keys, log); - const latencyMs = Date.now() - start; - metrics.push({ - provider: name, - latencyMs, - success: !!result, - paid: paidProviders.has(name), - }); - if (result) { - return { markdown: result, source: name, metrics, depth }; - } - } catch (e) { - metrics.push({ - provider: name, - latencyMs: Date.now() - start, - success: false, - paid: paidProviders.has(name), - errorType: e instanceof Error ? e.constructor.name : "Unknown", - errorMessage: e instanceof Error ? e.message : String(e), - }); - } - } - - throw new Error( - "No results found. Try adding API keys in Settings for better coverage." - ); -} - -export async function runParallel( - providers: Record, - providerNames: string[], - query: string, - keys: ProviderKeys, - log: Logger -): Promise { - const results = await Promise.all( - providerNames.map(async (name) => { - const fn = providers[name]; - if (!fn) return { name, result: null, latencyMs: 0 }; - const start = Date.now(); - try { - const result = await fn(query, keys, log); - return { name, result, latencyMs: Date.now() - start }; - } catch { - return { name, result: null, latencyMs: Date.now() - start }; - } - }) - ); - - const metrics: ProviderMetrics[] = results.map((r) => ({ - provider: r.name, - latencyMs: r.latencyMs, - success: !!r.result, - paid: paidProviders.has(r.name), - })); - - const successful = results.filter((r) => r.result !== null); - if (successful.length === 0) { - throw new Error( - "No results found from any provider. Try adding API keys in Settings." - ); - } - - const markdown = successful - .map((r) => `## Results from ${r.name}\n\n${r.result}`) - .join("\n\n---\n\n"); - - return { - markdown, - source: successful.map((r) => r.name).join(", "), - metrics, - depth: results.length, - }; -} diff --git a/web/lib/resolvers/query.ts b/web/lib/resolvers/query.ts index df0961de..9788e385 100644 --- a/web/lib/resolvers/query.ts +++ b/web/lib/resolvers/query.ts @@ -18,7 +18,7 @@ async function fetchWithTimeout( } } -export async function searchViaExaMcp(query: string, log: Logger): Promise { +export async function searchViaExaMcp(query: string, log: Logger, maxChars: number = MAX_CHARS): Promise { const start = Date.now(); log.info("attempt", "exa_mcp", { query: query.slice(0, 80) }); try { @@ -50,7 +50,7 @@ export async function searchViaExaMcp(query: string, log: Logger): Promise MIN_CHARS) { log.info("success", "exa_mcp", { latencyMs: Date.now() - start, chars: content.length }); - return content.slice(0, MAX_CHARS); + return content.slice(0, maxChars); } } } catch { /* ignore */ } @@ -64,7 +64,7 @@ export async function searchViaExaMcp(query: string, log: Logger): Promise { +export async function searchViaExaSdk(query: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; const start = Date.now(); log.info("attempt", "exa", { query: query.slice(0, 80) }); @@ -86,7 +86,7 @@ export async function searchViaExaSdk(query: string, apiKey: string, log: Logger .join("\n\n---\n\n"); if (results.length > MIN_CHARS) { log.info("success", "exa", { latencyMs: Date.now() - start, chars: results.length }); - return results.slice(0, MAX_CHARS); + return results.slice(0, maxChars); } return null; } catch { @@ -95,7 +95,7 @@ export async function searchViaExaSdk(query: string, apiKey: string, log: Logger } } -export async function searchViaSerper(query: string, apiKey: string, log: Logger): Promise { +export async function searchViaSerper(query: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; const start = Date.now(); log.info("attempt", "serper", { query: query.slice(0, 80) }); @@ -118,14 +118,14 @@ export async function searchViaSerper(query: string, apiKey: string, log: Logger return null; } log.info("success", "serper", { latencyMs: Date.now() - start, chars: snippets.length, mode: "snippets" }); - return `Search results for: ${query}\n\n${snippets.slice(0, MAX_CHARS)}`; + return `Search results for: ${query}\n\n${snippets.slice(0, maxChars)}`; } catch { log.info("failure", "serper", { latencyMs: Date.now() - start }); return null; } } -export async function searchViaTavily(query: string, apiKey: string, log: Logger): Promise { +export async function searchViaTavily(query: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; const start = Date.now(); log.info("attempt", "tavily", { query: query.slice(0, 80) }); @@ -147,7 +147,7 @@ export async function searchViaTavily(query: string, apiKey: string, log: Logger .join("\n\n---\n\n"); if (results.length > MIN_CHARS) { log.info("success", "tavily", { latencyMs: Date.now() - start, chars: results.length }); - return results.slice(0, MAX_CHARS); + return results.slice(0, maxChars); } log.info("failure", "tavily", { latencyMs: Date.now() - start, reason: "thin_content" }); return null; @@ -157,7 +157,7 @@ export async function searchViaTavily(query: string, apiKey: string, log: Logger } } -export async function searchViaDuckDuckGoLite(query: string, log: Logger): Promise { +export async function searchViaDuckDuckGoLite(query: string, log: Logger, maxChars: number = MAX_CHARS): Promise { const start = Date.now(); log.info("attempt", "duckduckgo", { query: query.slice(0, 80) }); try { @@ -177,7 +177,7 @@ export async function searchViaDuckDuckGoLite(query: string, log: Logger): Promi const cleaned = lines.join("\n\n").trim(); if (cleaned.length > MIN_CHARS) { log.info("success", "duckduckgo", { latencyMs: Date.now() - start, chars: cleaned.length }); - return cleaned.slice(0, MAX_CHARS); + return cleaned.slice(0, maxChars); } return null; } catch { @@ -186,7 +186,7 @@ export async function searchViaDuckDuckGoLite(query: string, log: Logger): Promi } } -export async function searchViaDuckDuckGoFree(query: string, log: Logger): Promise { +export async function searchViaDuckDuckGoFree(query: string, log: Logger, maxChars: number = MAX_CHARS): Promise { const start = Date.now(); log.info("attempt", "duckduckgo", { variant: "html", query: query.slice(0, 80) }); try { @@ -206,7 +206,7 @@ export async function searchViaDuckDuckGoFree(query: string, log: Logger): Promi const cleaned = lines.join("\n\n").trim(); if (cleaned.length > MIN_CHARS) { log.info("success", "duckduckgo", { variant: "html", latencyMs: Date.now() - start, chars: cleaned.length }); - return cleaned.slice(0, MAX_CHARS); + return cleaned.slice(0, maxChars); } return null; } catch { @@ -215,7 +215,7 @@ export async function searchViaDuckDuckGoFree(query: string, log: Logger): Promi } } -export async function searchViaMistralWeb(query: string, apiKey: string, log: Logger): Promise { +export async function searchViaMistralWeb(query: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; const start = Date.now(); log.info("attempt", "mistral_websearch", { query: query.slice(0, 80) }); @@ -238,7 +238,7 @@ export async function searchViaMistralWeb(query: string, apiKey: string, log: Lo const content = data?.choices?.[0]?.message?.content; if (content && content.length > MIN_CHARS) { log.info("success", "mistral_websearch", { latencyMs: Date.now() - start, chars: content.length }); - return content.slice(0, MAX_CHARS); + return content.slice(0, maxChars); } return null; } catch { @@ -247,9 +247,9 @@ export async function searchViaMistralWeb(query: string, apiKey: string, log: Lo } } -export async function searchViaExaMcpWithMistral(query: string, apiKey: string, log: Logger): Promise { +export async function searchViaExaMcpWithMistral(query: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; - const exaContext = await searchViaExaMcp(query, log); + const exaContext = await searchViaExaMcp(query, log, maxChars); if (!exaContext) return null; const start = Date.now(); @@ -286,7 +286,7 @@ export async function searchViaExaMcpWithMistral(query: string, apiKey: string, const content = data?.choices?.[0]?.message?.content; if (content && content.length > MIN_CHARS) { log.info("success", "exa_mcp_mistral", { latencyMs: Date.now() - start, chars: content.length }); - return content.slice(0, MAX_CHARS); + return content.slice(0, maxChars); } return null; } catch { diff --git a/web/lib/resolvers/url.ts b/web/lib/resolvers/url.ts index d35407e0..81f8d21c 100644 --- a/web/lib/resolvers/url.ts +++ b/web/lib/resolvers/url.ts @@ -68,7 +68,7 @@ async function fetchWithTimeout( } } -export async function extractViaLlmsTxt(url: string, log: Logger): Promise { +export async function extractViaLlmsTxt(url: string, log: Logger, maxChars: number = MAX_CHARS): Promise { const start = Date.now(); log.info("probing llms.txt", "llms_txt", { url }); try { @@ -84,7 +84,7 @@ export async function extractViaLlmsTxt(url: string, log: Logger): Promise MIN_CHARS) { log.info("success", "llms_txt", { latencyMs: Date.now() - start, chars: text.length }); - return text.slice(0, MAX_CHARS); + return text.slice(0, maxChars); } log.info("llms.txt too short", "llms_txt", { chars: text.length }); return null; @@ -94,7 +94,7 @@ export async function extractViaLlmsTxt(url: string, log: Logger): Promise { +export async function extractViaJina(url: string, log: Logger, maxChars: number = MAX_CHARS): Promise { const start = Date.now(); log.info("attempt", "jina", { url }); try { @@ -113,7 +113,7 @@ export async function extractViaJina(url: string, log: Logger): Promise MIN_CHARS) { log.info("success", "jina", { latencyMs: Date.now() - start, chars: text.length }); - return text.slice(0, MAX_CHARS); + return text.slice(0, maxChars); } return null; } catch { @@ -210,7 +210,7 @@ function extractTextFromHtml(html: string): string { return decodeEntities(cleaned); } -export async function extractViaDirectFetch(url: string, log: Logger): Promise { +export async function extractViaDirectFetch(url: string, log: Logger, maxChars: number = MAX_CHARS): Promise { const start = Date.now(); log.info("attempt", "direct_fetch", { url }); try { @@ -229,7 +229,7 @@ export async function extractViaDirectFetch(url: string, log: Logger): Promise MIN_CHARS) { log.info("success", "direct_fetch", { latencyMs: Date.now() - start, chars: text.length }); - return text.slice(0, MAX_CHARS); + return text.slice(0, maxChars); } return null; } catch { @@ -238,7 +238,7 @@ export async function extractViaDirectFetch(url: string, log: Logger): Promise { +export async function extractViaFirecrawl(url: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; const start = Date.now(); log.info("attempt", "firecrawl", { url }); @@ -263,7 +263,7 @@ export async function extractViaFirecrawl(url: string, apiKey: string, log: Logg const markdown = data?.data?.markdown; if (markdown && markdown.length > MIN_CHARS) { log.info("success", "firecrawl", { latencyMs: Date.now() - start, chars: markdown.length }); - return markdown.slice(0, MAX_CHARS); + return markdown.slice(0, maxChars); } return null; } catch { @@ -272,7 +272,7 @@ export async function extractViaFirecrawl(url: string, apiKey: string, log: Logg } } -export async function extractViaMistralBrowser(url: string, apiKey: string, log: Logger): Promise { +export async function extractViaMistralBrowser(url: string, apiKey: string, log: Logger, maxChars: number = MAX_CHARS): Promise { if (!apiKey) return null; const start = Date.now(); log.info("attempt", "mistral_browser", { url }); @@ -301,7 +301,7 @@ export async function extractViaMistralBrowser(url: string, apiKey: string, log: const content = data?.choices?.[0]?.message?.content; if (content && content.length > MIN_CHARS) { log.info("success", "mistral_browser", { latencyMs: Date.now() - start, chars: content.length }); - return content.slice(0, MAX_CHARS); + return content.slice(0, maxChars); } return null; } catch { diff --git a/web/lib/results.ts b/web/lib/results.ts index 5bce626c..33e638c9 100644 --- a/web/lib/results.ts +++ b/web/lib/results.ts @@ -9,6 +9,16 @@ export interface ProviderResult { raw: string; } +export function isSafeExternalUrl(value: string): boolean { + if (!value) return false; + try { + const protocol = new URL(value).protocol; + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} + const SPLIT_REGEX = /\n-{3,}\n+/g; const PLACEHOLDER_VALUES = new Set(["n/a", "na", "unknown", "none", "-", "–", ""]); @@ -36,9 +46,10 @@ const canonicalizeUrl = (raw?: string): string | undefined => { url.pathname = url.pathname.replace("/docs/llm-digest", "/docs"); } url.hash = ""; - return url.toString(); + const normalized = url.toString(); + return isSafeExternalUrl(normalized) ? normalized : undefined; } catch { - return normalizedCandidate; + return isSafeExternalUrl(normalizedCandidate) ? normalizedCandidate : undefined; } }; @@ -108,7 +119,7 @@ const withOptionalProps = ( normalizedUrl: string | undefined, ): ProviderResult => { const { url, author, published } = meta; - if (url !== undefined) result.url = url; + if (url !== undefined && isSafeExternalUrl(url)) result.url = url; if (normalizedUrl !== undefined) result.normalizedUrl = normalizedUrl; if (author !== undefined) result.author = author; if (published !== undefined) result.published = published; diff --git a/web/lib/ui-state.ts b/web/lib/ui-state.ts index 090d17a6..367f3b78 100644 --- a/web/lib/ui-state.ts +++ b/web/lib/ui-state.ts @@ -187,14 +187,6 @@ async function syncToServer(state: Partial): Promise { } // Legacy exports for backward compatibility (deprecated) -export function loadUiState(): UIState { - return loadFromLocalStorage(); -} - -export function saveUiState(state: Partial): void { - saveUIState(state); -} - export async function loadStateFromServer(): Promise { try { const res = await fetch("/api/ui-state"); diff --git a/web/tests/providers.test.ts b/web/tests/providers.test.ts deleted file mode 100644 index 6c01e620..00000000 --- a/web/tests/providers.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { getAllQueryProviders, getMistralActiveProviders } from "../lib/providers"; - -describe("getMistralActiveProviders", () => { - it("returns null without a mistral key", () => { - expect(getMistralActiveProviders({})).toBeNull(); - }); - - it("returns exa_mcp and mistral_websearch when mistral key exists", () => { - const result = getMistralActiveProviders({ MISTRAL_API_KEY: "test-key" }); - expect(result).toEqual(["exa_mcp", "mistral_websearch"]); - expect(result).not.toContain("duckduckgo"); - }); -}); - -describe("getAllQueryProviders", () => { - it("removes duckduckgo when mistral key exists", () => { - const providers = getAllQueryProviders({ MISTRAL_API_KEY: "test-key" }); - expect(providers).toContain("exa_mcp"); - expect(providers).toContain("mistral_websearch"); - expect(providers).not.toContain("duckduckgo"); - }); - - it("includes duckduckgo when mistral key is missing", () => { - const providers = getAllQueryProviders({}); - expect(providers).toContain("duckduckgo"); - }); -});