From cdbd48b443793a7898031e23f94723dd5b06a1df Mon Sep 17 00:00:00 2001 From: Zachary Roth <100426704+zacharyr0th@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:52:01 -0700 Subject: [PATCH] feat: add evidence workflow contracts --- Makefile | 12 +- README.md | 28 +- bench/uv.lock | 2 +- docs/CHANGELOG.md | 32 ++ docs/adr/0001-evidence-acquisition-engine.md | 64 +++ docs/competitor-tracker-integration.md | 56 ++ docs/context-packs.md | 20 + docs/contracts.md | 86 +++ docs/migrations/6.2.md | 54 ++ docs/release-post-v6.2.md | 14 + docs/surface-contract.md | 4 +- package.json | 13 +- plugin/.claude-plugin/plugin.json | 2 +- plugin/.codex-plugin/plugin.json | 2 +- plugin/README.md | 6 +- pyproject.toml | 4 +- scripts/generate_contract_schemas.py | 66 +++ server.json | 4 +- src/docpull/__init__.py | 42 +- src/docpull/change_events.py | 209 ++++++++ src/docpull/cli.py | 30 ++ src/docpull/context_packs/__init__.py | 11 + src/docpull/context_packs/_legacy_cli.py | 7 +- src/docpull/context_packs/common.py | 331 ++++++++++-- src/docpull/context_packs/policy_pack.py | 354 ++++++++++++ src/docpull/context_packs/product.py | 156 +++++- src/docpull/context_packs/visuals.py | 1 - src/docpull/context_packs/workflow_cli.py | 52 ++ src/docpull/contracts.py | 505 ++++++++++++++++++ src/docpull/contracts_cli.py | 40 ++ src/docpull/document_parse.py | 15 + src/docpull/eval_grade.py | 9 +- src/docpull/evidence.py | 154 ++++++ src/docpull/mcp/server.py | 371 +++++++++++++ src/docpull/pack_tools.py | 352 +++++++++++- src/docpull/project.py | 64 ++- .../schemas/artifact-manifest.v1.schema.json | 99 ++++ src/docpull/schemas/basis.v2.schema.json | 86 +++ .../schemas/change-event.v1.schema.json | 243 +++++++++ .../schemas/citation-map.v1.schema.json | 30 ++ src/docpull/schemas/document.v3.schema.json | 275 ++++++++++ .../intelligence-bundle.v1.schema.json | 428 +++++++++++++++ src/docpull/schemas/pack.v3.schema.json | 44 ++ src/docpull/schemas/provenance.v1.schema.json | 32 ++ src/docpull/schemas/rights.v1.schema.json | 34 ++ .../schemas/run-identity.v1.schema.json | 121 +++++ .../schemas/workflow-request.v1.schema.json | 95 ++++ .../schemas/workflow-result.v1.schema.json | 381 +++++++++++++ src/docpull/surface.py | 65 ++- src/docpull/workflows.py | 262 +++++++++ tests/fixtures/evidence_failure_modes.json | 17 + tests/test_context_packs.py | 7 +- tests/test_workflow_contracts.py | 249 +++++++++ 53 files changed, 5521 insertions(+), 119 deletions(-) create mode 100644 docs/adr/0001-evidence-acquisition-engine.md create mode 100644 docs/competitor-tracker-integration.md create mode 100644 docs/contracts.md create mode 100644 docs/migrations/6.2.md create mode 100644 docs/release-post-v6.2.md create mode 100644 scripts/generate_contract_schemas.py create mode 100644 src/docpull/change_events.py create mode 100644 src/docpull/context_packs/policy_pack.py create mode 100644 src/docpull/context_packs/workflow_cli.py create mode 100644 src/docpull/contracts.py create mode 100644 src/docpull/contracts_cli.py create mode 100644 src/docpull/evidence.py create mode 100644 src/docpull/schemas/artifact-manifest.v1.schema.json create mode 100644 src/docpull/schemas/basis.v2.schema.json create mode 100644 src/docpull/schemas/change-event.v1.schema.json create mode 100644 src/docpull/schemas/citation-map.v1.schema.json create mode 100644 src/docpull/schemas/document.v3.schema.json create mode 100644 src/docpull/schemas/intelligence-bundle.v1.schema.json create mode 100644 src/docpull/schemas/pack.v3.schema.json create mode 100644 src/docpull/schemas/provenance.v1.schema.json create mode 100644 src/docpull/schemas/rights.v1.schema.json create mode 100644 src/docpull/schemas/run-identity.v1.schema.json create mode 100644 src/docpull/schemas/workflow-request.v1.schema.json create mode 100644 src/docpull/schemas/workflow-result.v1.schema.json create mode 100644 src/docpull/workflows.py create mode 100644 tests/fixtures/evidence_failure_modes.json create mode 100644 tests/test_workflow_contracts.py diff --git a/Makefile b/Makefile index f91faf3..2a4bedb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean clean-pyc clean-build clean-test help test test-inventory test-all-local benchmark benchmark-quick benchmark-parallel benchmark-compare benchmark-matrix benchmark-raindrop license-year metrics metrics-check metadata-check metadata-sync lint format release-pr release-publish release-publish-replace-tag release-dispatch +.PHONY: clean clean-pyc clean-build clean-test help test test-inventory test-all-local benchmark benchmark-quick benchmark-parallel benchmark-compare benchmark-matrix benchmark-raindrop contracts-check contracts-sync license-year metrics metrics-check metadata-check metadata-sync lint format release-pr release-publish release-publish-replace-tag release-dispatch PYTHON ?= .venv/bin/python VERSION_ARG := $(if $(VERSION),--version $(VERSION),) @@ -31,6 +31,8 @@ help: @echo "metrics-check - fail if METRICS.md is older than METRICS_MAX_AGE_HOURS" @echo "metadata-check - fail if generated release/plugin metadata is stale" @echo "metadata-sync - refresh generated release/plugin metadata from source" + @echo "contracts-check - fail if bundled contract schemas are stale" + @echo "contracts-sync - regenerate bundled contract schemas" @echo "lint - check style with ruff" @echo "format - format code with ruff" @echo "release-pr - push current release branch and open a protected-main PR" @@ -127,7 +129,13 @@ metadata-check: metadata-sync: $(PYTHON) scripts/sync_release_metadata.py --write -lint: metadata-check +contracts-check: + $(PYTHON) scripts/generate_contract_schemas.py --check + +contracts-sync: + $(PYTHON) scripts/generate_contract_schemas.py --write + +lint: metadata-check contracts-check $(PYTHON) -m ruff check . format: license-year metadata-sync diff --git a/README.md b/README.md index f3b0fc7..5e96060 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ web sources an agent depends on, sync them into cited context packs, diff what changed, and export reproducible context for Cursor, Claude, OpenAI, LlamaIndex, LangChain, MCP clients, and RAG pipelines. +Architecturally, DocPull is the evidence and acquisition engine: it acquires, +versions, cites, hashes, and replays evidence. Downstream products own +scheduling, review, approved claims, legal conclusions, and notifications. See +the [architecture decision](docs/adr/0001-evidence-acquisition-engine.md). + The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`, similar in spirit to code dependency manifests and lockfiles: @@ -170,6 +175,11 @@ docpull standards-pack rfc:9110 -o packs/standard docpull dataset-pack ./metrics.csv -o packs/dataset docpull transcript-pack ./meeting.vtt -o packs/transcript docpull wiki-pack wiki:Web_scraping -o packs/wiki +docpull brand-pack example.com -o packs/brand +docpull product-pack https://example.com/pricing -o packs/product +docpull styleguide-pack example.com -o packs/styleguide +docpull image-pack example.com -o packs/visuals +docpull policy-pack example.com -o packs/policies docpull pack prepare packs/docs --eval-grade docpull pack validate packs/docs --level eval docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl @@ -444,6 +454,11 @@ special handling for common web, documentation, and API surfaces. | Local datasets | `docpull dataset-pack` emits bounded schema, exact row counts where streamable, column, null-count, and sample summaries | | Transcripts | `docpull transcript-pack` emits timestamped segment records from VTT, SRT, text, JSON, or direct transcript URLs | | Wikimedia / Wikipedia | `docpull wiki-pack` emits MediaWiki REST page metadata, license/revision metadata, and section-level records | +| Brand evidence | `docpull brand-pack` emits cited identity, firmographic, social, logo, and color observations | +| Product and pricing | `docpull product-pack` emits plans, currencies, intervals, trials, feature gates, and page-text provenance | +| Styleguide | `docpull styleguide-pack` emits cited CSS variables, colors, fonts, spacing, and component samples | +| Visual assets | `docpull image-pack` emits bounded image candidates and optional validated local assets | +| Policy documents | `docpull policy-pack` discovers policy/security pages and emits effective dates, stable clauses, and textual change candidates without legal conclusions | | Docusaurus / Sphinx / MkDocs | Extracts static article or document regions | | VitePress / VuePress / Astro Starlight | Extracts static content regions | | GitBook / ReadMe.io | Extracts available article or content regions | @@ -496,6 +511,16 @@ Every file-backed run writes `corpus.manifest.json` with stable document IDs, chunk IDs, hashes, output paths, and chunk counts. See [Corpus Manifest](docs/corpus-manifest.md). +Evidence-pack workflows also write `workflow.request.json`, +`workflow.result.json`, and `artifact.manifest.json`. Export their Draft 2020-12 +schemas with `docpull contracts export -o schemas/`. The complete inventory and +compatibility rules are in [Public Contracts](docs/contracts.md). + +For tracker imports, run `docpull pack intelligence-bundle PACK`. The canonical +output is `intelligence.bundle.v1.json`; `company_brain.bundle.json` remains a +compatibility alias. See the +[competitor-tracker integration contract](docs/competitor-tracker-integration.md). + ## Profiles ```bash @@ -612,7 +637,8 @@ part of the package release contract. `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`, `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`, `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`, - `docpull wiki-pack`, + `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`, + `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`, `docpull pack validate`, `docpull pack audit`, `docpull export`, `docpull serve`, `docpull share`, `docpull render`, `docpull auth check`, diff --git a/bench/uv.lock b/bench/uv.lock index 0a60cd6..2a6f631 100644 --- a/bench/uv.lock +++ b/bench/uv.lock @@ -584,7 +584,7 @@ wheels = [ [[package]] name = "docpull" -version = "6.1.0" +version = "6.2.0" source = { editable = "../" } dependencies = [ { name = "aiohttp" }, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 64b7954..9f8a2c5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,38 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [6.2.0] - 2026-07-16 + +### Added +- Define DocPull as the evidence and acquisition engine and publish versioned + `WorkflowRequest`, `WorkflowResult`, `ArtifactManifest`, + `intelligence.bundle.v1`, and `ChangeEvent` contracts with bundled JSON + Schemas. +- Add generic progress, warnings, failures, budget usage, SHA-256 artifact + hashes, pack/run identity, and scheduler-neutral replay settings. +- Promote brand, product, styleguide, image, screenshot, and new policy packs to + public CLI, SDK, and MCP workflows, plus declarative project source types for + brand, product, styleguide, visual, and policy evidence. +- Add policy discovery/classification, effective dates, stable clause hashes, + and clause-level before/after candidates without legal conclusions. +- Add deterministic tracker bundles with source snapshots, document versions, + precise evidence spans, observations, confidence, evidence strength, source + authority, warnings, and change candidates. +- Add idempotent change events separating structural, textual, and semantic + candidates for pricing, positioning, product, security, and policy changes. + +### Changed +- Enrich product pricing extraction with plans, currencies, billing intervals, + trials, feature gating, page-text provenance, and incidental/visual price + exclusion. +- Make JSON-only eval preparation retain the required `PACK_CARD.md` artifact. +- Keep `company_brain.bundle.json` and its SDK builder as compatibility aliases + for `intelligence.bundle.v1`. + +### Compatibility +- Validate old packs remain readable and retain legacy result filenames while + new workflow sidecars and schema fields are additive. + ## [6.1.0] - 2026-07-14 ### Added diff --git a/docs/adr/0001-evidence-acquisition-engine.md b/docs/adr/0001-evidence-acquisition-engine.md new file mode 100644 index 0000000..eeddcef --- /dev/null +++ b/docs/adr/0001-evidence-acquisition-engine.md @@ -0,0 +1,64 @@ +# ADR 0001: DocPull is the evidence and acquisition engine + +- Status: Accepted +- Date: 2026-07-16 +- Release: 6.2.0 + +## Decision + +DocPull owns reproducible acquisition, normalization, evidence identity, rights +and provenance metadata, budget accounting, artifact hashing, replay settings, +and change-candidate production. It is the evidence and acquisition engine. + +DocPull does not own tracker scheduling, organizational review state, approved +claims, legal conclusions, competitive scoring, notifications, or product UI. +Those are downstream responsibilities. A scheduler may invoke DocPull, but no +scheduler is required: every workflow and replay configuration runs locally. + +```mermaid +flowchart LR + S["Public or explicitly authorized sources"] --> D["DocPull acquisition"] + D --> E["Documents, evidence spans, rights, provenance"] + E --> W["WorkflowResult and ArtifactManifest"] + E --> C["ChangeEvent candidates"] + W --> T["Downstream tracker or agent"] + C --> T + T --> R["Human review and approved claims"] +``` + +## Boundary rules + +- Local-first HTTP acquisition remains the default. +- Browser rendering and paid/cloud routes remain off unless explicitly enabled + with their existing trust and budget controls. +- Robots, source policy, rights, cache, lock, and provenance semantics remain + part of acquisition rather than downstream interpretation. +- Machine-derived statements cross the boundary as `observation` or + `candidate`, never as an approved claim. +- Authority tiers describe source relationship only. They are not review or + product-specific approval decisions. +- Policy changes report structural and textual evidence. They do not express a + legal opinion. +- Result contracts contain progress, warnings, failures, budget use, hashes, + artifacts, and replay settings so another repository does not need to infer + run state from logs. + +## Consequences + +DocPull publishes versioned JSON Schemas and maintains compatibility for the +CLI, Python SDK, MCP, existing packs, and `company_brain.bundle.json`. A +downstream tracker can pin a DocPull version and validate at the repository +boundary without importing DocPull's internal modules. + +Schedulers, approval workflows, and notification systems remain replaceable. +This keeps monitoring runnable in a local shell or CI job and prevents hosted +control-plane assumptions from entering the acquisition contract. + +## Rejected alternatives + +- Embedding competitor-specific approval states in pack records would couple + evidence acquisition to one consumer. +- Making browser rendering automatic would violate the local-first and explicit + trust boundary. +- Treating extracted summaries as approved claims would erase the distinction + between evidence and review. diff --git a/docs/competitor-tracker-integration.md b/docs/competitor-tracker-integration.md new file mode 100644 index 0000000..906eb6d --- /dev/null +++ b/docs/competitor-tracker-integration.md @@ -0,0 +1,56 @@ +# Competitor-tracker integration contract + +Pin DocPull `6.2.0` and validate `intelligence.bundle.v1.json` against +`intelligence-bundle.v1.schema.json` at import time. + +## Producer flow + +```bash +docpull sync +docpull pack intelligence-bundle .docpull/runs/ \ + --objective "Track pricing, positioning, product, security, and policy" +``` + +Python: + +```python +from pathlib import Path +from docpull import build_intelligence_bundle + +bundle = build_intelligence_bundle( + Path(".docpull/runs/run_20260716"), + objective="Track competitor changes", + market="Developer tools", +) +``` + +MCP clients call `intelligence_bundle`. Existing integrations may continue to +read `company_brain.bundle.json`; it is an alias containing the v1 fields and a +deprecated compatibility envelope. + +## Import requirements + +1. Reject an unknown `contract_version`. +2. Recompute the canonical bundle core hash and compare `bundle_hash`. +3. Deduplicate by `bundle_id` and change `idempotency_key`. +4. Store pack ID, run ID, source snapshot ID, document ID, and document version. +5. Treat `observations[*].status == "observation"` as unapproved machine output. +6. Verify each evidence span against the referenced document version before + presenting it for review. +7. Keep `change_candidates` in a review queue; do not infer approval from source + authority or confidence. +8. Surface warnings and retain replay configuration for audit/reproduction. + +## Ownership split + +DocPull owns acquisition, evidence spans, source snapshots, rights/provenance, +budget receipts, hashes, and change candidates. Competitor-tracker owns +scheduling, entity resolution across imported packs, reviewer assignments, +approval/rejection, product-specific severity, alert delivery, and UI. + +## Replay and monitoring + +`WorkflowRequest.replay` and `ChangeEvent.replay_configuration` deliberately set +`scheduler` to null. A local cron job, CI workflow, desktop agent, or hosted +scheduler can replay the same configuration. Browser and paid-route flags must +remain false unless the operator explicitly enables and budgets those paths. diff --git a/docs/context-packs.md b/docs/context-packs.md index 0af79ee..49e8b1f 100644 --- a/docs/context-packs.md +++ b/docs/context-packs.md @@ -18,6 +18,11 @@ docpull standards-pack rfc:9110 --output-dir packs/standard docpull dataset-pack ./metrics.csv --output-dir packs/dataset docpull transcript-pack ./meeting.vtt --output-dir packs/transcript docpull wiki-pack wiki:Web_scraping --output-dir packs/wiki +docpull brand-pack example.com --output-dir packs/brand +docpull product-pack https://example.com/pricing --output-dir packs/product +docpull styleguide-pack example.com --output-dir packs/styleguide +docpull image-pack example.com --output-dir packs/visuals +docpull policy-pack example.com --output-dir packs/policies docpull pack prepare packs/example --eval-grade docpull pack validate packs/example --level eval @@ -71,6 +76,21 @@ Remote typed lanes support `--cache --cache-dir .docpull-cache/typed-packs` for repeatable official API/metadata calls. Python SDK users can call the matching `async_build_*_pack` helpers when already inside an event loop. +## Evidence Workflow Protocol + +Brand, product, styleguide, visual/image, screenshot, and policy lanes implement +the common workflow protocol. Their existing result JSON and Markdown remain, +and every run additionally writes: + +- `workflow.request.json` with stable input, policy, budget, and replay fields; +- `workflow.result.json` with progress, warnings, failures, budget usage, + identities, and hashes; +- `artifact.manifest.json` with byte sizes and SHA-256 for emitted artifacts. + +Use `docpull pack intelligence-bundle PACK` to create the deterministic tracker +import. Use `docpull pack diff OLD NEW` to write `change.events.jsonl` alongside +the existing diff artifacts. + ## Release Smoke The full free/local public surface can be checked against real public data with diff --git a/docs/contracts.md b/docs/contracts.md new file mode 100644 index 0000000..0da4f78 --- /dev/null +++ b/docs/contracts.md @@ -0,0 +1,86 @@ +# Public contract inventory + +DocPull 6.2 freezes the existing artifact envelopes and adds transport-neutral +cross-repository contracts. Bundled schemas live in `src/docpull/schemas/` and +are installed with the Python package. + +Use `docpull contracts list`, `docpull contracts show NAME`, or +`docpull contracts export -o schemas/` to inspect them. + +## Frozen compatibility envelopes + +| Contract | Version | Canonical implementation | Compatibility rule | +|---|---:|---|---| +| Pack metadata | existing v1/v3 | `*.pack.json`, `pack.v3.schema.json` | Readers continue accepting old pack metadata and unknown additive fields. | +| Document record | v3 | `DocumentRecord`, `document.v3.schema.json` | Existing field names, IDs, citations, rights, route, and chunk fields remain readable. | +| Run identity | v1 | `RunIdentity`, `run-identity.v1.schema.json` | Fingerprints remain deterministic and secret-free. | +| Citation map | v1 | `citations.json`, `citation-map.v1.schema.json` | `citation_id`, URL, title, and record citations remain stable; authority and versions are additive. | +| Rights | v1 envelope | `rights.manifest.json`, `rights.v1.schema.json` | Unknown remains the conservative default. | +| Provenance | v1 envelope | `provenance.graph.json`, `provenance.v1.schema.json` | Additive nodes and edges are allowed. | +| Basis | v2 | `basis.ndjson`, `basis.v2.schema.json` | Legacy basis rows normalize to v2. | +| Company Brain | compatibility alias | `company_brain.bundle.json` | Alias remains written and readable; canonical contract is `intelligence.bundle.v1`. | + +The freeze means existing required fields are not renamed or removed during the +6.x line. Additive fields and sidecars are allowed. Existing CLI commands and +concrete SDK builders retain their result payloads while also writing the new +generic contracts. + +## Cross-repository contracts + +### `workflow.request.v1` + +A scheduler-neutral invocation containing a stable request ID, workflow name, +input, output location, options, source policy, budget, and replay settings. +The request ID excludes ephemeral timestamps. + +### `workflow.result.v1` + +The common result for brand, product, styleguide, visual/image, screenshot, and +policy workflows. It contains: + +- pack and run identities; +- lifecycle progress events; +- structured warnings and failures; +- budget limit, estimated/actual spend, HTTP/cache counts, browser time, and + blocked actions; +- SHA-256 request, manifest, legacy-result, and pack hashes; +- replay configuration and compatibility artifact paths. + +### `artifact.manifest.v1` + +A sorted list of named artifacts with relative path, role, media type, bytes, +and SHA-256. `aggregate_sha256` hashes the canonical sorted entry list. The +manifest does not hash itself or `workflow.result.json`, avoiding circular +identity. + +### `intelligence.bundle.v1` + +The supported tracker-import contract. It contains pack/run identity, source +snapshots, document versions, precise observations, evidence strength, +confidence, source authority, warnings, and before/after change candidates. +The `bundle_hash` covers the canonical v1 core without the self-referential +bundle ID/hash or legacy Company Brain envelope. + +### `change.event.v1` + +One idempotent event per changed URL. It carries old/new document IDs and +hashes, precise old/new evidence, separate structural and textual changes, and +semantic candidates classified as pricing, positioning, product, security, +policy, or other. Classifications are candidates requiring review. + +## Evidence spans + +An evidence span contains the citation and record citation IDs, document ID, +content-hash document version, URL, zero-based `char_start`/`char_end`, exact +text, and exact-text SHA-256. Consumers must verify both the document version +and exact text before promoting an observation. + +## Compatibility policy + +- JSON consumers must ignore unknown fields. +- DocPull readers continue to accept old packs. +- Existing legacy result filenames remain present. +- New generic files are additive: `workflow.request.json`, + `workflow.result.json`, and `artifact.manifest.json`. +- `company_brain.bundle.json` remains a deterministic compatibility alias. +- Breaking changes require a new contract version and migration notes. diff --git a/docs/migrations/6.2.md b/docs/migrations/6.2.md new file mode 100644 index 0000000..16eb18a --- /dev/null +++ b/docs/migrations/6.2.md @@ -0,0 +1,54 @@ +# Migrating to DocPull 6.2 + +No existing pack migration is required. Old packs, concrete SDK builders, and +legacy result filenames remain readable. + +## Adopt the common workflow protocol + +Existing code may keep calling `build_brand_pack`, `build_product_pack`, +`build_styleguide_pack`, `build_image_pack`, or `capture_screenshot_pack`. +Each now writes generic request/result/manifest sidecars. + +New integrations should construct `WorkflowRequest` with +`create_workflow_request` and call `run_workflow` (or `async_run_workflow`). MCP +clients may call `workflow_run` or a dedicated pack tool. + +## Project configuration + +`docpull.yaml` accepts these additional source types: + +```yaml +sources: + - name: acme-brand + url: https://acme.example + type: brand + - name: acme-products + url: https://acme.example/pricing + type: product + - name: acme-style + url: https://acme.example + type: styleguide + - name: acme-visuals + url: https://acme.example + type: visual + - name: acme-policies + url: https://acme.example + type: policy +``` + +`sync`, `history`, `review`, and `diff` use the same project run/index paths for +these sources. Declarative visual sync remains browser-free; screenshot capture +is a separate explicitly trusted workflow. + +## Tracker bundle rename + +Prefer `intelligence.bundle.v1.json` and `build_intelligence_bundle`. +`company_brain.bundle.json` and `build_company_brain_bundle` remain compatibility +aliases. Replace any use of legacy `source_supported_claims` as approval state +with canonical `observations` and a downstream review step. + +## Change events + +Pack and project diffs now write `change.events.jsonl`. Use event ID or +idempotency key for deduplication. Semantic classifications are review +candidates, not conclusions. diff --git a/docs/release-post-v6.2.md b/docs/release-post-v6.2.md new file mode 100644 index 0000000..2e27ce2 --- /dev/null +++ b/docs/release-post-v6.2.md @@ -0,0 +1,14 @@ +# DocPull 6.2: evidence workflows and tracker contracts + +DocPull 6.2 formalizes the project as a local-first evidence and acquisition +engine. Brand, product, styleguide, visual, screenshot, and policy lanes now +share a versioned workflow request/result protocol with artifact hashes, budget +receipts, progress, warnings, failures, and replay configuration. + +The release adds deterministic `intelligence.bundle.v1` imports and idempotent +`change.event.v1` records for downstream trackers. Machine output is represented +as observations and change candidates with precise evidence spans; approval, +legal conclusions, scheduling, and notifications remain downstream concerns. + +All cross-repository contracts ship as Draft 2020-12 JSON Schemas. Existing +packs and `company_brain.bundle.json` remain readable. diff --git a/docs/surface-contract.md b/docs/surface-contract.md index 55a6bd2..d1fa0ae 100644 --- a/docs/surface-contract.md +++ b/docs/surface-contract.md @@ -48,9 +48,11 @@ DocPull targets capability alignment, not 1:1 flag parity. MCP should not mirror | Refresh/score/diff/audit context packs | `docpull refresh`, `docpull pack score`, `docpull pack sources`, `docpull pack diff`, `docpull pack audit` | Pack helper modules, `refresh_pack`, `audit_pack` | `refresh_pack`, `pack_score`, `pack_diff`, `audit_pack` | Core-aligned | | Build pack citations/entities/search/briefs | `docpull pack citations`, `docpull pack entities`, `docpull pack search`, `docpull pack brief` | `build_citation_map`, `extract_pack_entities`, `search_pack`, `build_research_brief` | `pack_citations`, `pack_entities`, `pack_search`, `pack_brief` | Core-aligned | | Prepare full pack intelligence bundle | `docpull pack prepare` | `prepare_pack` in `docpull.pack_tools` | `pack_prepare` | Core-aligned | +| Evidence-pack workflow protocol | `brand-pack`, `product-pack`, `styleguide-pack`, `image-pack`, `screenshot-pack`, `policy-pack` | concrete `build_*_pack` builders plus `WorkflowRequest`, `run_workflow`, `async_run_workflow` | `workflow_run` plus dedicated pack tools | Core-aligned | +| Tracker import bundle | `docpull pack intelligence-bundle` (`company-brain` alias) | `build_intelligence_bundle` (`build_company_brain_bundle` alias) | `intelligence_bundle` | Core-aligned | | Context CI and eval-grade context | `docpull ci`, `docpull pack prepare --eval-grade`, `docpull pack validate --level eval` | `run_context_ci`, `validate_pack_contract`, pack preparation helpers | Not exposed yet | Core-aligned | | Build/query local source graphs | `docpull graph build`, `docpull graph status`, `docpull graph query`, `docpull graph neighbors`, `docpull graph refresh` | `build_graph`, `load_graph`, `graph_status`, `query_graph`, `graph_neighbors`, `refresh_graph` | `graph_build`, `graph_status`, `graph_query`, `graph_neighbors`, `graph_refresh` | Core-aligned | -| Local document/API/feed/typed ingestion | `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`, `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`, `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`, `docpull wiki-pack` | `parse_documents`, `parse_one_document`, `build_openapi_pack`, `build_feed_pack`, typed `build_*_pack` and `async_build_*_pack` helpers | Not exposed yet | Adapted | +| Local document/API/feed/typed ingestion | `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`, `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`, `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`, `docpull wiki-pack` | `parse_documents`, `parse_one_document`, `build_openapi_pack`, `build_feed_pack`, typed `build_*_pack` and `async_build_*_pack` helpers | Evidence workflow lanes are exposed; other typed lanes remain SDK/CLI adapted | Adapted | | Source policy files | `docpull policy validate`, `docpull policy explain`, and policy-aware core workflows | `PolicyConfig` in `docpull.policy` | `validate_policy` over the same typed config | Core-aligned | | Budget/accounting policy | `--budget`, `--explain-route`, `run.accounting.json`, policy `budget.maximum_paid_cost_usd` | `BudgetConfig`, accounting helpers, `PolicyConfig.budget` | `budget` on explicit cloud rendering tools | Core-aligned | | Exports, local pack server, and report sharing | `docpull export` for JSONL, Sheets CSV/TSV, n8n JSON, Vercel AI JSON, CrewAI JSON, warehouse NDJSON, Parquet, and agent references; `docpull serve`; `docpull share` for Markdown/HTML report URLs | `export_pack`, `create_pack_app`, `create_report_server`, `render_report_document`, `load_pack` | `export_pack`, `serve_pack_status` | Adapted | diff --git a/package.json b/package.json index 92f5327..08b8c84 100644 --- a/package.json +++ b/package.json @@ -2,10 +2,21 @@ "name": "docpull", "private": true, "packageManager": "bun@1.3.11", + "scripts": { + "build": "uv run --locked --with-requirements requirements-release.txt python scripts/build_release.py --verify-reproducible --clean", + "test": "uv run --locked python -m pytest -q && bun run --cwd mcp test", + "lint": "uv run --locked python scripts/sync_release_metadata.py --check && uv run --locked python scripts/generate_contract_schemas.py --check && uv run --locked python -m ruff check . && uv run --locked python -m ruff format --check .", + "typecheck": "uv run --locked python -m mypy src && bun run --cwd mcp typecheck", + "validate": "bun run lint && bun run typecheck && bun run test", + "validate:full": "bun run validate && bun run build" + }, "overrides": { "postcss": "^8.5.10" }, "workspaces": [ "mcp" - ] + ], + "engines": { + "node": "24.x" + } } diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index ef0d5ac..e99afd1 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "docpull", - "version": "6.1.0", + "version": "6.2.0", "description": "Pull public web sources into Claude Code. Indexes static and server-rendered sites as local Markdown with conditional-GET caching, then exposes them as MCP tools. Local, browser-free, no API keys.", "author": { "name": "Raintree Technology", diff --git a/plugin/.codex-plugin/plugin.json b/plugin/.codex-plugin/plugin.json index 0fe25c5..a8c7c71 100644 --- a/plugin/.codex-plugin/plugin.json +++ b/plugin/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "id": "docpull", "name": "docpull", - "version": "6.1.0", + "version": "6.2.0", "description": "Pull public web sources into Codex as local, searchable Markdown through docpull's MCP server.", "skills": "./skills/", "mcpServers": "./.mcp.json", diff --git a/plugin/README.md b/plugin/README.md index 11d6eea..e64bd40 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -9,9 +9,9 @@ for the boundary between the plugin's MCP tools and the broader CLI/SDK. ## What you get -- **MCP server** (26 tools): +- **MCP server** (34 tools): - Read: `fetch_url`, `list_sources`, `list_indexed`, `grep_docs`, `read_doc`, `pack_score`, `pack_diff`, `pack_citations`, `pack_entities`, `pack_search`, `pack_brief`, `graph_status`, `graph_query`, `graph_neighbors`, `validate_policy`, `serve_pack_status` - - Write: `render_url`, `ensure_docs`, `refresh_pack`, `audit_pack`, `pack_prepare`, `graph_build`, `graph_refresh`, `export_pack`, `add_source`, `remove_source` + - Write: `render_url`, `ensure_docs`, `workflow_run`, `brand_pack`, `product_pack`, `styleguide_pack`, `image_pack`, `screenshot_pack`, `policy_pack`, `intelligence_bundle`, `refresh_pack`, `audit_pack`, `pack_prepare`, `graph_build`, `graph_refresh`, `export_pack`, `add_source`, `remove_source` - All read tools advertise `readOnlyHint` so hosts that auto-approve safe tools won't prompt for them. - **Claude Code slash commands**: @@ -31,7 +31,7 @@ MCP server is available: ```bash pip install 'docpull[mcp]' # or: pipx install 'docpull[mcp]' # uv tool install 'docpull[mcp]' -docpull --version # should print 6.1.0 or newer +docpull --version # should print 6.2.0 or newer docpull mcp --help # confirm the MCP subcommand is wired ``` diff --git a/pyproject.toml b/pyproject.toml index 246b313..78d33d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "docpull" -version = "6.1.0" +version = "6.2.0" dynamic = [] description = "Declare, sync, diff, and lock context dependencies for AI agents" readme = {file = "README.md", content-type = "text/markdown"} @@ -202,7 +202,7 @@ where = ["src"] include = ["docpull*"] [tool.setuptools.package-data] -docpull = ["py.typed", "fixtures/*.json"] +docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"] [tool.ruff] line-length = 110 diff --git a/scripts/generate_contract_schemas.py b/scripts/generate_contract_schemas.py new file mode 100644 index 0000000..704332f --- /dev/null +++ b/scripts/generate_contract_schemas.py @@ -0,0 +1,66 @@ +"""Generate or verify the JSON Schemas shipped for cross-repository contracts.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from docpull.contracts import CONTRACT_MODELS # noqa: E402 + +SCHEMA_DIR = ROOT / "src" / "docpull" / "schemas" + + +def expected_schemas() -> dict[Path, str]: + expected: dict[Path, str] = {} + for filename, model in sorted(CONTRACT_MODELS.items()): + payload = model.model_json_schema(mode="serialization") + payload["$schema"] = "https://json-schema.org/draft/2020-12/schema" + payload["$id"] = f"https://docpull.dev/schemas/{filename}" + expected[SCHEMA_DIR / filename] = json.dumps(payload, indent=2, sort_keys=True) + "\n" + return expected + + +def check() -> int: + drifted = [ + path + for path, expected in expected_schemas().items() + if not path.exists() or path.read_text(encoding="utf-8") != expected + ] + unexpected = sorted(path for path in SCHEMA_DIR.glob("*.schema.json") if path not in expected_schemas()) + if drifted or unexpected: + print("Contract schemas are stale:", file=sys.stderr) + for path in [*drifted, *unexpected]: + print(f" - {path.relative_to(ROOT)}", file=sys.stderr) + return 1 + print(f"Contract schemas are synchronized ({len(CONTRACT_MODELS)} schemas).") + return 0 + + +def write() -> int: + SCHEMA_DIR.mkdir(parents=True, exist_ok=True) + expected = expected_schemas() + for path, content in expected.items(): + path.write_text(content, encoding="utf-8") + for path in SCHEMA_DIR.glob("*.schema.json"): + if path not in expected: + path.unlink() + print(f"Wrote {len(expected)} contract schemas to {SCHEMA_DIR.relative_to(ROOT)}.") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--check", action="store_true") + mode.add_argument("--write", action="store_true") + args = parser.parse_args(argv) + return check() if args.check else write() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server.json b/server.json index 135ed58..98571c6 100644 --- a/server.json +++ b/server.json @@ -8,12 +8,12 @@ "source": "github" }, "websiteUrl": "https://github.com/raintree-technology/docpull", - "version": "6.1.0", + "version": "6.2.0", "packages": [ { "registryType": "pypi", "identifier": "docpull", - "version": "6.1.0", + "version": "6.2.0", "transport": { "type": "stdio" } diff --git a/src/docpull/__init__.py b/src/docpull/__init__.py index c5bea64..027bcfd 100644 --- a/src/docpull/__init__.py +++ b/src/docpull/__init__.py @@ -14,7 +14,7 @@ print(event) """ -__version__ = "6.1.0" +__version__ = "6.2.0" from .cache import CacheManager, StreamingDeduplicator from .context_ci import CIThresholds, ContextCIError, run_context_ci @@ -26,15 +26,32 @@ async_build_standards_pack, async_build_transcript_pack, async_build_wiki_pack, + build_brand_pack, build_dataset_pack, build_feed_pack, + build_image_pack, build_openapi_pack, build_package_pack, build_paper_pack, + build_policy_pack, + build_product_pack, build_repo_pack, build_standards_pack, + build_styleguide_pack, build_transcript_pack, build_wiki_pack, + capture_screenshot_pack, +) +from .contracts import ( + ArtifactManifest, + ChangeEvent, + EvidenceSpan, + IntelligenceBundle, + SourceAuthority, + WorkflowRequest, + WorkflowResult, + bundled_schema_path, + write_contract_schemas, ) from .conversion.chunking import Chunk, TokenCounter, chunk_markdown from .core.fetcher import Fetcher, fetch_blocking, fetch_one @@ -69,6 +86,8 @@ from .pack_reader import LocalPack, PackReadError, PackSource, load_pack from .pack_tools import ( build_citation_map, + build_company_brain_bundle, + build_intelligence_bundle, build_research_brief, diff_packs, extract_pack_entities, @@ -100,6 +119,7 @@ from .server import PackASGIApp, PackServerError, create_pack_app from .share import ReportHTTPServer, ShareError, create_report_server, render_report_document from .surface import PUBLIC_SDK_EXPORTS +from .workflows import async_run_workflow, create_workflow_request, run_workflow __all__ = [ "__version__", @@ -110,20 +130,38 @@ "async_build_standards_pack", "async_build_transcript_pack", "async_build_wiki_pack", + "async_run_workflow", + "ArtifactManifest", + "ChangeEvent", + "EvidenceSpan", + "IntelligenceBundle", + "SourceAuthority", + "WorkflowRequest", + "WorkflowResult", + "bundled_schema_path", + "write_contract_schemas", + "create_workflow_request", + "run_workflow", "Fetcher", "fetch_blocking", "fetch_one", "refresh_pack", "audit_pack", "build_dataset_pack", + "build_brand_pack", "build_feed_pack", "build_openapi_pack", "build_package_pack", "build_paper_pack", + "build_policy_pack", + "build_product_pack", "build_repo_pack", "build_standards_pack", + "build_styleguide_pack", "build_transcript_pack", "build_wiki_pack", + "build_image_pack", + "capture_screenshot_pack", "parse_documents", "parse_one_document", "DocumentParseError", @@ -132,6 +170,8 @@ "score_pack_sources", "diff_packs", "build_citation_map", + "build_intelligence_bundle", + "build_company_brain_bundle", "extract_pack_entities", "search_pack", "build_research_brief", diff --git a/src/docpull/change_events.py b/src/docpull/change_events.py new file mode 100644 index 0000000..aefd2fc --- /dev/null +++ b/src/docpull/change_events.py @@ -0,0 +1,209 @@ +"""Versioned, scheduler-neutral change-event generation.""" + +from __future__ import annotations + +import difflib +import json +from pathlib import Path +from typing import Any, Literal + +from .contracts import ChangeEvent, ReplayConfiguration, canonical_sha256, stable_id +from .evidence import evidence_span + +ChangeClassification = Literal["pricing", "positioning", "product", "security", "policy", "other"] + +_CLASSIFICATION_TERMS: dict[ChangeClassification, tuple[str, ...]] = { + "pricing": ("price", "pricing", "plan", "$", "usd", "billing", "trial"), + "positioning": ("positioning", "mission", "leader", "best", "platform for", "built for"), + "product": ("feature", "product", "launch", "available", "integration", "api"), + "security": ("security", "encryption", "soc 2", "iso 27001", "vulnerability", "breach"), + "policy": ("terms", "privacy", "cookie", "dpa", "subprocessor", "refund", "effective date"), +} + + +def build_change_events( + old_records: dict[str, list[dict[str, Any]]], + new_records: dict[str, list[dict[str, Any]]], + *, + workflow: str = "pack-diff", +) -> list[dict[str, Any]]: + """Build idempotent events from already acquired document versions.""" + + events: list[dict[str, Any]] = [] + for url in sorted(set(old_records) | set(new_records)): + old_items = old_records.get(url, []) + new_items = new_records.get(url, []) + old_hashes = sorted(_hashes(old_items)) + new_hashes = sorted(_hashes(new_items)) + if old_hashes == new_hashes and _titles(old_items) == _titles(new_items): + continue + old_primary = old_items[0] if old_items else None + new_primary = new_items[0] if new_items else None + old_text = _combined_content(old_items) + new_text = _combined_content(new_items) + classifications = _classifications(old_text, new_text, url) + structural = _structural_changes(old_items, new_items) + textual = _textual_changes(old_text, new_text) + old_evidence = _record_evidence(old_primary, old_text) if old_primary else [] + new_evidence = _record_evidence(new_primary, new_text) if new_primary else [] + identity = { + "workflow": workflow, + "url": url, + "old_hashes": old_hashes, + "new_hashes": new_hashes, + } + idempotency_key = canonical_sha256(identity) + semantic_candidates = [ + { + "classification": item, + "status": "candidate", + "confidence": _classification_confidence(item, old_text, new_text, url), + "requires_review": True, + } + for item in classifications + ] + event = ChangeEvent( + event_id=stable_id("change_event", identity), + idempotency_key=idempotency_key, + workflow=workflow, + url=url, + old_document_id=_string(old_primary, "document_id"), + new_document_id=_string(new_primary, "document_id"), + old_hash=canonical_sha256(old_hashes) if old_hashes else None, + new_hash=canonical_sha256(new_hashes) if new_hashes else None, + old_evidence=old_evidence, + new_evidence=new_evidence, + structural_changes=structural, + textual_changes=textual, + semantic_candidates=semantic_candidates, + classifications=classifications, + replay_configuration=ReplayConfiguration( + configuration={ + "workflow": workflow, + "old_hashes": old_hashes, + "new_hashes": new_hashes, + } + ), + ) + events.append(event.model_dump(mode="json", exclude_none=True)) + return events + + +def write_change_events(path: Path, events: list[dict[str, Any]]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n" for event in events), + encoding="utf-8", + ) + return path + + +def _record_evidence(record: dict[str, Any], content: str) -> list[Any]: + if not content: + return [] + excerpt = content[: min(600, len(content))] + citation_id = str(record.get("source_citation_id") or "S0") + record_citation_id = record.get("record_citation_id") + return [ + evidence_span( + url=str(record.get("url") or ""), + content=content, + exact_text=excerpt, + citation_id=citation_id, + record_citation_id=str(record_citation_id) if record_citation_id else None, + ) + ] + + +def _structural_changes( + old_items: list[dict[str, Any]], + new_items: list[dict[str, Any]], +) -> list[dict[str, Any]]: + changes: list[dict[str, Any]] = [] + if not old_items: + changes.append({"type": "document_added", "new_record_count": len(new_items)}) + elif not new_items: + changes.append({"type": "document_removed", "old_record_count": len(old_items)}) + if len(old_items) != len(new_items): + changes.append( + { + "type": "record_count_changed", + "before": len(old_items), + "after": len(new_items), + } + ) + if _titles(old_items) != _titles(new_items): + changes.append({"type": "title_changed", "before": _titles(old_items), "after": _titles(new_items)}) + return changes + + +def _textual_changes(old_text: str, new_text: str) -> list[dict[str, Any]]: + old_lines = old_text.splitlines() + new_lines = new_text.splitlines() + matcher = difflib.SequenceMatcher(a=old_lines, b=new_lines, autojunk=False) + changes: list[dict[str, Any]] = [] + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag == "equal": + continue + before = "\n".join(old_lines[i1:i2])[:1000] + after = "\n".join(new_lines[j1:j2])[:1000] + changes.append( + { + "type": tag, + "old_line_start": i1, + "old_line_end": i2, + "new_line_start": j1, + "new_line_end": j2, + "before": before, + "after": after, + "before_sha256": canonical_sha256(before), + "after_sha256": canonical_sha256(after), + } + ) + if len(changes) >= 100: + break + return changes + + +def _classifications(old_text: str, new_text: str, url: str) -> list[ChangeClassification]: + haystack = f"{url}\n{old_text}\n{new_text}".lower() + output = [ + classification + for classification, terms in _CLASSIFICATION_TERMS.items() + if any(term in haystack for term in terms) + ] + return output or ["other"] + + +def _classification_confidence( + classification: ChangeClassification, + old_text: str, + new_text: str, + url: str, +) -> float: + terms = _CLASSIFICATION_TERMS.get(classification, ()) + haystack = f"{url}\n{old_text}\n{new_text}".lower() + hits = sum(1 for term in terms if term in haystack) + return min(0.95, 0.5 + hits * 0.1) if terms else 0.4 + + +def _combined_content(items: list[dict[str, Any]]) -> str: + return "\n\n".join(str(item.get("content") or "") for item in items) + + +def _hashes(items: list[dict[str, Any]]) -> set[str]: + return {str(item.get("content_hash")) for item in items if item.get("content_hash")} + + +def _titles(items: list[dict[str, Any]]) -> list[str]: + return sorted({str(item.get("title") or "") for item in items}) + + +def _string(item: dict[str, Any] | None, key: str) -> str | None: + if not item or item.get(key) is None: + return None + value = str(item[key]).strip() + return value or None + + +__all__ = ["build_change_events", "write_change_events"] diff --git a/src/docpull/cli.py b/src/docpull/cli.py index e64497e..75c6c77 100644 --- a/src/docpull/cli.py +++ b/src/docpull/cli.py @@ -1489,6 +1489,10 @@ def main(argv: list[str] | None = None) -> int: from .pack_tools import run_pack_cli return run_pack_cli(raw_argv[1:]) + if raw_argv and raw_argv[0] == "contracts": + from .contracts_cli import run_contracts_cli + + return run_contracts_cli(raw_argv[1:]) if raw_argv and raw_argv[0] == "graph": from .graph import run_graph_cli @@ -1525,6 +1529,32 @@ def main(argv: list[str] | None = None) -> int: from .monitor import run_monitor_cli return run_monitor_cli(raw_argv[1:]) + if raw_argv and raw_argv[0] in { + "brand-pack", + "product-pack", + "styleguide-pack", + "image-pack", + "screenshot-pack", + "policy-pack", + }: + from .context_packs.workflow_cli import ( + run_brand_pack_cli, + run_image_pack_cli, + run_policy_pack_cli, + run_product_pack_cli, + run_screenshot_pack_cli, + run_styleguide_pack_cli, + ) + + workflow_runners = { + "brand-pack": run_brand_pack_cli, + "product-pack": run_product_pack_cli, + "styleguide-pack": run_styleguide_pack_cli, + "image-pack": run_image_pack_cli, + "screenshot-pack": run_screenshot_pack_cli, + "policy-pack": run_policy_pack_cli, + } + return workflow_runners[raw_argv[0]](raw_argv[1:]) if raw_argv and raw_argv[0] == "openapi-pack": from .context_packs.cli import run_openapi_pack_cli diff --git a/src/docpull/context_packs/__init__.py b/src/docpull/context_packs/__init__.py index f184e2b..6fc07f8 100644 --- a/src/docpull/context_packs/__init__.py +++ b/src/docpull/context_packs/__init__.py @@ -3,14 +3,19 @@ from __future__ import annotations from ..surface import PUBLIC_CONTEXT_PACK_EXPORTS +from .brand import build_brand_pack from .dataset import async_build_dataset_pack, build_dataset_pack from .feed import build_feed_pack from .openapi import build_openapi_pack from .package import async_build_package_pack, build_package_pack from .paper import async_build_paper_pack, build_paper_pack +from .policy_pack import build_policy_pack +from .product import build_product_pack from .repo import async_build_repo_pack, build_repo_pack from .standards import async_build_standards_pack, build_standards_pack +from .styleguide import build_styleguide_pack from .transcript import async_build_transcript_pack, build_transcript_pack +from .visuals import build_image_pack, capture_screenshot_pack from .wiki import async_build_wiki_pack, build_wiki_pack __all__ = [ @@ -21,15 +26,21 @@ "async_build_standards_pack", "async_build_transcript_pack", "async_build_wiki_pack", + "build_brand_pack", "build_dataset_pack", "build_feed_pack", "build_openapi_pack", "build_package_pack", "build_paper_pack", + "build_policy_pack", + "build_product_pack", "build_repo_pack", "build_standards_pack", + "build_styleguide_pack", "build_transcript_pack", "build_wiki_pack", + "build_image_pack", + "capture_screenshot_pack", ] assert tuple(__all__) == PUBLIC_CONTEXT_PACK_EXPORTS diff --git a/src/docpull/context_packs/_legacy_cli.py b/src/docpull/context_packs/_legacy_cli.py index 8d70508..d4f4e4b 100644 --- a/src/docpull/context_packs/_legacy_cli.py +++ b/src/docpull/context_packs/_legacy_cli.py @@ -1,8 +1,7 @@ -"""Private CLI adapters for legacy context-pack experiments. +"""Compatibility import path for the original context-pack CLI adapters. -These functions are intentionally not dispatched from ``docpull``. They remain -only for internal compatibility tests and private workflows that still exercise -the old experimental pack builders. +The commands are public again through :mod:`docpull.context_packs.workflow_cli`; +this module remains importable so pre-6.2 integrations do not break. """ from __future__ import annotations diff --git a/src/docpull/context_packs/common.py b/src/docpull/context_packs/common.py index e8c1a6e..346ba5d 100644 --- a/src/docpull/context_packs/common.py +++ b/src/docpull/context_packs/common.py @@ -15,12 +15,30 @@ from bs4 import BeautifulSoup from ..accounting import RunAccounting, default_route_steps, write_run_accounting +from ..contracts import ( + ArtifactManifest, + BudgetUsage, + HashDigest, + ReplayConfiguration, + WorkflowFailure, + WorkflowProgressEvent, + WorkflowResult, + WorkflowWarning, + artifact_entries, + build_workflow_request, + canonical_sha256, + file_sha256, + new_progress_event, + stable_id, +) from ..core.fetcher import Fetcher +from ..evidence import classify_source_authority, document_identity, evidence_span_payload from ..http.client import AsyncHttpClient from ..http.rate_limiter import PerHostRateLimiter from ..models.config import DocpullConfig, ProfileName from ..models.document import DocumentRecord from ..models.run import RunIdentity +from ..output_contract import OUTPUT_CONTRACT_SCHEMA_VERSION, write_raw_contract_sidecars from ..pack_tools import build_citation_map from ..policy import PolicyConfig, policy_domain_matches from ..security.download_policy import SafeDownloadPolicy, UnsafeDownloadError, content_type_base @@ -107,9 +125,14 @@ class EvidenceRef: citation_id: str url: str + record_citation_id: str | None = None title: str | None = None field: str | None = None excerpt: str | None = None + evidence_span: dict[str, Any] | None = None + source_authority: dict[str, Any] | None = None + evidence_strength: str = "strong" + confidence: float = 1.0 def to_dict(self) -> dict[str, Any]: payload: dict[str, Any] = { @@ -118,10 +141,18 @@ def to_dict(self) -> dict[str, Any]: } if self.title: payload["title"] = self.title + if self.record_citation_id: + payload["record_citation_id"] = self.record_citation_id if self.field: payload["field"] = self.field if self.excerpt: payload["excerpt"] = self.excerpt[:500] + if self.evidence_span: + payload["evidence_span"] = self.evidence_span + if self.source_authority: + payload["source_authority"] = self.source_authority + payload["evidence_strength"] = self.evidence_strength + payload["confidence"] = self.confidence return payload @@ -181,12 +212,40 @@ class ContextPackRun: errors: list[dict[str, Any]] = field(default_factory=list) http_request_count: int = 0 cache_hit_count: int = 0 + progress_events: list[dict[str, Any]] = field(default_factory=list) + + def __post_init__(self) -> None: + self.progress("run", "started", message=f"Started {self.workflow}", timestamp=self.started_at) def warn(self, code: str, message: str, **metadata: Any) -> None: payload: dict[str, Any] = {"code": code, "message": message} if metadata: payload["metadata"] = jsonable(metadata) self.warnings.append(payload) + self.progress("workflow", "warning", message=message, metadata={"code": code, **metadata}) + + def progress( + self, + phase: str, + status: Literal["started", "progress", "completed", "warning", "failed"], + *, + message: str | None = None, + current: int | None = None, + total: int | None = None, + metadata: dict[str, Any] | None = None, + timestamp: str | None = None, + ) -> None: + self.progress_events.append( + new_progress_event( + phase=phase, + status=status, + timestamp=timestamp, + message=message, + current=current, + total=total, + metadata=metadata, + ) + ) class ContextAssetDownloadPolicy(SafeDownloadPolicy): @@ -381,11 +440,12 @@ async def fetch_pages( if not selected: return [] + run.progress("acquisition", "started", total=len(selected), message="Acquiring source pages") config = DocpullConfig(url=selected[0], profile=ProfileName.CUSTOM) config.network.log_retry_warnings = False snapshots: list[PageSnapshot] = [] async with Fetcher(config) as fetcher: - for url in selected: + for index, url in enumerate(selected, start=1): ctx = await fetcher.fetch_one(url, save=False) run.http_request_count += 1 if ctx.error: @@ -408,6 +468,20 @@ async def fetch_pages( source_type=ctx.source_type, ) ) + run.progress( + "acquisition", + "progress", + current=index, + total=len(selected), + message=f"Processed {public_url(url)}", + ) + run.progress( + "acquisition", + "completed", + current=len(selected), + total=len(selected), + message=f"Acquired {len(snapshots)} source pages", + ) return snapshots @@ -437,11 +511,21 @@ def text_excerpt(text: str, needle: str | None = None, *, limit: int = 280) -> s def citation_map_for_pages(pages: list[PageSnapshot]) -> dict[str, Any]: + official_domain = domain_from_input(pages[0].url) if pages else None sources = [ { "citation_id": f"S{index}", "url": page.url, "title": page.title or page.url, + "document_id": document_identity( + page.url, + hashlib.sha256(page.markdown.encode("utf-8")).hexdigest(), + ), + "document_version": hashlib.sha256(page.markdown.encode("utf-8")).hexdigest(), + "source_authority": classify_source_authority( + page.url, + official_domain=official_domain, + ).model_dump(mode="json"), } for index, page in enumerate(pages, start=1) ] @@ -461,12 +545,26 @@ def evidence_for_page( excerpt: str | None = None, ) -> EvidenceRef: index = pages.index(page) + 1 if page in pages else 1 + selected_excerpt = excerpt or text_excerpt(page.markdown) + official_domain = domain_from_input(pages[0].url) if pages else domain_from_input(page.url) return EvidenceRef( citation_id=f"S{index}", + record_citation_id=f"S{index}.1", url=page.url, title=page.title, field=field, - excerpt=excerpt or text_excerpt(page.markdown), + excerpt=selected_excerpt, + evidence_span=evidence_span_payload( + url=page.url, + content=page.markdown, + exact_text=selected_excerpt, + citation_id=f"S{index}", + record_citation_id=f"S{index}.1", + ), + source_authority=classify_source_authority( + page.url, + official_domain=official_domain, + ).model_dump(mode="json"), ) @@ -790,6 +888,9 @@ def write_basic_pack_files( "source_policy": "source_policy.json", "agent_context": "AGENT_CONTEXT.md", "pack_metadata": pack_filename, + "workflow_request": "workflow.request.json", + "workflow_result": "workflow.result.json", + "artifact_manifest": "artifact.manifest.json", } if extra_artifacts: artifacts.update(extra_artifacts) @@ -800,6 +901,7 @@ def write_basic_pack_files( "documents_ndjson": artifact_ref(output_dir, records_path), "corpus_manifest": artifact_ref(output_dir, manifest_path), "sources": artifact_ref(output_dir, sources_path), + "acquisition_routes": "acquisition.routes.json", } ) artifacts["accounting"] = "run.accounting.json" @@ -830,19 +932,177 @@ def write_basic_pack_files( write_json(result_path, result_payload) write_json(pack_path, pack_payload) - write_run_accounting( + accounting = RunAccounting( + budget_limit_usd=run.policy.budget.maximum_paid_cost_usd, + estimated_paid_cost_usd=0.0, + http_request_count=run.http_request_count, + cache_hit_count=run.cache_hit_count, + route_steps=default_route_steps(), + command=run.workflow, + metadata={"input": public_url(run.input_value)}, + ) + accounting_payload = accounting.to_dict() + write_run_accounting(output_dir, accounting) + run.progress("artifacts", "completed", message="Wrote workflow artifacts") + run.progress("run", "completed", message=f"Completed {run.workflow}") + _write_workflow_contract_files( + run=run, + result_payload=result_payload, + source_policy=source_policy, + artifacts=artifacts, + accounting_payload=accounting_payload, + ) + return result_payload + + +def _write_workflow_contract_files( + *, + run: ContextPackRun, + result_payload: dict[str, Any], + source_policy: dict[str, Any], + artifacts: dict[str, str], + accounting_payload: dict[str, Any], +) -> None: + output_dir = run.output_dir.resolve() + replay_raw = result_payload.get("replay_config") + replay = replay_raw if isinstance(replay_raw, dict) else {} + input_raw = result_payload.get("input") + input_payload = input_raw if isinstance(input_raw, dict) else {"value": run.input_value} + browser_enabled = bool(replay.get("render")) or run.workflow == "screenshot-pack" + request = build_workflow_request( + workflow=run.workflow, + input_payload=input_payload, + output_dir=output_dir, + options=replay, + source_policy=source_policy, + budget={"maximum_paid_cost_usd": run.policy.budget.maximum_paid_cost_usd}, + browser_enabled=browser_enabled, + paid_routes_enabled=False, + ) + from ..workflows import current_workflow_request + + request = current_workflow_request() or request + request_path = output_dir / artifacts["workflow_request"] + write_json(request_path, request.model_dump(mode="json", exclude_none=True)) + + entries = artifact_entries( output_dir, - RunAccounting( - budget_limit_usd=run.policy.budget.maximum_paid_cost_usd, - estimated_paid_cost_usd=0.0, - http_request_count=run.http_request_count, - cache_hit_count=run.cache_hit_count, - route_steps=default_route_steps(), - command=run.workflow, - metadata={"input": public_url(run.input_value)}, + artifacts, + excluded={"workflow_result", "artifact_manifest"}, + ) + aggregate = canonical_sha256([entry.model_dump(mode="json") for entry in entries]) + pack_id = stable_id("pack", {"workflow": run.workflow, "aggregate_sha256": aggregate}) + run_id = stable_id( + "run", + {"request_id": request.request_id, "started_at": run.started_at, "pack_id": pack_id}, + ) + manifest = ArtifactManifest( + pack_id=pack_id, + run_id=run_id, + entries=entries, + aggregate_sha256=aggregate, + ) + manifest_path = output_dir / artifacts["artifact_manifest"] + write_json(manifest_path, manifest.model_dump(mode="json", exclude_none=True)) + + warning_models = [ + WorkflowWarning.model_validate( + { + "code": str(item.get("code") or "warning"), + "message": str(item.get("message") or "Workflow warning"), + "metadata": item.get("metadata") if isinstance(item.get("metadata"), dict) else {}, + } + ) + for item in run.warnings + ] + failure_models = [ + WorkflowFailure( + message=str(item.get("error") or item.get("message") or "Acquisition failure"), + source_url=str(item.get("url")) if item.get("url") else None, + metadata={ + str(key): value for key, value in item.items() if key not in {"error", "message", "url"} + }, + ) + for item in run.errors + ] + status: Literal["completed", "completed_with_warnings", "failed", "cancelled"] + if failure_models and not result_payload.get("summary"): + status = "failed" + elif failure_models or warning_models: + status = "completed_with_warnings" + else: + status = "completed" + data = { + key: value + for key, value in result_payload.items() + if key + not in { + "schema_version", + "generated_at", + "workflow", + "provider", + "status", + "input", + "output_dir", + "summary", + "warnings", + "errors", + "replay_config", + "artifacts", + } + } + raw_summary = result_payload.get("summary") + summary: dict[str, Any] = raw_summary if isinstance(raw_summary, dict) else {} + result = WorkflowResult( + request_id=request.request_id, + workflow=run.workflow, + status=status, + started_at=run.started_at, + finished_at=utc_now_iso(), + pack_identity={ + "pack_id": pack_id, + "aggregate_sha256": aggregate, + "workflow": run.workflow, + }, + run_identity={ + "run_id": run_id, + "request_id": request.request_id, + "scheduler": None, + }, + summary=summary, + data=data, + progress_events=[WorkflowProgressEvent.model_validate(item) for item in run.progress_events], + warnings=warning_models, + failures=failure_models, + budget_usage=BudgetUsage( + limit_usd=accounting_payload.get("budget_limit_usd"), + estimated_usd=float(accounting_payload.get("estimated_paid_cost_usd") or 0.0), + actual_usd=accounting_payload.get("actual_paid_cost_usd"), + paid_request_count=int(accounting_payload.get("paid_request_count") or 0), + http_request_count=int(accounting_payload.get("http_request_count") or 0), + cache_hit_count=int(accounting_payload.get("cache_hit_count") or 0), + local_browser_seconds=float(accounting_payload.get("local_browser_seconds") or 0.0), + blocked_actions=list(accounting_payload.get("blocked_actions") or []), ), + hashes={ + "request": HashDigest(digest=file_sha256(request_path)), + "artifact_manifest": HashDigest(digest=file_sha256(manifest_path)), + "legacy_result": HashDigest(digest=canonical_sha256(result_payload)), + "pack": HashDigest(digest=aggregate), + }, + replay_configuration=ReplayConfiguration( + browser_enabled=browser_enabled, + paid_routes_enabled=False, + configuration=replay, + ), + compatibility_artifacts={ + name: path + for name, path in artifacts.items() + if name not in {"workflow_request", "workflow_result", "artifact_manifest"} + }, ) - return result_payload + result_path = output_dir / artifacts["workflow_result"] + write_json(result_path, result.model_dump(mode="json", exclude_none=True)) def write_documents_pack( @@ -870,6 +1130,8 @@ def write_documents_pack( extraction={**page.extraction, "workflow": workflow}, source_type=page.source_type or workflow, run_identity=run_identity, + source_citation_id=f"S{index}", + record_citation_id=f"S{index}.1", ) records.append(record) source_path = source_dir / f"{index:03d}.md" @@ -888,32 +1150,35 @@ def write_documents_pack( encoding="utf-8", ) manifest_path = output_dir / "corpus.manifest.json" - write_json( - manifest_path, - { - "schema_version": CONTEXT_PACK_SCHEMA_VERSION, - "generated_at": utc_now_iso(), - "output_format": "ndjson", - "document_count": len({record.document_id for record in records}), - "record_count": len(records), - "records": [ - { - "document_id": record.document_id, - "url": record.url, - "title": record.title, - "content_hash": record.content_hash, - "source_type": record.source_type, - "output_path": sources[index]["path"] if index < len(sources) else None, - } - for index, record in enumerate(records) - ], - }, - ) + manifest_payload = { + "schema_version": OUTPUT_CONTRACT_SCHEMA_VERSION, + "generated_at": utc_now_iso(), + "output_format": "ndjson", + "document_count": len({record.document_id for record in records}), + "record_count": len(records), + "records": [ + { + "document_id": record.document_id, + "url": record.url, + "title": record.title, + "content_hash": record.content_hash, + "source_type": record.source_type, + "output_path": sources[index]["path"] if index < len(sources) else None, + } + for index, record in enumerate(records) + ], + } + write_json(manifest_path, manifest_payload) sources_path = output_dir / "sources.md" lines = ["# Sources", ""] for source in sources: lines.append(f"- {source['index']}. [{source['title']}]({source['url']})") sources_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + write_raw_contract_sidecars( + output_dir, + manifest_payload=manifest_payload, + output_format="ndjson", + ) return records_path, manifest_path, sources_path diff --git a/src/docpull/context_packs/policy_pack.py b/src/docpull/context_packs/policy_pack.py new file mode 100644 index 0000000..23dac9a --- /dev/null +++ b/src/docpull/context_packs/policy_pack.py @@ -0,0 +1,354 @@ +"""Local policy-document discovery and clause extraction workflow.""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlparse + +from ..contracts import stable_id +from ..policy import PolicyConfig +from .common import ( + CONTEXT_PACK_SCHEMA_VERSION, + ContextPackError, + ContextPackRun, + PageSnapshot, + append_ndjson, + artifact_ref, + domain_from_input, + ensure_policy_for_domain, + evidence_for_page, + extract_links, + fetch_pages_blocking, + homepage_url_for_domain, + public_url, + quote_markdown, + same_policy_domain, + status_from_errors, + write_basic_pack_files, + write_json, +) + +POLICY_WORKFLOW = "policy-pack" +DEFAULT_POLICY_OUTPUT_DIR = Path("packs/policies") +POLICY_DOCUMENT_TYPES = ( + "terms", + "privacy", + "dpa", + "cookies", + "ai_terms", + "subprocessors", + "security", + "refund", +) +_TYPE_TERMS: dict[str, tuple[str, ...]] = { + "terms": ("terms", "terms-of-service", "terms-of-use", "tos"), + "privacy": ("privacy", "privacy-policy"), + "dpa": ("dpa", "data-processing", "data processing addendum"), + "cookies": ("cookie", "cookies"), + "ai_terms": ("ai-terms", "ai terms", "generative-ai", "artificial intelligence terms"), + "subprocessors": ("subprocessor", "sub-processors"), + "security": ("security", "trust-center", "trust center"), + "refund": ("refund", "cancellation", "return-policy"), +} +_EFFECTIVE_DATE_RE = re.compile( + r"(?:effective|last\s+updated|updated|revision\s+date)\s*(?:date)?\s*[:\-]?\s*" + r"(?P(?:[A-Z][a-z]+\s+\d{1,2},?\s+\d{4})|(?:\d{4}-\d{2}-\d{2})|" + r"(?:\d{1,2}/\d{1,2}/\d{4}))", + re.IGNORECASE, +) +_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE) + + +def build_policy_pack( + domain_or_url: str, + *, + output_dir: Path = DEFAULT_POLICY_OUTPUT_DIR, + policy: PolicyConfig | None = None, + max_pages: int = 16, + baseline_pack: Path | None = None, +) -> dict[str, Any]: + """Discover policy pages and emit neutral, clause-level evidence records.""" + + domain = domain_from_input(domain_or_url) + if not domain: + raise ContextPackError("Could not resolve a domain from policy-pack input.") + effective_policy = ensure_policy_for_domain(policy, domain) + run = ContextPackRun( + workflow=POLICY_WORKFLOW, + output_dir=output_dir.resolve(), + policy=effective_policy, + input_value=domain_or_url, + ) + start_url = public_url(domain_or_url if "://" in domain_or_url else homepage_url_for_domain(domain)) + discovery_pages = fetch_pages_blocking([start_url], run=run, max_pages=1) + if not discovery_pages: + raise ContextPackError(f"Could not fetch policy discovery target: {start_url}") + + urls = _policy_urls(discovery_pages[0], domain=domain, max_pages=max_pages) + if _policy_target_hint(discovery_pages[0]) and start_url not in urls: + urls.insert(0, start_url) + pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages) if urls else [] + if not pages: + run.warn( + "policy_documents_not_found", + ( + "No linked policy documents were found; the discovery page is retained " + "as unknown policy evidence." + ), + ) + pages = discovery_pages + + policy_documents = [_policy_document(page, pages) for page in pages] + clauses = [ + clause + for document, page in zip(policy_documents, pages, strict=False) + for clause in _stable_clauses(page, pages, document_type=str(document["document_type"])) + ] + changes = _clause_changes(baseline_pack, clauses) if baseline_pack else [] + run.warn( + "no_legal_conclusions", + ( + "Policy extraction reports source text and change candidates only; " + "it does not provide legal conclusions." + ), + ) + + policies_path = run.output_dir / "policies.ndjson" + clauses_path = run.output_dir / "policy.clauses.ndjson" + changes_path = run.output_dir / "policy.changes.json" + append_ndjson(policies_path, policy_documents) + append_ndjson(clauses_path, clauses) + write_json( + changes_path, + { + "schema_version": CONTEXT_PACK_SCHEMA_VERSION, + "baseline_pack": str(baseline_pack.resolve()) if baseline_pack else None, + "change_count": len(changes), + "changes": changes, + "disclaimer": "Textual change candidates only; no legal conclusions.", + }, + ) + result_payload = { + "workflow": POLICY_WORKFLOW, + "provider": "local", + "status": status_from_errors(run.errors), + "input": {"value": public_url(domain_or_url), "domain": domain}, + "summary": { + "domain": domain, + "page_count": len(pages), + "policy_document_count": len(policy_documents), + "clause_count": len(clauses), + "change_candidate_count": len(changes), + "document_types": sorted({str(item["document_type"]) for item in policy_documents}), + }, + "policies": policy_documents, + "clauses": clauses, + "change_candidates": changes, + "warnings": run.warnings, + "errors": run.errors, + "replay_config": { + "domain_or_url": domain_or_url, + "max_pages": max_pages, + "baseline_pack": str(baseline_pack) if baseline_pack else None, + }, + } + return write_basic_pack_files( + run=run, + pages=pages, + result_filename="policy.result.json", + result_payload=result_payload, + markdown_filename="POLICIES.md", + markdown_text=_policy_markdown(policy_documents, clauses, changes), + pack_filename="policy.pack.json", + extra_artifacts={ + "policies_ndjson": artifact_ref(run.output_dir, policies_path), + "policy_clauses": artifact_ref(run.output_dir, clauses_path), + "policy_changes": artifact_ref(run.output_dir, changes_path), + }, + ) + + +def classify_policy_document(page: PageSnapshot) -> str: + haystack = " ".join((page.url, page.title or "", page.markdown[:2000])).lower() + scored = [ + (sum(1 for term in terms if term in haystack), document_type) + for document_type, terms in _TYPE_TERMS.items() + ] + score, document_type = max(scored, default=(0, "other")) + return document_type if score else "other" + + +def _policy_target_hint(page: PageSnapshot) -> bool: + haystack = f"{urlparse(page.url).path} {page.title or ''}".lower() + return any(term in haystack for terms in _TYPE_TERMS.values() for term in terms) + + +def _policy_urls(page: PageSnapshot, *, domain: str, max_pages: int) -> list[str]: + candidates: list[tuple[int, str]] = [] + for link in extract_links(page): + url = public_url(urljoin(page.url, link["url"])) + if not same_policy_domain(url, domain): + continue + haystack = f"{urlparse(url).path} {link['text']}".lower() + score = sum(1 for terms in _TYPE_TERMS.values() for term in terms if term in haystack) + if score: + candidates.append((score, url)) + output: list[str] = [] + for _score, url in sorted(candidates, key=lambda item: (-item[0], item[1])): + if url not in output: + output.append(url) + if len(output) >= max_pages: + break + return output + + +def _policy_document(page: PageSnapshot, pages: list[PageSnapshot]) -> dict[str, Any]: + document_type = classify_policy_document(page) + content_hash = hashlib.sha256(page.markdown.encode("utf-8")).hexdigest() + effective_match = _EFFECTIVE_DATE_RE.search(page.markdown) + effective_date = effective_match.group("date") if effective_match else None + evidence_text = effective_match.group(0) if effective_match else page.markdown[:280] + return { + "schema_version": CONTEXT_PACK_SCHEMA_VERSION, + "policy_document_id": stable_id("policy", {"url": page.url, "type": document_type}), + "document_type": document_type, + "title": page.title, + "url": page.url, + "effective_date": effective_date, + "content_hash": content_hash, + "classification_status": "heuristic", + "evidence": evidence_for_page( + page, + pages, + field="effective_date" if effective_date else "policy_document", + excerpt=evidence_text, + ).to_dict(), + } + + +def _stable_clauses( + page: PageSnapshot, + pages: list[PageSnapshot], + *, + document_type: str, +) -> list[dict[str, Any]]: + content = page.markdown.strip() + matches = list(_HEADING_RE.finditer(content)) + sections: list[tuple[str, str, int]] = [] + if matches: + for index, match in enumerate(matches): + start = match.start() + end = matches[index + 1].start() if index + 1 < len(matches) else len(content) + sections.append((match.group(2).strip(), content[start:end].strip(), start)) + else: + paragraphs = [item.strip() for item in re.split(r"\n\s*\n", content) if item.strip()] + cursor = 0 + for index, paragraph in enumerate(paragraphs, start=1): + start = content.find(paragraph, cursor) + cursor = max(cursor, start + len(paragraph)) + sections.append((f"Clause {index}", paragraph, max(0, start))) + + clauses: list[dict[str, Any]] = [] + duplicate_headings: dict[str, int] = {} + for heading, text, _start in sections: + normalized_heading = " ".join(heading.lower().split()) + duplicate_headings[normalized_heading] = duplicate_headings.get(normalized_heading, 0) + 1 + ordinal = duplicate_headings[normalized_heading] + clause_key = { + "url": page.url, + "document_type": document_type, + "heading": normalized_heading, + "ordinal": ordinal, + } + clauses.append( + { + "schema_version": CONTEXT_PACK_SCHEMA_VERSION, + "clause_id": stable_id("clause", clause_key), + "document_type": document_type, + "heading": heading, + "ordinal": ordinal, + "url": page.url, + "section_hash": hashlib.sha256(text.encode("utf-8")).hexdigest(), + "text": text, + "evidence": evidence_for_page(page, pages, field="clause", excerpt=text).to_dict(), + } + ) + return clauses + + +def _clause_changes(baseline_pack: Path, current: list[dict[str, Any]]) -> list[dict[str, Any]]: + path = baseline_pack.resolve() / "policy.clauses.ndjson" + if not path.exists(): + raise ContextPackError(f"Baseline policy pack has no policy.clauses.ndjson: {baseline_pack}") + previous = [ + value + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() and isinstance((value := json.loads(line)), dict) + ] + old_by_id = {str(item.get("clause_id")): item for item in previous if item.get("clause_id")} + new_by_id = {str(item.get("clause_id")): item for item in current if item.get("clause_id")} + changes: list[dict[str, Any]] = [] + for clause_id in sorted(old_by_id.keys() | new_by_id.keys()): + before = old_by_id.get(clause_id) + after = new_by_id.get(clause_id) + if before and after and before.get("section_hash") == after.get("section_hash"): + continue + kind = "modified" if before and after else ("removed" if before else "added") + payload = { + "clause_id": clause_id, + "change_type": kind, + "before": before.get("evidence") if before else None, + "after": after.get("evidence") if after else None, + "before_hash": before.get("section_hash") if before else None, + "after_hash": after.get("section_hash") if after else None, + "status": "candidate", + "classification": "policy", + } + payload["change_candidate_id"] = stable_id("change", payload) + changes.append(payload) + return changes + + +def _policy_markdown( + documents: list[dict[str, Any]], + clauses: list[dict[str, Any]], + changes: list[dict[str, Any]], +) -> str: + lines = [ + "# Policy Evidence", + "", + "> Source text and change candidates only. DocPull does not provide legal conclusions.", + "", + ] + for document in documents: + lines.extend( + [ + f"## {quote_markdown(str(document.get('title') or document['document_type']))}", + f"- Type: `{document['document_type']}`", + f"- Effective date: `{document.get('effective_date') or 'not detected'}`", + f"- Source: {document['url']}", + "", + ] + ) + lines.extend( + [ + "## Summary", + "", + f"- Stable clauses: {len(clauses)}", + f"- Change candidates: {len(changes)}", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +__all__ = [ + "DEFAULT_POLICY_OUTPUT_DIR", + "POLICY_DOCUMENT_TYPES", + "POLICY_WORKFLOW", + "build_policy_pack", + "classify_policy_document", +] diff --git a/src/docpull/context_packs/product.py b/src/docpull/context_packs/product.py index 73d901f..b5d521f 100644 --- a/src/docpull/context_packs/product.py +++ b/src/docpull/context_packs/product.py @@ -36,11 +36,27 @@ PRODUCT_WORKFLOW = "product-pack" DEFAULT_PRODUCT_OUTPUT_DIR = Path("packs/products") PRICE_RE = re.compile( - r"(?P\$|USD|EUR|GBP|€|£)\s?(?P\d[\d,]*(?:\.\d+)?)" - r"(?:\s?/(?Pmo|month|monthly|yr|year|yearly|user|seat))?", + r"(?PUS\$|CA\$|AU\$|\$|USD|CAD|AUD|EUR|GBP|JPY|INR|€|£|¥|₹)\s?" + r"(?P\d[\d,]*(?:\.\d+)?)" + r"(?:\s?(?:/|per\s+)(?Pmo|month|monthly|yr|year|yearly|user|seat|request|unit))?", re.IGNORECASE, ) +TRIAL_RE = re.compile(r"(?P\d{1,3})[-\s]day\s+(?:free\s+)?trial|free\s+trial", re.IGNORECASE) PRODUCT_LINK_KEYWORDS = ("pricing", "product", "products", "plans", "shop", "store") +PRICING_CONTEXT_TERMS = ( + "pricing", + "price", + "plan", + "billing", + "monthly", + "annually", + "per month", + "per year", + "subscription", + "free trial", + "buy", + "checkout", +) def build_product_pack( @@ -110,6 +126,11 @@ def build_product_pack( }, "products": products, "pricing_matrix": pricing_rows, + "pricing_extraction": { + "accepted_medium": "page_text", + "excluded_mediums": ["screenshot", "mockup", "image_alt_text"], + "incidental_value_policy": "Require pricing context outside explicit pricing components.", + }, "warnings": run.warnings, "errors": run.errors, "replay_config": {"url_or_domain": url_or_domain, "mode": mode, "max_pages": max_pages}, @@ -227,6 +248,10 @@ def _offers_from_jsonld(raw: Any, page: PageSnapshot, pages: list[PageSnapshot]) "price": price, "currency": _string_or_none(item.get("priceCurrency")), "billing_frequency": _billing_from_text(json.dumps(item, ensure_ascii=False)), + "billing_interval": _billing_interval(json.dumps(item, ensure_ascii=False)), + "trial": _trial_from_text(json.dumps(item, ensure_ascii=False)), + "feature_gates": _feature_gates(json.dumps(item, ensure_ascii=False)), + "price_source": {"medium": "structured_data", "context": "jsonld_offer"}, "availability": _string_or_none(item.get("availability")), "url": public_url(str(item.get("url") or page.url)), "evidence": evidence_for_page( @@ -243,7 +268,7 @@ def _offers_from_jsonld(raw: Any, page: PageSnapshot, pages: list[PageSnapshot]) def _pricing_rows(page: PageSnapshot, pages: list[PageSnapshot]) -> list[dict[str, Any]]: soup = soup_for(page) rows: list[dict[str, Any]] = [] - text_blocks: list[str] = [] + text_blocks: list[tuple[str, str]] = [] for selector in ("table tr", '[class*="pricing" i]', '[class*="plan" i]', '[class*="price" i]'): tags: list[Any] try: @@ -253,28 +278,44 @@ def _pricing_rows(page: PageSnapshot, pages: list[PageSnapshot]) -> list[dict[st for tag in tags[:80]: text = " ".join(tag.get_text(" ").split()) if PRICE_RE.search(text): - text_blocks.append(text) + text_blocks.append((text, f"pricing_component:{selector}")) if not text_blocks: - text_blocks = [line.strip() for line in page.markdown.splitlines() if PRICE_RE.search(line)] - seen: set[str] = set() - for block in text_blocks: - if block in seen: - continue - seen.add(block) - match = PRICE_RE.search(block) - if not match: - continue - rows.append( - { - "plan_name": _plan_name_from_text(block), - "price": _parse_price_value(match.group("amount")), - "currency": _currency_code(match.group("currency")), - "billing_frequency": _billing_from_text(block) or match.group("period"), - "raw_text": block[:500], - "source_url": page.url, - "evidence": evidence_for_page(page, pages, field="pricing", excerpt=block[:240]).to_dict(), - } - ) + text_blocks = [ + (line.strip(), "page_text:contextual_line") + for line in page.markdown.splitlines() + if PRICE_RE.search(line) and _looks_like_pricing_text(line) + ] + seen: set[tuple[str, float | None, str | None]] = set() + for block, context in text_blocks: + for match in PRICE_RE.finditer(block): + price = _parse_price_value(match.group("amount")) + currency = _currency_code(match.group("currency")) + key = (block, price, currency) + if key in seen: + continue + seen.add(key) + rows.append( + { + "plan_name": _plan_name_from_text(block), + "price": price, + "currency": currency, + "billing_frequency": _billing_from_text(block) or match.group("period"), + "billing_interval": _billing_interval(block, fallback=match.group("period")), + "trial": _trial_from_text(block), + "feature_gates": _feature_gates(block), + "raw_text": block[:500], + "source_url": page.url, + "price_source": {"medium": "page_text", "context": context}, + "evidence": evidence_for_page( + page, + pages, + field="pricing", + excerpt=block[:500], + ).to_dict(), + } + ) + if len(rows) >= 60: + return rows return rows[:60] @@ -360,7 +401,23 @@ def _parse_price_value(value: Any) -> float | None: def _currency_code(value: str | None) -> str | None: if not value: return None - mapping = {"$": "USD", "usd": "USD", "€": "EUR", "eur": "EUR", "£": "GBP", "gbp": "GBP"} + mapping = { + "$": "USD", + "us$": "USD", + "usd": "USD", + "ca$": "CAD", + "cad": "CAD", + "au$": "AUD", + "aud": "AUD", + "€": "EUR", + "eur": "EUR", + "£": "GBP", + "gbp": "GBP", + "¥": "JPY", + "jpy": "JPY", + "₹": "INR", + "inr": "INR", + } return mapping.get(value.lower(), value.upper()) @@ -377,6 +434,55 @@ def _billing_from_text(text: str) -> str | None: return None +def _billing_interval(text: str, *, fallback: str | None = None) -> dict[str, Any] | None: + lowered = text.lower() + if any(value in lowered for value in ("/mo", "monthly", "per month")): + return {"unit": "month", "count": 1} + if any(value in lowered for value in ("/yr", "yearly", "annually", "per year")): + return {"unit": "year", "count": 1} + if any(value in lowered for value in ("per user", "/user")): + return {"unit": "user", "count": 1} + if any(value in lowered for value in ("per seat", "/seat")): + return {"unit": "seat", "count": 1} + if fallback: + return {"unit": fallback.lower(), "count": 1} + return None + + +def _trial_from_text(text: str) -> dict[str, Any] | None: + match = TRIAL_RE.search(text) + if not match: + return None + days = match.groupdict().get("days") + return { + "available": True, + "duration_days": int(days) if days else None, + "requires_payment_method": "unknown", + } + + +def _feature_gates(text: str) -> list[dict[str, str]]: + gates: list[dict[str, str]] = [] + for fragment in re.split(r"[•|;]", text): + cleaned = " ".join(fragment.split()) + lowered = cleaned.lower() + if len(cleaned) < 4 or len(cleaned) > 180: + continue + if any(term in lowered for term in ("included", "unlimited", "only", "limit", "add-on")): + availability = "included" + if "only" in lowered or "add-on" in lowered: + availability = "gated" + elif "limit" in lowered and "unlimited" not in lowered: + availability = "limited" + gates.append({"feature": cleaned, "availability": availability}) + return gates[:20] + + +def _looks_like_pricing_text(text: str) -> bool: + lowered = text.lower() + return any(term in lowered for term in PRICING_CONTEXT_TERMS) + + def _plan_name_from_text(text: str) -> str | None: before_price = PRICE_RE.split(text, maxsplit=1)[0].strip(" :-|") if before_price: diff --git a/src/docpull/context_packs/visuals.py b/src/docpull/context_packs/visuals.py index c6954d6..11535d8 100644 --- a/src/docpull/context_packs/visuals.py +++ b/src/docpull/context_packs/visuals.py @@ -198,7 +198,6 @@ def capture_screenshot_pack( markdown_text=_screenshot_markdown(payload), pack_filename="screenshot.pack.json", extra_artifacts={"screenshot": payload["path"]}, - local_pack_records=False, ) diff --git a/src/docpull/context_packs/workflow_cli.py b/src/docpull/context_packs/workflow_cli.py new file mode 100644 index 0000000..71a2b3f --- /dev/null +++ b/src/docpull/context_packs/workflow_cli.py @@ -0,0 +1,52 @@ +"""Public CLI adapters for evidence-backed workflow-protocol packs.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from ..policy import PolicyConfig +from ._legacy_cli import ( + run_brand_pack_cli, + run_image_pack_cli, + run_product_pack_cli, + run_screenshot_pack_cli, + run_styleguide_pack_cli, +) +from .cli import _positive_int, _run_and_print +from .policy_pack import DEFAULT_POLICY_OUTPUT_DIR, build_policy_pack + + +def run_policy_pack_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="docpull policy-pack", + description="Discover policy documents and emit clause-level evidence without legal conclusions", + ) + parser.add_argument("domain_or_url") + parser.add_argument("--output-dir", "-o", type=Path, default=DEFAULT_POLICY_OUTPUT_DIR) + parser.add_argument("--policy", type=Path) + parser.add_argument("--max-pages", type=_positive_int, default=16) + parser.add_argument("--baseline-pack", type=Path) + parser.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + return _run_and_print( + lambda: build_policy_pack( + args.domain_or_url, + output_dir=args.output_dir, + policy=PolicyConfig.from_file(args.policy) if args.policy else None, + max_pages=args.max_pages, + baseline_pack=args.baseline_pack, + ), + json_output=args.json_output, + success_label="Policy pack", + ) + + +__all__ = [ + "run_brand_pack_cli", + "run_image_pack_cli", + "run_policy_pack_cli", + "run_product_pack_cli", + "run_screenshot_pack_cli", + "run_styleguide_pack_cli", +] diff --git a/src/docpull/contracts.py b/src/docpull/contracts.py new file mode 100644 index 0000000..c714cd7 --- /dev/null +++ b/src/docpull/contracts.py @@ -0,0 +1,505 @@ +"""Versioned cross-repository contracts for DocPull acquisition workflows. + +The models in this module are intentionally transport-neutral. They describe +what DocPull acquired and emitted without prescribing a scheduler, reviewer, or +downstream product model. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Final, Literal + +from pydantic import BaseModel, ConfigDict, Field + +from .models.document import DocumentRecord +from .models.run import RunIdentity +from .time_utils import utc_now_iso + +WORKFLOW_REQUEST_CONTRACT: Final[Literal["workflow.request.v1"]] = "workflow.request.v1" +WORKFLOW_RESULT_CONTRACT: Final[Literal["workflow.result.v1"]] = "workflow.result.v1" +ARTIFACT_MANIFEST_CONTRACT: Final[Literal["artifact.manifest.v1"]] = "artifact.manifest.v1" +INTELLIGENCE_BUNDLE_CONTRACT: Final[Literal["intelligence.bundle.v1"]] = "intelligence.bundle.v1" +CHANGE_EVENT_CONTRACT: Final[Literal["change.event.v1"]] = "change.event.v1" + + +class ContractModel(BaseModel): + """Forward-compatible base for public wire contracts.""" + + model_config = ConfigDict(extra="allow") + + +class HashDigest(ContractModel): + algorithm: Literal["sha256"] = "sha256" + digest: str + + +class BudgetUsage(ContractModel): + limit_usd: float | None = None + estimated_usd: float = 0.0 + actual_usd: float | None = None + paid_request_count: int = 0 + http_request_count: int = 0 + cache_hit_count: int = 0 + local_browser_seconds: float = 0.0 + blocked_actions: list[dict[str, Any]] = Field(default_factory=list) + + +class WorkflowProgressEvent(ContractModel): + event_id: str + phase: str + status: Literal["started", "progress", "completed", "warning", "failed"] + timestamp: str + message: str | None = None + current: int | None = None + total: int | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class WorkflowWarning(ContractModel): + code: str + message: str + metadata: dict[str, Any] = Field(default_factory=dict) + + +class WorkflowFailure(ContractModel): + code: str = "workflow_error" + message: str + retryable: bool = False + source_url: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class ReplayConfiguration(ContractModel): + local_first: bool = True + browser_enabled: bool = False + paid_routes_enabled: bool = False + scheduler: None = None + configuration: dict[str, Any] = Field(default_factory=dict) + + +class WorkflowRequest(ContractModel): + contract_version: Literal["workflow.request.v1"] = WORKFLOW_REQUEST_CONTRACT + schema_version: int = 1 + request_id: str + workflow: str + input: dict[str, Any] + output: dict[str, Any] = Field(default_factory=dict) + options: dict[str, Any] = Field(default_factory=dict) + source_policy: dict[str, Any] = Field(default_factory=dict) + budget: dict[str, Any] = Field(default_factory=dict) + replay: ReplayConfiguration = Field(default_factory=ReplayConfiguration) + + +class ArtifactEntry(ContractModel): + name: str + path: str + role: str + media_type: str | None = None + bytes: int + sha256: str + + +class ArtifactManifest(ContractModel): + contract_version: Literal["artifact.manifest.v1"] = ARTIFACT_MANIFEST_CONTRACT + schema_version: int = 1 + pack_id: str + run_id: str + hash_algorithm: Literal["sha256"] = "sha256" + entries: list[ArtifactEntry] + aggregate_sha256: str + + +class WorkflowResult(ContractModel): + contract_version: Literal["workflow.result.v1"] = WORKFLOW_RESULT_CONTRACT + schema_version: int = 1 + request_id: str + workflow: str + status: Literal["completed", "completed_with_warnings", "failed", "cancelled"] + started_at: str + finished_at: str + pack_identity: dict[str, Any] + run_identity: dict[str, Any] + summary: dict[str, Any] = Field(default_factory=dict) + data: dict[str, Any] = Field(default_factory=dict) + progress_events: list[WorkflowProgressEvent] = Field(default_factory=list) + warnings: list[WorkflowWarning] = Field(default_factory=list) + failures: list[WorkflowFailure] = Field(default_factory=list) + budget_usage: BudgetUsage = Field(default_factory=BudgetUsage) + hashes: dict[str, HashDigest] = Field(default_factory=dict) + replay_configuration: ReplayConfiguration = Field(default_factory=ReplayConfiguration) + artifact_manifest: str = "artifact.manifest.json" + compatibility_artifacts: dict[str, str] = Field(default_factory=dict) + + +class EvidenceSpan(ContractModel): + citation_id: str + record_citation_id: str | None = None + document_id: str + document_version: str + url: str + char_start: int = Field(ge=0) + char_end: int = Field(ge=0) + exact_text: str + exact_text_sha256: str + + +class SourceAuthority(ContractModel): + role: Literal[ + "official_product", + "legal", + "documentation", + "social", + "marketplace", + "third_party", + ] + tier: Literal["tier_1_authoritative", "tier_2_owned", "tier_3_distribution", "tier_4_external"] + rationale: str + + +class Observation(ContractModel): + observation_id: str + type: str + text: str + status: Literal["observation"] = "observation" + evidence_strength: Literal["strong", "moderate", "weak", "unknown"] + confidence: float = Field(ge=0.0, le=1.0) + source_authority: SourceAuthority + evidence: list[EvidenceSpan] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + + +class SourceSnapshot(ContractModel): + source_snapshot_id: str + source_id: str + url: str + document_id: str | None = None + document_version: str | None = None + content_hash: str | None = None + fetched_at: str | None = None + authority: SourceAuthority + + +class ChangeCandidate(ContractModel): + change_candidate_id: str + classification: Literal["pricing", "positioning", "product", "security", "policy", "other"] + status: Literal["candidate"] = "candidate" + before: list[EvidenceSpan] = Field(default_factory=list) + after: list[EvidenceSpan] = Field(default_factory=list) + confidence: float = Field(ge=0.0, le=1.0) + warnings: list[str] = Field(default_factory=list) + + +class IntelligenceBundle(ContractModel): + contract_version: Literal["intelligence.bundle.v1"] = INTELLIGENCE_BUNDLE_CONTRACT + schema_version: int = 1 + bundle_id: str + bundle_hash: str + pack_identity: dict[str, Any] + run_identity: dict[str, Any] + workspace: dict[str, Any] + source_snapshots: list[SourceSnapshot] = Field(default_factory=list) + document_versions: list[dict[str, Any]] = Field(default_factory=list) + observations: list[Observation] = Field(default_factory=list) + change_candidates: list[ChangeCandidate] = Field(default_factory=list) + warnings: list[WorkflowWarning] = Field(default_factory=list) + summary: dict[str, Any] = Field(default_factory=dict) + artifacts: dict[str, str] = Field(default_factory=dict) + + +class ChangeEvent(ContractModel): + contract_version: Literal["change.event.v1"] = CHANGE_EVENT_CONTRACT + schema_version: int = 1 + event_id: str + idempotency_key: str + workflow: str + url: str + old_document_id: str | None = None + new_document_id: str | None = None + old_hash: str | None = None + new_hash: str | None = None + old_evidence: list[EvidenceSpan] = Field(default_factory=list) + new_evidence: list[EvidenceSpan] = Field(default_factory=list) + structural_changes: list[dict[str, Any]] = Field(default_factory=list) + textual_changes: list[dict[str, Any]] = Field(default_factory=list) + semantic_candidates: list[dict[str, Any]] = Field(default_factory=list) + classifications: list[Literal["pricing", "positioning", "product", "security", "policy", "other"]] = ( + Field(default_factory=list) + ) + replay_configuration: ReplayConfiguration = Field(default_factory=ReplayConfiguration) + + +class PackContractV3(ContractModel): + """Frozen compatibility envelope for existing ``*.pack.json`` files.""" + + schema_version: int + provider: str + workflow: str + status: str | None = None + artifacts: dict[str, Any] = Field(default_factory=dict) + + +class CitationMapContractV1(ContractModel): + schema_version: int + source_count: int + sources: list[dict[str, Any]] + + +class RightsContractV1(ContractModel): + status: str + allowed_use: dict[str, str] + obligations: list[Any] = Field(default_factory=list) + basis: str + + +class ProvenanceContractV1(ContractModel): + schema_version: int + nodes: list[dict[str, Any]] = Field(default_factory=list) + edges: list[dict[str, Any]] = Field(default_factory=list) + + +class BasisContractV2(ContractModel): + schema_version: Literal[2] = 2 + basis_id: str + claim_path: str + claim: str + evidence_state: Literal["supported", "partial", "insufficient"] + confidence: Literal["high", "medium", "low"] + citation_ids: list[str] = Field(default_factory=list) + source_urls: list[str] = Field(default_factory=list) + excerpts: list[dict[str, Any]] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + producer: str + + +CONTRACT_MODELS: dict[str, type[BaseModel]] = { + "workflow-request.v1.schema.json": WorkflowRequest, + "workflow-result.v1.schema.json": WorkflowResult, + "artifact-manifest.v1.schema.json": ArtifactManifest, + "intelligence-bundle.v1.schema.json": IntelligenceBundle, + "change-event.v1.schema.json": ChangeEvent, + "document.v3.schema.json": DocumentRecord, + "run-identity.v1.schema.json": RunIdentity, + "pack.v3.schema.json": PackContractV3, + "citation-map.v1.schema.json": CitationMapContractV1, + "rights.v1.schema.json": RightsContractV1, + "provenance.v1.schema.json": ProvenanceContractV1, + "basis.v2.schema.json": BasisContractV2, +} + + +def canonical_json(value: Any) -> str: + """Serialize a contract value for stable cross-runtime hashing.""" + + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def stable_id(prefix: str, value: Any, *, length: int = 24) -> str: + return f"{prefix}_{canonical_sha256(value)[:length]}" + + +def build_workflow_request( + *, + workflow: str, + input_payload: dict[str, Any], + output_dir: Path, + options: dict[str, Any], + source_policy: dict[str, Any], + budget: dict[str, Any], + browser_enabled: bool = False, + paid_routes_enabled: bool = False, +) -> WorkflowRequest: + identity_payload = { + "workflow": workflow, + "input": input_payload, + "output": {"directory": str(output_dir.resolve())}, + "options": options, + "source_policy": _without_ephemeral(source_policy), + "budget": budget, + "replay": { + "local_first": True, + "browser_enabled": browser_enabled, + "paid_routes_enabled": paid_routes_enabled, + "scheduler": None, + "configuration": options, + }, + } + return WorkflowRequest( + request_id=stable_id("request", identity_payload), + workflow=workflow, + input=input_payload, + output={"directory": str(output_dir.resolve())}, + options=options, + source_policy=source_policy, + budget=budget, + replay=ReplayConfiguration( + browser_enabled=browser_enabled, + paid_routes_enabled=paid_routes_enabled, + configuration=options, + ), + ) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def artifact_entries( + output_dir: Path, + artifacts: dict[str, str], + *, + excluded: set[str] | None = None, +) -> list[ArtifactEntry]: + entries: list[ArtifactEntry] = [] + for name, relative in sorted(artifacts.items()): + if excluded and name in excluded: + continue + path = Path(relative) + candidate = path if path.is_absolute() else output_dir / path + if not candidate.exists() or not candidate.is_file(): + continue + entries.append( + ArtifactEntry( + name=name, + path=str(path), + role=_artifact_role(name), + media_type=_media_type(candidate), + bytes=candidate.stat().st_size, + sha256=file_sha256(candidate), + ) + ) + return entries + + +def write_contract_schemas(output_dir: Path) -> list[Path]: + """Write the canonical JSON Schemas shipped with the package.""" + + output_dir.mkdir(parents=True, exist_ok=True) + written: list[Path] = [] + for filename, model in sorted(CONTRACT_MODELS.items()): + payload = model.model_json_schema(mode="serialization") + payload["$schema"] = "https://json-schema.org/draft/2020-12/schema" + payload["$id"] = f"https://docpull.dev/schemas/{filename}" + path = output_dir / filename + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + written.append(path) + return written + + +def bundled_schema_path(name: str) -> Path: + filename = name if name.endswith(".schema.json") else f"{name}.schema.json" + if filename not in CONTRACT_MODELS: + raise KeyError(f"Unknown DocPull contract schema: {name}") + return Path(__file__).with_name("schemas") / filename + + +def new_progress_event( + *, + phase: str, + status: Literal["started", "progress", "completed", "warning", "failed"], + timestamp: str | None = None, + message: str | None = None, + current: int | None = None, + total: int | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + payload = { + "phase": phase, + "status": status, + "timestamp": timestamp or utc_now_iso(), + "message": message, + "current": current, + "total": total, + "metadata": metadata or {}, + } + payload["event_id"] = stable_id("event", payload) + return WorkflowProgressEvent.model_validate(payload).model_dump(mode="json", exclude_none=True) + + +def _artifact_role(name: str) -> str: + if "citation" in name or "source" in name: + return "evidence" + if "manifest" in name or "pack" in name: + return "manifest" + if "account" in name: + return "budget_usage" + if "markdown" in name or "context" in name: + return "human_readable" + return "workflow_output" + + +def _without_ephemeral(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): _without_ephemeral(item) + for key, item in value.items() + if key not in {"generated_at", "requested_at", "started_at", "finished_at"} + } + if isinstance(value, list): + return [_without_ephemeral(item) for item in value] + return value + + +def _media_type(path: Path) -> str | None: + return { + ".json": "application/json", + ".jsonl": "application/x-ndjson", + ".ndjson": "application/x-ndjson", + ".md": "text/markdown", + ".css": "text/css", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".svg": "image/svg+xml", + }.get(path.suffix.lower()) + + +__all__ = [ + "ARTIFACT_MANIFEST_CONTRACT", + "CHANGE_EVENT_CONTRACT", + "CONTRACT_MODELS", + "INTELLIGENCE_BUNDLE_CONTRACT", + "WORKFLOW_REQUEST_CONTRACT", + "WORKFLOW_RESULT_CONTRACT", + "ArtifactEntry", + "ArtifactManifest", + "BasisContractV2", + "BudgetUsage", + "ChangeCandidate", + "ChangeEvent", + "CitationMapContractV1", + "EvidenceSpan", + "HashDigest", + "IntelligenceBundle", + "Observation", + "PackContractV3", + "ProvenanceContractV1", + "ReplayConfiguration", + "RightsContractV1", + "SourceAuthority", + "SourceSnapshot", + "WorkflowFailure", + "WorkflowProgressEvent", + "WorkflowRequest", + "WorkflowResult", + "WorkflowWarning", + "artifact_entries", + "build_workflow_request", + "bundled_schema_path", + "canonical_json", + "canonical_sha256", + "file_sha256", + "new_progress_event", + "stable_id", + "write_contract_schemas", +] diff --git a/src/docpull/contracts_cli.py b/src/docpull/contracts_cli.py new file mode 100644 index 0000000..ea896e9 --- /dev/null +++ b/src/docpull/contracts_cli.py @@ -0,0 +1,40 @@ +"""CLI for inspecting and exporting DocPull cross-repository schemas.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .contracts import CONTRACT_MODELS, bundled_schema_path, write_contract_schemas + + +def run_contracts_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="docpull contracts", description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + list_parser = subparsers.add_parser("list", help="List bundled contract schemas") + list_parser.add_argument("--json", action="store_true", dest="json_output") + export = subparsers.add_parser("export", help="Export all contract schemas") + export.add_argument("--output-dir", "-o", type=Path, required=True) + show = subparsers.add_parser("show", help="Print one bundled JSON Schema") + show.add_argument("name", choices=sorted(CONTRACT_MODELS)) + args = parser.parse_args(argv) + + if args.command == "list": + names = sorted(CONTRACT_MODELS) + if args.json_output: + print(json.dumps({"schemas": names}, indent=2)) + else: + print("\n".join(names)) + return 0 + if args.command == "export": + paths = write_contract_schemas(args.output_dir.resolve()) + print(f"Exported {len(paths)} schemas to {args.output_dir.resolve()}") + return 0 + if args.command == "show": + print(bundled_schema_path(args.name).read_text(encoding="utf-8"), end="") + return 0 + return 1 + + +__all__ = ["run_contracts_cli"] diff --git a/src/docpull/document_parse.py b/src/docpull/document_parse.py index b421304..d53cfe6 100644 --- a/src/docpull/document_parse.py +++ b/src/docpull/document_parse.py @@ -814,6 +814,7 @@ def _parse_text(path: Path) -> tuple[str, dict[str, Any]]: def _parse_pypdf(path: Path) -> tuple[str, dict[str, Any]]: + _preflight_pdf_container(path) try: module = importlib.import_module("pypdf") except ImportError as err: @@ -870,6 +871,20 @@ def _validate_pdf_structure(path: Path) -> dict[str, Any]: return metadata +def _preflight_pdf_container(path: Path) -> None: + # Reject obviously truncated containers before importing or invoking an + # optional parser backend. This keeps malformed input out of third-party + # parsers and makes the safety result independent of installed extras. + with path.open("rb") as stream: + header = stream.read(8) + stream.seek(0, os.SEEK_END) + size = stream.tell() + stream.seek(max(0, size - 1024)) + trailer = stream.read() + if not header.startswith(b"%PDF-") or b"%%EOF" not in trailer: + raise DocumentParseError("Malformed or truncated PDF container.", code="malformed") + + def _pypdf_page_image_count(page: Any) -> int: try: resources = page.get("/Resources") or {} diff --git a/src/docpull/eval_grade.py b/src/docpull/eval_grade.py index a3ce403..a88e52d 100644 --- a/src/docpull/eval_grade.py +++ b/src/docpull/eval_grade.py @@ -104,10 +104,11 @@ def prepare_eval_grade_pack( _write_json(paths["rights_manifest"], rights) _write_json(paths["provenance_graph"], provenance) _write_json(paths["citation_index"], citation_index) - if markdown: - card_path = pack_dir / "PACK_CARD.md" - card_path.write_text(card, encoding="utf-8") - output_artifacts["pack_card"] = _artifact_ref(pack_dir, card_path) + # PACK_CARD.md is mandatory at eval grade even when optional prepare + # summaries are requested as JSON-only. + card_path = pack_dir / "PACK_CARD.md" + card_path.write_text(card, encoding="utf-8") + output_artifacts["pack_card"] = _artifact_ref(pack_dir, card_path) for key, path in paths.items(): output_artifacts[key] = _artifact_ref(pack_dir, path) diff --git a/src/docpull/evidence.py b/src/docpull/evidence.py new file mode 100644 index 0000000..8a56cdb --- /dev/null +++ b/src/docpull/evidence.py @@ -0,0 +1,154 @@ +"""Evidence-span and source-authority helpers shared by pack workflows.""" + +from __future__ import annotations + +import hashlib +from typing import Any +from urllib.parse import urlparse + +from .contracts import EvidenceSpan, SourceAuthority + +_LEGAL_TERMS = ( + "terms", + "privacy", + "legal", + "dpa", + "data-processing", + "cookie", + "subprocessor", + "security", + "refund", + "ai-terms", +) +_SOCIAL_DOMAINS = { + "facebook.com", + "instagram.com", + "linkedin.com", + "medium.com", + "threads.net", + "tiktok.com", + "twitter.com", + "x.com", + "youtube.com", +} +_MARKETPLACE_DOMAINS = { + "apps.apple.com", + "play.google.com", + "aws.amazon.com", + "marketplace.atlassian.com", + "marketplace.visualstudio.com", + "chromewebstore.google.com", +} + + +def classify_source_authority(url: str, *, official_domain: str | None = None) -> SourceAuthority: + """Classify source role and authority without making review decisions.""" + + parsed = urlparse(url) + host = (parsed.hostname or "").lower().removeprefix("www.") + path = (parsed.path or "/").lower() + official = (official_domain or "").lower().removeprefix("www.").rstrip(".") + same_official = bool(official) and (host == official or host.endswith(f".{official}")) + + if same_official and any(term in path for term in _LEGAL_TERMS): + return SourceAuthority( + role="legal", + tier="tier_1_authoritative", + rationale="Official-domain legal, policy, security, or contractual source.", + ) + if same_official and (host.startswith("docs.") or "/docs" in path or "/documentation" in path): + return SourceAuthority( + role="documentation", + tier="tier_1_authoritative", + rationale="Official-domain product documentation source.", + ) + if any(host == domain or host.endswith(f".{domain}") for domain in _SOCIAL_DOMAINS): + return SourceAuthority( + role="social", + tier="tier_2_owned", + rationale="Organization-controlled social distribution source; authorship still requires review.", + ) + if any(host == domain or host.endswith(f".{domain}") for domain in _MARKETPLACE_DOMAINS): + return SourceAuthority( + role="marketplace", + tier="tier_3_distribution", + rationale="Platform marketplace listing rather than the canonical product site.", + ) + if same_official: + return SourceAuthority( + role="official_product", + tier="tier_1_authoritative", + rationale="Official-domain product or company source.", + ) + return SourceAuthority( + role="third_party", + tier="tier_4_external", + rationale="External source; downstream review must decide how it may be used.", + ) + + +def evidence_span( + *, + url: str, + content: str, + exact_text: str, + citation_id: str, + record_citation_id: str | None = None, + occurrence: int = 0, +) -> EvidenceSpan: + """Return a character-precise span tied to a deterministic document version.""" + + content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + document_id = document_identity(url, content_hash) + start = _nth_index(content, exact_text, occurrence) + selected = exact_text + if start < 0: + selected = exact_text.strip() + start = content.find(selected) + if start < 0: + selected = content[: min(len(content), max(1, len(exact_text)))] + start = 0 + end = start + len(selected) + return EvidenceSpan( + citation_id=citation_id, + record_citation_id=record_citation_id, + document_id=document_id, + document_version=content_hash, + url=url, + char_start=start, + char_end=end, + exact_text=selected, + exact_text_sha256=hashlib.sha256(selected.encode("utf-8")).hexdigest(), + ) + + +def evidence_span_payload(**kwargs: Any) -> dict[str, Any]: + return evidence_span(**kwargs).model_dump(mode="json", exclude_none=True) + + +def document_identity(url: str, content_hash: str) -> str: + """Match :meth:`DocumentRecord.from_page` document identity exactly.""" + + digest = hashlib.sha256(f"{url}\x1f{content_hash}".encode()).hexdigest() + return f"doc_{digest[:24]}" + + +def _nth_index(content: str, text: str, occurrence: int) -> int: + if not text: + return 0 + start = -1 + cursor = 0 + for _index in range(max(0, occurrence) + 1): + start = content.find(text, cursor) + if start < 0: + return -1 + cursor = start + len(text) + return start + + +__all__ = [ + "classify_source_authority", + "document_identity", + "evidence_span", + "evidence_span_payload", +] diff --git a/src/docpull/mcp/server.py b/src/docpull/mcp/server.py index 277046d..4e06e8c 100644 --- a/src/docpull/mcp/server.py +++ b/src/docpull/mcp/server.py @@ -61,6 +61,7 @@ from ..pack_reader import load_pack from ..pack_tools import ( build_citation_map, + build_intelligence_bundle, build_research_brief, diff_packs, extract_pack_entities, @@ -71,6 +72,7 @@ from ..policy import PolicyConfig from ..rendering import render_url_to_directory from ..surface import PRUNED_MCP_TOOLS +from ..workflows import create_workflow_request, run_workflow from .tools import ( ToolResult, add_source, @@ -504,6 +506,64 @@ "required": ["status", "pack_dir", "document_count", "source_count"], } +_WORKFLOW_RESULT_OUTPUT_SCHEMA = { + "type": "object", + "properties": { + "contract_version": {"const": "workflow.result.v1"}, + "request_id": {"type": "string"}, + "workflow": {"type": "string"}, + "status": {"type": "string"}, + "pack_identity": {"type": "object"}, + "run_identity": {"type": "object"}, + "progress_events": {"type": "array", "items": {"type": "object"}}, + "warnings": {"type": "array", "items": {"type": "object"}}, + "failures": {"type": "array", "items": {"type": "object"}}, + "budget_usage": {"type": "object"}, + "hashes": {"type": "object"}, + "replay_configuration": {"type": "object"}, + }, + "required": [ + "contract_version", + "request_id", + "workflow", + "status", + "pack_identity", + "run_identity", + "progress_events", + "warnings", + "failures", + "budget_usage", + "hashes", + "replay_configuration", + ], +} + +_INTELLIGENCE_BUNDLE_OUTPUT_SCHEMA = { + "type": "object", + "properties": { + "contract_version": {"const": "intelligence.bundle.v1"}, + "bundle_id": {"type": "string"}, + "bundle_hash": {"type": "string"}, + "pack_identity": {"type": "object"}, + "run_identity": {"type": "object"}, + "source_snapshots": {"type": "array", "items": {"type": "object"}}, + "document_versions": {"type": "array", "items": {"type": "object"}}, + "observations": {"type": "array", "items": {"type": "object"}}, + "change_candidates": {"type": "array", "items": {"type": "object"}}, + }, + "required": [ + "contract_version", + "bundle_id", + "bundle_hash", + "pack_identity", + "run_identity", + "source_snapshots", + "document_versions", + "observations", + "change_candidates", + ], +} + def _coerce_int(value: Any, *, name: str, default: int) -> int: """Accept int or numeric string; reject anything else with a clear error.""" @@ -811,6 +871,93 @@ async def _dispatch_tool( line_end=_coerce_int(line_end, name="line_end", default=0) or None, ) + elif name in { + "workflow_run", + "brand_pack", + "product_pack", + "styleguide_pack", + "image_pack", + "screenshot_pack", + "policy_pack", + }: + workflow_aliases = { + "brand_pack": ("brand-pack", "domain_or_url", "packs/brand"), + "product_pack": ("product-pack", "url_or_domain", "packs/products"), + "styleguide_pack": ("styleguide-pack", "domain_or_url", "packs/styleguide"), + "image_pack": ("image-pack", "url_or_pack", "packs/images"), + "screenshot_pack": ("screenshot-pack", "url", "packs/screenshot"), + "policy_pack": ("policy-pack", "domain_or_url", "packs/policies"), + } + if name == "workflow_run": + workflow_name = _require_str(arguments, "workflow") + value = _require_str(arguments, "value") + default_output = f"packs/{workflow_name.replace('-pack', '').replace('_', '-')}" + else: + workflow_name, input_key, default_output = workflow_aliases[name] + value = _require_str(arguments, input_key) + output_dir = _path_arg(arguments, "output_dir", default_output) + policy_path_raw = arguments.get("policy") + if policy_path_raw is not None and not isinstance(policy_path_raw, str): + raise ValueError("'policy' must be a path string") + workflow_policy = PolicyConfig.from_file(Path(policy_path_raw)) if policy_path_raw else None + reserved = { + "workflow", + "value", + "domain_or_url", + "url_or_domain", + "url_or_pack", + "url", + "output_dir", + "policy", + "options", + } + options = {key: value for key, value in arguments.items() if key not in reserved} + nested_options = arguments.get("options") + if nested_options is not None: + if not isinstance(nested_options, dict): + raise ValueError("'options' must be an object") + options.update(nested_options) + workflow_request = create_workflow_request( + workflow_name, + value, + output_dir=output_dir, + options=options, + policy=workflow_policy, + ) + workflow_payload = await asyncio.to_thread(run_workflow, workflow_request) + result = ToolResult( + f"Workflow {workflow_payload['workflow']}: {workflow_payload['status']}", + data=workflow_payload, + ) + + elif name == "intelligence_bundle": + objective = arguments.get("objective") + market = arguments.get("market") + if objective is not None and not isinstance(objective, str): + raise ValueError("'objective' must be a string") + if market is not None and not isinstance(market, str): + raise ValueError("'market' must be a string") + output_raw = arguments.get("output") + if output_raw is not None and not isinstance(output_raw, str): + raise ValueError("'output' must be a path string") + bundle_payload = await asyncio.to_thread( + build_intelligence_bundle, + _path_arg(arguments, "pack_dir"), + objective=objective, + market=market, + search_queries=_string_list_arg(arguments, "search_queries") or None, + default_search=bool(arguments.get("default_search", True)), + required_domains=_string_list_arg(arguments, "required_domains"), + max_excerpts=_coerce_int(arguments.get("max_excerpts"), name="max_excerpts", default=8), + entity_limit=_coerce_int(arguments.get("entity_limit"), name="entity_limit", default=20), + search_limit=_coerce_int(arguments.get("search_limit"), name="search_limit", default=10), + output=Path(output_raw) if output_raw else None, + ) + result = ToolResult( + f"Intelligence bundle: {bundle_payload['bundle_id']}", + data=bundle_payload, + ) + elif name == "pack_score": payload = await asyncio.to_thread( score_pack, @@ -1417,6 +1564,230 @@ async def _list_tools() -> list[Tool]: # type: ignore[no-any-unimported] }, outputSchema=_READ_DOC_OUTPUT_SCHEMA, ), + Tool( + name="workflow_run", + description=( + "Run a registered evidence-pack workflow through workflow.request.v1 and return " + "workflow.result.v1. Supported workflows are brand, product, styleguide, visual/image, " + "screenshot, and policy. Browser use remains explicitly gated." + ), + annotations=ToolAnnotations( + title="Run an evidence-pack workflow", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "workflow": { + "type": "string", + "enum": [ + "brand-pack", + "product-pack", + "styleguide-pack", + "visual-pack", + "image-pack", + "screenshot-pack", + "policy-pack", + ], + }, + "value": {"type": "string"}, + "output_dir": {"type": "string"}, + "policy": {"type": "string"}, + "options": {"type": "object"}, + }, + "required": ["workflow", "value", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="brand_pack", + description="Build an evidence-backed brand pack through the common workflow protocol.", + annotations=ToolAnnotations( + title="Build a brand pack", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "domain_or_url": {"type": "string"}, + "output_dir": {"type": "string"}, + "email": {"type": "string"}, + "name": {"type": "string"}, + "ticker": {"type": "string"}, + "download_assets": {"type": "boolean", "default": True}, + "max_pages": {"type": "integer", "minimum": 1, "default": 6}, + "policy": {"type": "string"}, + }, + "required": ["domain_or_url", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="product_pack", + description="Build product and pricing evidence through the common workflow protocol.", + annotations=ToolAnnotations( + title="Build a product pack", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "url_or_domain": {"type": "string"}, + "output_dir": {"type": "string"}, + "mode": {"type": "string", "enum": ["page", "site"], "default": "page"}, + "max_pages": {"type": "integer", "minimum": 1, "default": 8}, + "policy": {"type": "string"}, + }, + "required": ["url_or_domain", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="styleguide_pack", + description=( + "Build styleguide and design-token evidence through the common workflow protocol." + ), + annotations=ToolAnnotations( + title="Build a styleguide pack", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "domain_or_url": {"type": "string"}, + "output_dir": {"type": "string"}, + "render": {"type": "boolean", "default": False}, + "max_stylesheets": {"type": "integer", "minimum": 1, "default": 12}, + "policy": {"type": "string"}, + }, + "required": ["domain_or_url", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="image_pack", + description="Build a bounded visual-asset manifest through the common workflow protocol.", + annotations=ToolAnnotations( + title="Build an image pack", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "url_or_pack": {"type": "string"}, + "output_dir": {"type": "string"}, + "download_assets": {"type": "boolean", "default": True}, + "max_assets": {"type": "integer", "minimum": 1, "default": 40}, + "policy": {"type": "string"}, + }, + "required": ["url_or_pack", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="screenshot_pack", + description=( + "Capture a screenshot pack through the explicitly trusted local browser gate and common " + "workflow protocol." + ), + annotations=ToolAnnotations( + title="Capture a screenshot pack", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "url": {"type": "string", "pattern": "^https://"}, + "output_dir": {"type": "string"}, + "viewport": {"type": "string", "default": "1280x720"}, + "full_page": {"type": "boolean", "default": False}, + "wait_for": { + "type": "string", + "enum": ["load", "domcontentloaded", "networkidle"], + "default": "load", + }, + "agent_browser_binary": {"type": "string"}, + "policy": {"type": "string"}, + }, + "required": ["url", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="policy_pack", + description=( + "Discover policy documents and emit stable clause evidence and textual change " + "candidates; " + "does not provide legal conclusions." + ), + annotations=ToolAnnotations( + title="Build a policy evidence pack", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=False, + openWorldHint=True, + ), + inputSchema={ + "type": "object", + "properties": { + "domain_or_url": {"type": "string"}, + "output_dir": {"type": "string"}, + "max_pages": {"type": "integer", "minimum": 1, "default": 16}, + "baseline_pack": {"type": "string"}, + "policy": {"type": "string"}, + }, + "required": ["domain_or_url", "output_dir"], + }, + outputSchema=_WORKFLOW_RESULT_OUTPUT_SCHEMA, + ), + Tool( + name="intelligence_bundle", + description=( + "Generate a deterministic intelligence.bundle.v1 tracker import with source snapshots, " + "document versions, precise observations, and change candidates." + ), + annotations=ToolAnnotations( + title="Generate an intelligence bundle", + readOnlyHint=False, + destructiveHint=False, + idempotentHint=True, + openWorldHint=False, + ), + inputSchema={ + "type": "object", + "properties": { + "pack_dir": {"type": "string"}, + "objective": {"type": "string"}, + "market": {"type": "string"}, + "search_queries": {"type": "array", "items": {"type": "string"}}, + "required_domains": {"type": "array", "items": {"type": "string"}}, + "max_excerpts": {"type": "integer", "minimum": 1, "default": 8}, + "entity_limit": {"type": "integer", "minimum": 0, "default": 20}, + "search_limit": {"type": "integer", "minimum": 1, "default": 10}, + "output": {"type": "string"}, + }, + "required": ["pack_dir"], + }, + outputSchema=_INTELLIGENCE_BUNDLE_OUTPUT_SCHEMA, + ), Tool( name="pack_score", description="Score a docpull context pack for agent-readiness without shelling out.", diff --git a/src/docpull/pack_tools.py b/src/docpull/pack_tools.py index e02b9b2..f685e05 100644 --- a/src/docpull/pack_tools.py +++ b/src/docpull/pack_tools.py @@ -13,6 +13,9 @@ from rich.console import Console from rich.markup import escape +from .change_events import build_change_events, write_change_events +from .contracts import canonical_sha256, stable_id +from .evidence import classify_source_authority, evidence_span_payload from .output_contract import ( VALIDATION_LEVELS, OutputContractError, @@ -285,6 +288,27 @@ def create_pack_parser() -> argparse.ArgumentParser: ) prepare.add_argument("--no-markdown", action="store_true", help="Write JSON artifacts only") + for command_name in ("intelligence-bundle", "company-brain"): + bundle = subparsers.add_parser( + command_name, + help=( + "Write deterministic intelligence.bundle.v1 tracker import" + if command_name == "intelligence-bundle" + else "Compatibility alias for intelligence-bundle" + ), + ) + bundle.add_argument("pack_dir", type=Path, help="Context pack directory") + bundle.add_argument("--objective") + bundle.add_argument("--market") + bundle.add_argument("--search-query", action="append", dest="search_queries", default=[]) + bundle.add_argument("--no-search", action="store_true") + bundle.add_argument("--require-domain", action="append", dest="required_domains", default=[]) + bundle.add_argument("--max-excerpts", type=int, default=DEFAULT_BRIEF_EXCERPTS) + bundle.add_argument("--entity-limit", type=int, default=DEFAULT_BRIEF_ENTITY_LIMIT) + bundle.add_argument("--search-limit", type=int, default=DEFAULT_SEARCH_LIMIT) + bundle.add_argument("--output", type=Path) + bundle.add_argument("--markdown", type=Path) + return parser @@ -386,6 +410,10 @@ def run_pack_cli(argv: list[str] | None = None) -> int: payload = diff_packs(args.old_pack_dir, args.new_pack_dir) output = args.output or (args.new_pack_dir / "pack.diff.json") _write_json(output, payload) + write_change_events( + args.new_pack_dir / "change.events.jsonl", + list(payload.get("change_events") or []), + ) semantic = payload.get("semantic_diff") if isinstance(semantic, dict): _write_json(args.new_pack_dir / "semantic.diff.json", semantic) @@ -491,6 +519,40 @@ def run_pack_cli(argv: list[str] | None = None) -> int: f"{payload['artifacts']['prepare']}" ) return 0 + if args.command in {"intelligence-bundle", "company-brain"}: + if args.command == "company-brain": + payload = build_company_brain_bundle( + args.pack_dir, + objective=args.objective, + market=args.market, + search_queries=[] if args.no_search else (args.search_queries or None), + default_search=not args.no_search, + required_domains=args.required_domains, + max_excerpts=args.max_excerpts, + entity_limit=args.entity_limit, + search_limit=args.search_limit, + output=args.output, + markdown_path=args.markdown, + ) + else: + payload = build_intelligence_bundle( + args.pack_dir, + objective=args.objective, + market=args.market, + search_queries=[] if args.no_search else (args.search_queries or None), + default_search=not args.no_search, + required_domains=args.required_domains, + max_excerpts=args.max_excerpts, + entity_limit=args.entity_limit, + search_limit=args.search_limit, + output=args.output, + markdown_path=args.markdown, + ) + console.print( + f"[green]Intelligence bundle:[/green] {payload['bundle_id']} -> " + f"{payload['artifacts']['intelligence_bundle']}" + ) + return 0 parser.error(f"Unknown command: {args.command}") except (PackToolError, OutputContractError) as err: console.print("[red]Pack error:[/red] " + escape(str(err))) @@ -768,7 +830,7 @@ def build_research_brief( } -def build_company_brain_bundle( +def build_intelligence_bundle( pack_dir: Path, *, objective: str | None = None, @@ -780,9 +842,10 @@ def build_company_brain_bundle( entity_limit: int = DEFAULT_BRIEF_ENTITY_LIMIT, search_limit: int = DEFAULT_SEARCH_LIMIT, output: Path | None = None, + compatibility_output: Path | None = None, markdown_path: Path | None = None, ) -> dict[str, Any]: - """Write an app-ready Company Brain import bundle from a local context pack.""" + """Write a deterministic ``intelligence.bundle.v1`` tracker import.""" if max_excerpts < 1: raise PackToolError("--max-excerpts must be at least 1.") if entity_limit < 0: @@ -838,19 +901,58 @@ def build_company_brain_bundle( artifacts=artifacts, ) + records = _read_pack_records(pack_dir) source_snapshots = _company_brain_source_snapshots(citations_payload) claims = _company_brain_claims(brief_payload) entities = _company_brain_entities(entities_payload) signals = _company_brain_signals(search_payloads) - bundle_path = (output or (pack_dir / "company_brain.bundle.json")).resolve() + bundle_path = (output or (pack_dir / "intelligence.bundle.v1.json")).resolve() + compatibility_path = ( + compatibility_output.resolve() + if compatibility_output + else (pack_dir / "company_brain.bundle.json").resolve() + ) summary_path = (markdown_path or (pack_dir / "COMPANY_BRAIN.md")).resolve() - artifacts["bundle"] = _artifact_ref(pack_dir, bundle_path) + artifacts["intelligence_bundle"] = _artifact_ref(pack_dir, bundle_path) + artifacts["bundle"] = _artifact_ref(pack_dir, compatibility_path) + artifacts["company_brain_compatibility_alias"] = _artifact_ref(pack_dir, compatibility_path) artifacts["company_brain_markdown"] = _artifact_ref(pack_dir, summary_path) - - payload = { + content_hashes = sorted( + str(record.get("content_hash")) for record in records if record.get("content_hash") + ) + pack_identity = { + "pack_id": stable_id("pack", {"content_hashes": content_hashes}), + "content_hash": canonical_sha256(content_hashes), + "record_count": len(records), + } + run_seed = { + "pack_id": pack_identity["pack_id"], + "objective": brain_objective, + "market": workspace_label, + "queries": queries, + } + run_identity = { + "run_id": stable_id("run", run_seed), + "scheduler": None, + "replay": { + "objective": brain_objective, + "market": workspace_label, + "search_queries": queries, + "required_domains": sorted(required_domains or []), + "max_excerpts": max_excerpts, + "entity_limit": entity_limit, + "search_limit": search_limit, + }, + } + canonical_snapshots = _intelligence_source_snapshots(source_snapshots, records) + observations = _intelligence_observations(claims, records, citations_payload) + document_versions = _intelligence_document_versions(records) + change_candidates = _intelligence_change_candidates(pack_dir) + bundle_core = { + "contract_version": "intelligence.bundle.v1", "schema_version": COMPANY_BRAIN_SCHEMA_VERSION, - "generated_at": utc_now_iso(), - "pack_dir": str(pack_dir), + "pack_identity": pack_identity, + "run_identity": run_identity, "workspace": { "name": workspace_label, "market": workspace_label, @@ -869,6 +971,20 @@ def build_company_brain_bundle( ), "expected_domains": citations_payload["expected_domains"], }, + "source_snapshots": canonical_snapshots, + "document_versions": document_versions, + "observations": observations, + "change_candidates": change_candidates, + "warnings": _intelligence_warnings(score_payload, source_scores_payload), + "artifacts": artifacts, + } + bundle_hash = canonical_sha256(bundle_core) + payload = { + **bundle_core, + "bundle_id": f"bundle_{bundle_hash[:24]}", + "bundle_hash": bundle_hash, + # Compatibility envelope for company_brain.bundle.json readers. + "pack_dir": str(pack_dir), "records": { "sources": citations_payload["sources"], "source_snapshots": source_snapshots, @@ -881,14 +997,14 @@ def build_company_brain_bundle( "key_excerpts": brief_payload["key_excerpts"], }, "gate_inputs": { - "pack_score": score_payload, - "source_scores": source_scores_payload, + "pack_score": _deterministic_contract_value(score_payload), + "source_scores": _deterministic_contract_value(source_scores_payload), "required_domains": citations_payload["expected_domains"], "claim_policy": "Every promoted claim must carry a citation_id and source URL.", }, }, "agent_run_seed": { - "trigger": "docpull.pack.company-brain", + "trigger": "docpull.pack.intelligence-bundle", "tool": "docpull", "status": "ready_for_control_plane_import", "policy_checks": [ @@ -898,14 +1014,46 @@ def build_company_brain_bundle( "pack_integrity_score", ], }, - "artifacts": artifacts, } summary_path.parent.mkdir(parents=True, exist_ok=True) summary_path.write_text(_company_brain_markdown(payload), encoding="utf-8") _write_json(bundle_path, payload) + if compatibility_path != bundle_path: + _write_json(compatibility_path, payload) return payload +def build_company_brain_bundle( + pack_dir: Path, + *, + objective: str | None = None, + market: str | None = None, + search_queries: list[str] | None = None, + default_search: bool = True, + required_domains: list[str] | None = None, + max_excerpts: int = DEFAULT_BRIEF_EXCERPTS, + entity_limit: int = DEFAULT_BRIEF_ENTITY_LIMIT, + search_limit: int = DEFAULT_SEARCH_LIMIT, + output: Path | None = None, + markdown_path: Path | None = None, +) -> dict[str, Any]: + """Compatibility alias for :func:`build_intelligence_bundle`.""" + + return build_intelligence_bundle( + pack_dir, + objective=objective, + market=market, + search_queries=search_queries, + default_search=default_search, + required_domains=required_domains, + max_excerpts=max_excerpts, + entity_limit=entity_limit, + search_limit=search_limit, + compatibility_output=output, + markdown_path=markdown_path, + ) + + def prepare_pack( pack_dir: Path, *, @@ -1368,6 +1516,179 @@ def _company_brain_signals(search_payloads: list[dict[str, Any]]) -> list[dict[s return signals +def _intelligence_source_snapshots( + snapshots: list[dict[str, Any]], + records: list[dict[str, Any]], +) -> list[dict[str, Any]]: + official_domain = str(snapshots[0].get("domain") or "") if snapshots else "" + records_by_url = { + str(record.get("url")): record for record in records if isinstance(record, dict) and record.get("url") + } + output: list[dict[str, Any]] = [] + for snapshot in snapshots: + url = str(snapshot.get("url") or "") + record = records_by_url.get(url, {}) + content_hashes = sorted(str(item) for item in snapshot.get("content_hashes") or []) + output.append( + { + "source_snapshot_id": str(snapshot.get("source_snapshot_id") or ""), + "source_id": str(snapshot.get("source_id") or ""), + "url": url, + "document_id": record.get("document_id"), + "document_version": record.get("content_hash"), + "content_hash": canonical_sha256(content_hashes) if content_hashes else None, + "fetched_at": snapshot.get("latest_fetched_at"), + "authority": classify_source_authority( + url, + official_domain=official_domain, + ).model_dump(mode="json"), + } + ) + return output + + +def _intelligence_document_versions(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + return sorted( + [ + { + "document_id": record.get("document_id"), + "document_version": record.get("content_hash"), + "url": record.get("url"), + "fetched_at": record.get("fetched_at"), + "title": record.get("title"), + } + for record in records + if record.get("document_id") and record.get("content_hash") and record.get("url") + ], + key=lambda item: (str(item["url"]), str(item["document_id"])), + ) + + +def _intelligence_observations( + claims: list[dict[str, Any]], + records: list[dict[str, Any]], + citations_payload: dict[str, Any], +) -> list[dict[str, Any]]: + expected_domains = [str(item) for item in citations_payload.get("expected_domains") or []] + official_domain = expected_domains[0] if expected_domains else "" + by_url = { + str(record.get("url")): record for record in records if isinstance(record, dict) and record.get("url") + } + observations: list[dict[str, Any]] = [] + for claim in claims: + url = str(claim.get("url") or "") + text = str(claim.get("text") or "").strip() + record = by_url.get(url) + if not url or not text or record is None: + continue + content = str(record.get("content") or "") + authority = classify_source_authority(url, official_domain=official_domain) + evidence = evidence_span_payload( + url=url, + content=content, + exact_text=text, + citation_id=str(claim.get("citation_id") or "S0"), + record_citation_id=( + str(claim.get("record_citation_id")) if claim.get("record_citation_id") else None + ), + ) + observation_seed = { + "type": "source_excerpt", + "text": text, + "document_version": evidence["document_version"], + "char_start": evidence["char_start"], + "char_end": evidence["char_end"], + } + observations.append( + { + "observation_id": stable_id("observation", observation_seed), + "type": "source_excerpt", + "text": text, + "status": "observation", + "evidence_strength": "strong" if claim.get("citation_id") else "moderate", + "confidence": 0.9 if claim.get("citation_id") else 0.65, + "source_authority": authority.model_dump(mode="json"), + "evidence": [evidence], + "warnings": [], + } + ) + return observations + + +def _intelligence_change_candidates(pack_dir: Path) -> list[dict[str, Any]]: + path = pack_dir / "change.events.jsonl" + if not path.exists(): + return [] + candidates: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + classifications = [str(item) for item in event.get("classifications") or []] + classification = classifications[0] if classifications else "other" + if classification not in {"pricing", "positioning", "product", "security", "policy", "other"}: + classification = "other" + candidate = { + "classification": classification, + "status": "candidate", + "before": list(event.get("old_evidence") or []), + "after": list(event.get("new_evidence") or []), + "confidence": 0.7, + "warnings": ["Semantic classification is a review candidate, not an approved claim."], + } + candidate["change_candidate_id"] = stable_id("change_candidate", candidate) + candidates.append(candidate) + return candidates + + +def _intelligence_warnings( + score_payload: dict[str, Any], + source_scores_payload: dict[str, Any], +) -> list[dict[str, Any]]: + warnings: list[dict[str, Any]] = [] + for item in [*list(score_payload.get("issues") or []), *list(score_payload.get("warnings") or [])]: + if not isinstance(item, dict): + continue + warnings.append( + { + "code": str(item.get("code") or "pack_quality"), + "message": str(item.get("message") or "Pack quality warning"), + "metadata": {"severity": item.get("severity")}, + } + ) + weak_sources = [ + source + for source in source_scores_payload.get("sources") or [] + if isinstance(source, dict) and _safe_int(source.get("score")) < 60 + ] + if weak_sources: + warnings.append( + { + "code": "weak_source_authority", + "message": f"{len(weak_sources)} source records scored below 60.", + "metadata": {}, + } + ) + return warnings + + +def _deterministic_contract_value(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): _deterministic_contract_value(item) + for key, item in value.items() + if key not in {"generated_at", "pack_dir", "output_dir"} + } + if isinstance(value, list): + return [_deterministic_contract_value(item) for item in value] + return value + + def search_pack( pack_dir: Path, query: str, @@ -1456,6 +1777,13 @@ def diff_packs(old_pack_dir: Path, new_pack_dir: Path) -> dict[str, Any]: new_pack_dir, diff_payload=payload, ) + change_events = build_change_events( + old_records, + new_records, + workflow="pack-diff", + ) + payload["change_events"] = change_events + payload["change_event_count"] = len(change_events) return payload diff --git a/src/docpull/project.py b/src/docpull/project.py index 2cc2b7c..03fd2a8 100644 --- a/src/docpull/project.py +++ b/src/docpull/project.py @@ -87,6 +87,11 @@ "dataset", "transcript", "wiki", + "brand", + "product", + "styleguide", + "visual", + "policy", ] OutputFormat = Literal["markdown", "ndjson", "sqlite", "context-pack"] SemanticMode = Literal["auto", "off", "on"] @@ -104,6 +109,11 @@ "dataset", "transcript", "wiki", + "brand", + "product", + "styleguide", + "visual", + "policy", ) SOURCE_TYPES: tuple[str, ...] = (*CRAWL_SOURCE_TYPES, *TYPED_PROJECT_SOURCE_TYPES) CONTEXT_TARGETS: tuple[str, ...] = ("cursor", "claude", "codex", "openai", "llamaindex", "langchain") @@ -1174,6 +1184,7 @@ def diff_project( "unchanged_count": len(base["unchanged_urls"]), "likely_api_behavior_change_count": len(likely_api), "pricing_change_count": len(pricing), + "change_event_count": len(base.get("change_events") or []), }, "likely_api_behavior_changes": likely_api, "pricing_changes": pricing, @@ -1182,10 +1193,14 @@ def diff_project( } diff_path = new_run_dir / "project.diff.json" semantic_diff_path = new_run_dir / "semantic.diff.json" + change_events_path = new_run_dir / "change.events.jsonl" markdown_path = new_run_dir / "PROJECT_DIFF.md" semantic_diff_payload = base.get("semantic_diff") if isinstance(semantic_diff_payload, dict): _write_json(semantic_diff_path, semantic_diff_payload) + from .change_events import write_change_events + + write_change_events(change_events_path, list(payload.get("change_events") or [])) _write_json(diff_path, payload) markdown_path.write_text(_project_diff_markdown(payload), encoding="utf-8") _index_diff(project_root, payload) @@ -1450,6 +1465,7 @@ def review_project_run(*, run_id: str | None = None, root: Path | None = None) - "removed_count": _safe_int(diff_summary.get("removed_count")), "likely_api_behavior_change_count": _safe_int(diff_summary.get("likely_api_behavior_change_count")), "pricing_change_count": _safe_int(diff_summary.get("pricing_change_count")), + "change_event_count": _safe_int(diff_summary.get("change_event_count")), } health = _read_json(run_dir / "source-health.json", default={"sources": []}) errors = _read_jsonl(run_dir / "errors.jsonl") @@ -1868,18 +1884,62 @@ def _sync_typed_project_source( chunk_tokens=DEFAULT_CHUNK_TOKENS, cache_dir=cache_dir, ) + elif source.type == "brand": + from .context_packs.brand import build_brand_pack + + build_brand_pack( + source_spec, + output_dir=output_dir, + download_assets=False, + max_pages=min(max_items, 6), + ) + elif source.type == "product": + from .context_packs.product import build_product_pack + + build_product_pack( + source_spec, + mode="site", + output_dir=output_dir, + max_pages=min(max_items, 8), + ) + elif source.type == "styleguide": + from .context_packs.styleguide import build_styleguide_pack + + build_styleguide_pack( + source_spec, + output_dir=output_dir, + render=False, + ) + elif source.type == "visual": + from .context_packs.visuals import build_image_pack + + build_image_pack( + source_spec, + output_dir=output_dir, + download_assets=False, + max_assets=min(max_items, 40), + ) + elif source.type == "policy": + from .context_packs.policy_pack import build_policy_pack + + build_policy_pack( + source_spec, + output_dir=output_dir, + max_pages=min(max_items, 16), + ) else: raise ProjectError(f"Unsupported typed project source type: {source.type}") records_path = output_dir / "documents.ndjson" records = _read_jsonl(records_path) if records_path.exists() else [] normalized = [_normalize_project_record(record, source) for record in records] + accounting_payload = _read_json(output_dir / "run.accounting.json", default={}) stats = { "pages_fetched": len(normalized), "pages_failed": 0, "pages_skipped": 0, "robots_blocked": 0, - "http_request_count": 0, + "http_request_count": _safe_int(accounting_payload.get("http_request_count")), } return { "records": normalized, @@ -2835,6 +2895,8 @@ def _typed_project_source_spec_allowed(source_type: str, value: str) -> bool: return not parsed.scheme if source_type == "wiki": return _is_wiki_page_url(parsed) or lowered.startswith(("wiki:", "wikipedia:")) + if source_type in {"brand", "product", "styleguide", "visual", "policy"}: + return is_https or bool(re.fullmatch(r"[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text)) return False diff --git a/src/docpull/schemas/artifact-manifest.v1.schema.json b/src/docpull/schemas/artifact-manifest.v1.schema.json new file mode 100644 index 0000000..b98f5bf --- /dev/null +++ b/src/docpull/schemas/artifact-manifest.v1.schema.json @@ -0,0 +1,99 @@ +{ + "$defs": { + "ArtifactEntry": { + "additionalProperties": true, + "properties": { + "bytes": { + "title": "Bytes", + "type": "integer" + }, + "media_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Media Type" + }, + "name": { + "title": "Name", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "role": { + "title": "Role", + "type": "string" + }, + "sha256": { + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "name", + "path", + "role", + "bytes", + "sha256" + ], + "title": "ArtifactEntry", + "type": "object" + } + }, + "$id": "https://docpull.dev/schemas/artifact-manifest.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "aggregate_sha256": { + "title": "Aggregate Sha256", + "type": "string" + }, + "contract_version": { + "const": "artifact.manifest.v1", + "default": "artifact.manifest.v1", + "title": "Contract Version", + "type": "string" + }, + "entries": { + "items": { + "$ref": "#/$defs/ArtifactEntry" + }, + "title": "Entries", + "type": "array" + }, + "hash_algorithm": { + "const": "sha256", + "default": "sha256", + "title": "Hash Algorithm", + "type": "string" + }, + "pack_id": { + "title": "Pack Id", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "schema_version": { + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "pack_id", + "run_id", + "entries", + "aggregate_sha256" + ], + "title": "ArtifactManifest", + "type": "object" +} diff --git a/src/docpull/schemas/basis.v2.schema.json b/src/docpull/schemas/basis.v2.schema.json new file mode 100644 index 0000000..38b3262 --- /dev/null +++ b/src/docpull/schemas/basis.v2.schema.json @@ -0,0 +1,86 @@ +{ + "$id": "https://docpull.dev/schemas/basis.v2.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "basis_id": { + "title": "Basis Id", + "type": "string" + }, + "citation_ids": { + "items": { + "type": "string" + }, + "title": "Citation Ids", + "type": "array" + }, + "claim": { + "title": "Claim", + "type": "string" + }, + "claim_path": { + "title": "Claim Path", + "type": "string" + }, + "confidence": { + "enum": [ + "high", + "medium", + "low" + ], + "title": "Confidence", + "type": "string" + }, + "evidence_state": { + "enum": [ + "supported", + "partial", + "insufficient" + ], + "title": "Evidence State", + "type": "string" + }, + "excerpts": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Excerpts", + "type": "array" + }, + "producer": { + "title": "Producer", + "type": "string" + }, + "schema_version": { + "const": 2, + "default": 2, + "title": "Schema Version", + "type": "integer" + }, + "source_urls": { + "items": { + "type": "string" + }, + "title": "Source Urls", + "type": "array" + }, + "warnings": { + "items": { + "type": "string" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "basis_id", + "claim_path", + "claim", + "evidence_state", + "confidence", + "producer" + ], + "title": "BasisContractV2", + "type": "object" +} diff --git a/src/docpull/schemas/change-event.v1.schema.json b/src/docpull/schemas/change-event.v1.schema.json new file mode 100644 index 0000000..7bea361 --- /dev/null +++ b/src/docpull/schemas/change-event.v1.schema.json @@ -0,0 +1,243 @@ +{ + "$defs": { + "EvidenceSpan": { + "additionalProperties": true, + "properties": { + "char_end": { + "minimum": 0, + "title": "Char End", + "type": "integer" + }, + "char_start": { + "minimum": 0, + "title": "Char Start", + "type": "integer" + }, + "citation_id": { + "title": "Citation Id", + "type": "string" + }, + "document_id": { + "title": "Document Id", + "type": "string" + }, + "document_version": { + "title": "Document Version", + "type": "string" + }, + "exact_text": { + "title": "Exact Text", + "type": "string" + }, + "exact_text_sha256": { + "title": "Exact Text Sha256", + "type": "string" + }, + "record_citation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Record Citation Id" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "citation_id", + "document_id", + "document_version", + "url", + "char_start", + "char_end", + "exact_text", + "exact_text_sha256" + ], + "title": "EvidenceSpan", + "type": "object" + }, + "ReplayConfiguration": { + "additionalProperties": true, + "properties": { + "browser_enabled": { + "default": false, + "title": "Browser Enabled", + "type": "boolean" + }, + "configuration": { + "additionalProperties": true, + "title": "Configuration", + "type": "object" + }, + "local_first": { + "default": true, + "title": "Local First", + "type": "boolean" + }, + "paid_routes_enabled": { + "default": false, + "title": "Paid Routes Enabled", + "type": "boolean" + }, + "scheduler": { + "default": null, + "title": "Scheduler", + "type": "null" + } + }, + "title": "ReplayConfiguration", + "type": "object" + } + }, + "$id": "https://docpull.dev/schemas/change-event.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "classifications": { + "items": { + "enum": [ + "pricing", + "positioning", + "product", + "security", + "policy", + "other" + ], + "type": "string" + }, + "title": "Classifications", + "type": "array" + }, + "contract_version": { + "const": "change.event.v1", + "default": "change.event.v1", + "title": "Contract Version", + "type": "string" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "idempotency_key": { + "title": "Idempotency Key", + "type": "string" + }, + "new_document_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "New Document Id" + }, + "new_evidence": { + "items": { + "$ref": "#/$defs/EvidenceSpan" + }, + "title": "New Evidence", + "type": "array" + }, + "new_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "New Hash" + }, + "old_document_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Old Document Id" + }, + "old_evidence": { + "items": { + "$ref": "#/$defs/EvidenceSpan" + }, + "title": "Old Evidence", + "type": "array" + }, + "old_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Old Hash" + }, + "replay_configuration": { + "$ref": "#/$defs/ReplayConfiguration" + }, + "schema_version": { + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "semantic_candidates": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Semantic Candidates", + "type": "array" + }, + "structural_changes": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Structural Changes", + "type": "array" + }, + "textual_changes": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Textual Changes", + "type": "array" + }, + "url": { + "title": "Url", + "type": "string" + }, + "workflow": { + "title": "Workflow", + "type": "string" + } + }, + "required": [ + "event_id", + "idempotency_key", + "workflow", + "url" + ], + "title": "ChangeEvent", + "type": "object" +} diff --git a/src/docpull/schemas/citation-map.v1.schema.json b/src/docpull/schemas/citation-map.v1.schema.json new file mode 100644 index 0000000..76fa4b3 --- /dev/null +++ b/src/docpull/schemas/citation-map.v1.schema.json @@ -0,0 +1,30 @@ +{ + "$id": "https://docpull.dev/schemas/citation-map.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "schema_version": { + "title": "Schema Version", + "type": "integer" + }, + "source_count": { + "title": "Source Count", + "type": "integer" + }, + "sources": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Sources", + "type": "array" + } + }, + "required": [ + "schema_version", + "source_count", + "sources" + ], + "title": "CitationMapContractV1", + "type": "object" +} diff --git a/src/docpull/schemas/document.v3.schema.json b/src/docpull/schemas/document.v3.schema.json new file mode 100644 index 0000000..d9a8809 --- /dev/null +++ b/src/docpull/schemas/document.v3.schema.json @@ -0,0 +1,275 @@ +{ + "$id": "https://docpull.dev/schemas/document.v3.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Versioned logical document shape independent of output container.", + "properties": { + "accession_number": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Accession Number" + }, + "chunk_heading": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chunk Heading" + }, + "chunk_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chunk Id" + }, + "chunk_index": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chunk Index" + }, + "cik": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Cik" + }, + "content": { + "title": "Content", + "type": "string" + }, + "content_hash": { + "title": "Content Hash", + "type": "string" + }, + "content_type": { + "default": "text/markdown", + "title": "Content Type", + "type": "string" + }, + "document_id": { + "title": "Document Id", + "type": "string" + }, + "extraction": { + "additionalProperties": true, + "title": "Extraction", + "type": "object" + }, + "fetched_at": { + "title": "Fetched At", + "type": "string" + }, + "filing_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Filing Date" + }, + "form": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Form" + }, + "issuer_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Issuer Name" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "mime_type": { + "default": "text/markdown", + "title": "Mime Type", + "type": "string" + }, + "primary_document_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Primary Document Url" + }, + "record_citation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Record Citation Id" + }, + "rendered_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rendered At" + }, + "retrieved_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Retrieved At" + }, + "rights": { + "additionalProperties": true, + "title": "Rights", + "type": "object" + }, + "route": { + "additionalProperties": true, + "title": "Route", + "type": "object" + }, + "run": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Run" + }, + "schema_version": { + "default": 3, + "title": "Schema Version", + "type": "integer" + }, + "source_citation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Citation Id" + }, + "source_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Type" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Title" + }, + "token_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Token Count" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "document_id", + "url", + "content", + "content_hash" + ], + "title": "DocumentRecord", + "type": "object" +} diff --git a/src/docpull/schemas/intelligence-bundle.v1.schema.json b/src/docpull/schemas/intelligence-bundle.v1.schema.json new file mode 100644 index 0000000..f8622b1 --- /dev/null +++ b/src/docpull/schemas/intelligence-bundle.v1.schema.json @@ -0,0 +1,428 @@ +{ + "$defs": { + "ChangeCandidate": { + "additionalProperties": true, + "properties": { + "after": { + "items": { + "$ref": "#/$defs/EvidenceSpan" + }, + "title": "After", + "type": "array" + }, + "before": { + "items": { + "$ref": "#/$defs/EvidenceSpan" + }, + "title": "Before", + "type": "array" + }, + "change_candidate_id": { + "title": "Change Candidate Id", + "type": "string" + }, + "classification": { + "enum": [ + "pricing", + "positioning", + "product", + "security", + "policy", + "other" + ], + "title": "Classification", + "type": "string" + }, + "confidence": { + "maximum": 1.0, + "minimum": 0.0, + "title": "Confidence", + "type": "number" + }, + "status": { + "const": "candidate", + "default": "candidate", + "title": "Status", + "type": "string" + }, + "warnings": { + "items": { + "type": "string" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "change_candidate_id", + "classification", + "confidence" + ], + "title": "ChangeCandidate", + "type": "object" + }, + "EvidenceSpan": { + "additionalProperties": true, + "properties": { + "char_end": { + "minimum": 0, + "title": "Char End", + "type": "integer" + }, + "char_start": { + "minimum": 0, + "title": "Char Start", + "type": "integer" + }, + "citation_id": { + "title": "Citation Id", + "type": "string" + }, + "document_id": { + "title": "Document Id", + "type": "string" + }, + "document_version": { + "title": "Document Version", + "type": "string" + }, + "exact_text": { + "title": "Exact Text", + "type": "string" + }, + "exact_text_sha256": { + "title": "Exact Text Sha256", + "type": "string" + }, + "record_citation_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Record Citation Id" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "citation_id", + "document_id", + "document_version", + "url", + "char_start", + "char_end", + "exact_text", + "exact_text_sha256" + ], + "title": "EvidenceSpan", + "type": "object" + }, + "Observation": { + "additionalProperties": true, + "properties": { + "confidence": { + "maximum": 1.0, + "minimum": 0.0, + "title": "Confidence", + "type": "number" + }, + "evidence": { + "items": { + "$ref": "#/$defs/EvidenceSpan" + }, + "title": "Evidence", + "type": "array" + }, + "evidence_strength": { + "enum": [ + "strong", + "moderate", + "weak", + "unknown" + ], + "title": "Evidence Strength", + "type": "string" + }, + "observation_id": { + "title": "Observation Id", + "type": "string" + }, + "source_authority": { + "$ref": "#/$defs/SourceAuthority" + }, + "status": { + "const": "observation", + "default": "observation", + "title": "Status", + "type": "string" + }, + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + }, + "warnings": { + "items": { + "type": "string" + }, + "title": "Warnings", + "type": "array" + } + }, + "required": [ + "observation_id", + "type", + "text", + "evidence_strength", + "confidence", + "source_authority" + ], + "title": "Observation", + "type": "object" + }, + "SourceAuthority": { + "additionalProperties": true, + "properties": { + "rationale": { + "title": "Rationale", + "type": "string" + }, + "role": { + "enum": [ + "official_product", + "legal", + "documentation", + "social", + "marketplace", + "third_party" + ], + "title": "Role", + "type": "string" + }, + "tier": { + "enum": [ + "tier_1_authoritative", + "tier_2_owned", + "tier_3_distribution", + "tier_4_external" + ], + "title": "Tier", + "type": "string" + } + }, + "required": [ + "role", + "tier", + "rationale" + ], + "title": "SourceAuthority", + "type": "object" + }, + "SourceSnapshot": { + "additionalProperties": true, + "properties": { + "authority": { + "$ref": "#/$defs/SourceAuthority" + }, + "content_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content Hash" + }, + "document_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Document Id" + }, + "document_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Document Version" + }, + "fetched_at": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fetched At" + }, + "source_id": { + "title": "Source Id", + "type": "string" + }, + "source_snapshot_id": { + "title": "Source Snapshot Id", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "source_snapshot_id", + "source_id", + "url", + "authority" + ], + "title": "SourceSnapshot", + "type": "object" + }, + "WorkflowWarning": { + "additionalProperties": true, + "properties": { + "code": { + "title": "Code", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + } + }, + "required": [ + "code", + "message" + ], + "title": "WorkflowWarning", + "type": "object" + } + }, + "$id": "https://docpull.dev/schemas/intelligence-bundle.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "artifacts": { + "additionalProperties": { + "type": "string" + }, + "title": "Artifacts", + "type": "object" + }, + "bundle_hash": { + "title": "Bundle Hash", + "type": "string" + }, + "bundle_id": { + "title": "Bundle Id", + "type": "string" + }, + "change_candidates": { + "items": { + "$ref": "#/$defs/ChangeCandidate" + }, + "title": "Change Candidates", + "type": "array" + }, + "contract_version": { + "const": "intelligence.bundle.v1", + "default": "intelligence.bundle.v1", + "title": "Contract Version", + "type": "string" + }, + "document_versions": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Document Versions", + "type": "array" + }, + "observations": { + "items": { + "$ref": "#/$defs/Observation" + }, + "title": "Observations", + "type": "array" + }, + "pack_identity": { + "additionalProperties": true, + "title": "Pack Identity", + "type": "object" + }, + "run_identity": { + "additionalProperties": true, + "title": "Run Identity", + "type": "object" + }, + "schema_version": { + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "source_snapshots": { + "items": { + "$ref": "#/$defs/SourceSnapshot" + }, + "title": "Source Snapshots", + "type": "array" + }, + "summary": { + "additionalProperties": true, + "title": "Summary", + "type": "object" + }, + "warnings": { + "items": { + "$ref": "#/$defs/WorkflowWarning" + }, + "title": "Warnings", + "type": "array" + }, + "workspace": { + "additionalProperties": true, + "title": "Workspace", + "type": "object" + } + }, + "required": [ + "bundle_id", + "bundle_hash", + "pack_identity", + "run_identity", + "workspace" + ], + "title": "IntelligenceBundle", + "type": "object" +} diff --git a/src/docpull/schemas/pack.v3.schema.json b/src/docpull/schemas/pack.v3.schema.json new file mode 100644 index 0000000..224b7ed --- /dev/null +++ b/src/docpull/schemas/pack.v3.schema.json @@ -0,0 +1,44 @@ +{ + "$id": "https://docpull.dev/schemas/pack.v3.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "description": "Frozen compatibility envelope for existing ``*.pack.json`` files.", + "properties": { + "artifacts": { + "additionalProperties": true, + "title": "Artifacts", + "type": "object" + }, + "provider": { + "title": "Provider", + "type": "string" + }, + "schema_version": { + "title": "Schema Version", + "type": "integer" + }, + "status": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Status" + }, + "workflow": { + "title": "Workflow", + "type": "string" + } + }, + "required": [ + "schema_version", + "provider", + "workflow" + ], + "title": "PackContractV3", + "type": "object" +} diff --git a/src/docpull/schemas/provenance.v1.schema.json b/src/docpull/schemas/provenance.v1.schema.json new file mode 100644 index 0000000..4d6fca4 --- /dev/null +++ b/src/docpull/schemas/provenance.v1.schema.json @@ -0,0 +1,32 @@ +{ + "$id": "https://docpull.dev/schemas/provenance.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "edges": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Edges", + "type": "array" + }, + "nodes": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Nodes", + "type": "array" + }, + "schema_version": { + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "schema_version" + ], + "title": "ProvenanceContractV1", + "type": "object" +} diff --git a/src/docpull/schemas/rights.v1.schema.json b/src/docpull/schemas/rights.v1.schema.json new file mode 100644 index 0000000..25370bf --- /dev/null +++ b/src/docpull/schemas/rights.v1.schema.json @@ -0,0 +1,34 @@ +{ + "$id": "https://docpull.dev/schemas/rights.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "allowed_use": { + "additionalProperties": { + "type": "string" + }, + "title": "Allowed Use", + "type": "object" + }, + "basis": { + "title": "Basis", + "type": "string" + }, + "obligations": { + "items": {}, + "title": "Obligations", + "type": "array" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "allowed_use", + "basis" + ], + "title": "RightsContractV1", + "type": "object" +} diff --git a/src/docpull/schemas/run-identity.v1.schema.json b/src/docpull/schemas/run-identity.v1.schema.json new file mode 100644 index 0000000..d0cc92f --- /dev/null +++ b/src/docpull/schemas/run-identity.v1.schema.json @@ -0,0 +1,121 @@ +{ + "$id": "https://docpull.dev/schemas/run-identity.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Stable, non-secret description of the semantics of a docpull run.", + "properties": { + "auth_type": { + "title": "Auth Type", + "type": "string" + }, + "emit_chunks": { + "title": "Emit Chunks", + "type": "boolean" + }, + "enable_special_cases": { + "title": "Enable Special Cases", + "type": "boolean" + }, + "exclude_paths": { + "items": { + "type": "string" + }, + "title": "Exclude Paths", + "type": "array" + }, + "extractor": { + "title": "Extractor", + "type": "string" + }, + "include_paths": { + "items": { + "type": "string" + }, + "title": "Include Paths", + "type": "array" + }, + "max_depth": { + "title": "Max Depth", + "type": "integer" + }, + "max_pages": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Pages" + }, + "max_tokens_per_file": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Tokens Per File" + }, + "naming_strategy": { + "title": "Naming Strategy", + "type": "string" + }, + "output_format": { + "title": "Output Format", + "type": "string" + }, + "profile": { + "title": "Profile", + "type": "string" + }, + "rich_metadata": { + "title": "Rich Metadata", + "type": "boolean" + }, + "schema_version": { + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "start_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Start Url" + }, + "strict_js_required": { + "title": "Strict Js Required", + "type": "boolean" + }, + "tokenizer": { + "title": "Tokenizer", + "type": "string" + } + }, + "required": [ + "profile", + "max_depth", + "output_format", + "naming_strategy", + "rich_metadata", + "extractor", + "enable_special_cases", + "strict_js_required", + "emit_chunks", + "tokenizer", + "auth_type" + ], + "title": "RunIdentity", + "type": "object" +} diff --git a/src/docpull/schemas/workflow-request.v1.schema.json b/src/docpull/schemas/workflow-request.v1.schema.json new file mode 100644 index 0000000..67e9345 --- /dev/null +++ b/src/docpull/schemas/workflow-request.v1.schema.json @@ -0,0 +1,95 @@ +{ + "$defs": { + "ReplayConfiguration": { + "additionalProperties": true, + "properties": { + "browser_enabled": { + "default": false, + "title": "Browser Enabled", + "type": "boolean" + }, + "configuration": { + "additionalProperties": true, + "title": "Configuration", + "type": "object" + }, + "local_first": { + "default": true, + "title": "Local First", + "type": "boolean" + }, + "paid_routes_enabled": { + "default": false, + "title": "Paid Routes Enabled", + "type": "boolean" + }, + "scheduler": { + "default": null, + "title": "Scheduler", + "type": "null" + } + }, + "title": "ReplayConfiguration", + "type": "object" + } + }, + "$id": "https://docpull.dev/schemas/workflow-request.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "budget": { + "additionalProperties": true, + "title": "Budget", + "type": "object" + }, + "contract_version": { + "const": "workflow.request.v1", + "default": "workflow.request.v1", + "title": "Contract Version", + "type": "string" + }, + "input": { + "additionalProperties": true, + "title": "Input", + "type": "object" + }, + "options": { + "additionalProperties": true, + "title": "Options", + "type": "object" + }, + "output": { + "additionalProperties": true, + "title": "Output", + "type": "object" + }, + "replay": { + "$ref": "#/$defs/ReplayConfiguration" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "schema_version": { + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "source_policy": { + "additionalProperties": true, + "title": "Source Policy", + "type": "object" + }, + "workflow": { + "title": "Workflow", + "type": "string" + } + }, + "required": [ + "request_id", + "workflow", + "input" + ], + "title": "WorkflowRequest", + "type": "object" +} diff --git a/src/docpull/schemas/workflow-result.v1.schema.json b/src/docpull/schemas/workflow-result.v1.schema.json new file mode 100644 index 0000000..4117947 --- /dev/null +++ b/src/docpull/schemas/workflow-result.v1.schema.json @@ -0,0 +1,381 @@ +{ + "$defs": { + "BudgetUsage": { + "additionalProperties": true, + "properties": { + "actual_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Actual Usd" + }, + "blocked_actions": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Blocked Actions", + "type": "array" + }, + "cache_hit_count": { + "default": 0, + "title": "Cache Hit Count", + "type": "integer" + }, + "estimated_usd": { + "default": 0.0, + "title": "Estimated Usd", + "type": "number" + }, + "http_request_count": { + "default": 0, + "title": "Http Request Count", + "type": "integer" + }, + "limit_usd": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Limit Usd" + }, + "local_browser_seconds": { + "default": 0.0, + "title": "Local Browser Seconds", + "type": "number" + }, + "paid_request_count": { + "default": 0, + "title": "Paid Request Count", + "type": "integer" + } + }, + "title": "BudgetUsage", + "type": "object" + }, + "HashDigest": { + "additionalProperties": true, + "properties": { + "algorithm": { + "const": "sha256", + "default": "sha256", + "title": "Algorithm", + "type": "string" + }, + "digest": { + "title": "Digest", + "type": "string" + } + }, + "required": [ + "digest" + ], + "title": "HashDigest", + "type": "object" + }, + "ReplayConfiguration": { + "additionalProperties": true, + "properties": { + "browser_enabled": { + "default": false, + "title": "Browser Enabled", + "type": "boolean" + }, + "configuration": { + "additionalProperties": true, + "title": "Configuration", + "type": "object" + }, + "local_first": { + "default": true, + "title": "Local First", + "type": "boolean" + }, + "paid_routes_enabled": { + "default": false, + "title": "Paid Routes Enabled", + "type": "boolean" + }, + "scheduler": { + "default": null, + "title": "Scheduler", + "type": "null" + } + }, + "title": "ReplayConfiguration", + "type": "object" + }, + "WorkflowFailure": { + "additionalProperties": true, + "properties": { + "code": { + "default": "workflow_error", + "title": "Code", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "retryable": { + "default": false, + "title": "Retryable", + "type": "boolean" + }, + "source_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Url" + } + }, + "required": [ + "message" + ], + "title": "WorkflowFailure", + "type": "object" + }, + "WorkflowProgressEvent": { + "additionalProperties": true, + "properties": { + "current": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Current" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Message" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "phase": { + "title": "Phase", + "type": "string" + }, + "status": { + "enum": [ + "started", + "progress", + "completed", + "warning", + "failed" + ], + "title": "Status", + "type": "string" + }, + "timestamp": { + "title": "Timestamp", + "type": "string" + }, + "total": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Total" + } + }, + "required": [ + "event_id", + "phase", + "status", + "timestamp" + ], + "title": "WorkflowProgressEvent", + "type": "object" + }, + "WorkflowWarning": { + "additionalProperties": true, + "properties": { + "code": { + "title": "Code", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + } + }, + "required": [ + "code", + "message" + ], + "title": "WorkflowWarning", + "type": "object" + } + }, + "$id": "https://docpull.dev/schemas/workflow-result.v1.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "properties": { + "artifact_manifest": { + "default": "artifact.manifest.json", + "title": "Artifact Manifest", + "type": "string" + }, + "budget_usage": { + "$ref": "#/$defs/BudgetUsage" + }, + "compatibility_artifacts": { + "additionalProperties": { + "type": "string" + }, + "title": "Compatibility Artifacts", + "type": "object" + }, + "contract_version": { + "const": "workflow.result.v1", + "default": "workflow.result.v1", + "title": "Contract Version", + "type": "string" + }, + "data": { + "additionalProperties": true, + "title": "Data", + "type": "object" + }, + "failures": { + "items": { + "$ref": "#/$defs/WorkflowFailure" + }, + "title": "Failures", + "type": "array" + }, + "finished_at": { + "title": "Finished At", + "type": "string" + }, + "hashes": { + "additionalProperties": { + "$ref": "#/$defs/HashDigest" + }, + "title": "Hashes", + "type": "object" + }, + "pack_identity": { + "additionalProperties": true, + "title": "Pack Identity", + "type": "object" + }, + "progress_events": { + "items": { + "$ref": "#/$defs/WorkflowProgressEvent" + }, + "title": "Progress Events", + "type": "array" + }, + "replay_configuration": { + "$ref": "#/$defs/ReplayConfiguration" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "run_identity": { + "additionalProperties": true, + "title": "Run Identity", + "type": "object" + }, + "schema_version": { + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "started_at": { + "title": "Started At", + "type": "string" + }, + "status": { + "enum": [ + "completed", + "completed_with_warnings", + "failed", + "cancelled" + ], + "title": "Status", + "type": "string" + }, + "summary": { + "additionalProperties": true, + "title": "Summary", + "type": "object" + }, + "warnings": { + "items": { + "$ref": "#/$defs/WorkflowWarning" + }, + "title": "Warnings", + "type": "array" + }, + "workflow": { + "title": "Workflow", + "type": "string" + } + }, + "required": [ + "request_id", + "workflow", + "status", + "started_at", + "finished_at", + "pack_identity", + "run_identity" + ], + "title": "WorkflowResult", + "type": "object" +} diff --git a/src/docpull/surface.py b/src/docpull/surface.py index a988ca3..7a12345 100644 --- a/src/docpull/surface.py +++ b/src/docpull/surface.py @@ -43,11 +43,18 @@ class CliCommand: CliCommand("dataset-pack", "Build a local v3 pack from local dataset files"), CliCommand("transcript-pack", "Build a local v3 pack from transcript files or URLs"), CliCommand("wiki-pack", "Build a local v3 pack from Wikimedia/MediaWiki REST page content"), + CliCommand("brand-pack", "Build an evidence-backed brand knowledge pack"), + CliCommand("product-pack", "Build cited product and pricing evidence"), + CliCommand("styleguide-pack", "Build a cited design-token and styleguide pack"), + CliCommand("image-pack", "Build a bounded visual asset manifest"), + CliCommand("screenshot-pack", "Capture an explicitly enabled local screenshot pack"), + CliCommand("policy-pack", "Discover policies and emit clause-level change evidence"), CliCommand("mcp", "Run the stdio MCP server"), CliCommand("policy", "Validate or explain source policy files"), CliCommand("auth", "Check authenticated source access without writing content"), CliCommand("refresh", "Refresh an existing local pack and write change reports"), CliCommand("pack", "Score, diff, audit, search, cite, and prepare context packs"), + CliCommand("contracts", "List or export versioned cross-repository JSON Schemas"), CliCommand("graph", "Build and query local cited source graphs for context packs"), CliCommand("export", "Export a pack for agent or RAG tools"), CliCommand("serve", "Serve a local pack over localhost JSON routes"), @@ -88,12 +95,7 @@ class CliCommand: "crawl-pack", "research-pack", "entities-pack", - "brand-pack", - "styleguide-pack", - "product-pack", "extract-schema", - "image-pack", - "screenshot-pack", "search-pack", } ) @@ -107,20 +109,38 @@ class CliCommand: "async_build_standards_pack", "async_build_transcript_pack", "async_build_wiki_pack", + "async_run_workflow", + "ArtifactManifest", + "ChangeEvent", + "EvidenceSpan", + "IntelligenceBundle", + "SourceAuthority", + "WorkflowRequest", + "WorkflowResult", + "bundled_schema_path", + "write_contract_schemas", + "create_workflow_request", + "run_workflow", "Fetcher", "fetch_blocking", "fetch_one", "refresh_pack", "audit_pack", "build_dataset_pack", + "build_brand_pack", "build_feed_pack", "build_openapi_pack", "build_package_pack", "build_paper_pack", + "build_policy_pack", + "build_product_pack", "build_repo_pack", "build_standards_pack", + "build_styleguide_pack", "build_transcript_pack", "build_wiki_pack", + "build_image_pack", + "capture_screenshot_pack", "parse_documents", "parse_one_document", "DocumentParseError", @@ -129,6 +149,8 @@ class CliCommand: "score_pack_sources", "diff_packs", "build_citation_map", + "build_intelligence_bundle", + "build_company_brain_bundle", "extract_pack_entities", "search_pack", "build_research_brief", @@ -206,26 +228,27 @@ class CliCommand: "async_build_standards_pack", "async_build_transcript_pack", "async_build_wiki_pack", + "build_brand_pack", "build_dataset_pack", "build_feed_pack", "build_openapi_pack", "build_package_pack", "build_paper_pack", + "build_policy_pack", + "build_product_pack", "build_repo_pack", "build_standards_pack", + "build_styleguide_pack", "build_transcript_pack", "build_wiki_pack", + "build_image_pack", + "capture_screenshot_pack", ) PRUNED_SDK_EXPORTS = frozenset( { "answer_pack", - "build_brand_pack", - "build_styleguide_pack", - "build_product_pack", "extract_schema", - "build_image_pack", - "capture_screenshot_pack", "build_search_pack", "extract_pack", "map_sources", @@ -245,12 +268,7 @@ class CliCommand: PRUNED_CONTEXT_PACK_EXPORTS = frozenset( { - "build_brand_pack", - "build_styleguide_pack", - "build_product_pack", "extract_schema", - "build_image_pack", - "capture_screenshot_pack", "build_search_pack", } ) @@ -264,6 +282,14 @@ class CliCommand: "list_indexed", "grep_docs", "read_doc", + "workflow_run", + "brand_pack", + "product_pack", + "styleguide_pack", + "image_pack", + "screenshot_pack", + "policy_pack", + "intelligence_bundle", "pack_score", "pack_diff", "refresh_pack", @@ -297,12 +323,7 @@ class CliCommand: "crawl_pack", "research_pack", "entities_pack", - "brand_pack", - "styleguide_pack", - "product_pack", "extract_schema", - "image_pack", - "screenshot_pack", "search_pack", "answer_pack", } @@ -323,9 +344,11 @@ class CliCommand: "search", "brief", "prepare", + "intelligence-bundle", + "company-brain", } ) -PRUNED_PACK_SUBCOMMANDS = frozenset({"company-brain"}) +PRUNED_PACK_SUBCOMMANDS: frozenset[str] = frozenset() PRUNED_PACKAGE_EXTRAS = frozenset({"playwright", "parallel", "observability"}) PRUNED_BUILTIN_SOURCE_ALIASES = frozenset({"parallel"}) diff --git a/src/docpull/workflows.py b/src/docpull/workflows.py new file mode 100644 index 0000000..b4554b2 --- /dev/null +++ b/src/docpull/workflows.py @@ -0,0 +1,262 @@ +"""Common execution protocol for evidence-backed knowledge-pack workflows.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Callable +from contextvars import ContextVar +from pathlib import Path +from typing import Any, Protocol + +from .contracts import WorkflowRequest, WorkflowResult, build_workflow_request +from .policy import PolicyConfig + + +class WorkflowExecutionError(RuntimeError): + """Raised when a generic workflow request cannot be executed.""" + + +_CURRENT_WORKFLOW_REQUEST: ContextVar[WorkflowRequest | None] = ContextVar( + "docpull_workflow_request", + default=None, +) + + +class PackWorkflow(Protocol): + """Transport-neutral protocol implemented by every registered pack lane.""" + + name: str + + def execute(self, request: WorkflowRequest) -> dict[str, Any]: ... + + +class FunctionPackWorkflow: + """Small adapter that gives existing SDK builders the common protocol.""" + + def __init__(self, name: str, executor: Callable[[WorkflowRequest], dict[str, Any]]) -> None: + self.name = name + self._executor = executor + + def execute(self, request: WorkflowRequest) -> dict[str, Any]: + return self._executor(request) + + +def create_workflow_request( + workflow: str, + value: str, + *, + output_dir: Path, + options: dict[str, Any] | None = None, + policy: PolicyConfig | None = None, +) -> WorkflowRequest: + effective_policy = policy or PolicyConfig() + merged_options = dict(options or {}) + if policy is not None: + merged_options["policy"] = policy.model_dump(mode="json") + return build_workflow_request( + workflow=_normalize_workflow(workflow), + input_payload={"value": value}, + output_dir=output_dir, + options=merged_options, + source_policy=effective_policy.to_source_policy_payload(source=workflow), + budget={"maximum_paid_cost_usd": effective_policy.budget.maximum_paid_cost_usd}, + browser_enabled=_normalize_workflow(workflow) == "screenshot-pack" + or bool(merged_options.get("render")), + paid_routes_enabled=False, + ) + + +def run_workflow(request: WorkflowRequest | dict[str, Any]) -> dict[str, Any]: + """Execute one request and return the canonical ``workflow.result.v1`` payload.""" + + parsed = request if isinstance(request, WorkflowRequest) else WorkflowRequest.model_validate(request) + workflow_name = _normalize_workflow(parsed.workflow) + workflow = WORKFLOW_REGISTRY.get(workflow_name) + if workflow is None: + available = ", ".join(sorted(WORKFLOW_REGISTRY)) + raise WorkflowExecutionError(f"Unsupported workflow {parsed.workflow!r}. Available: {available}") + token = _CURRENT_WORKFLOW_REQUEST.set(parsed) + try: + workflow.execute(parsed) + finally: + _CURRENT_WORKFLOW_REQUEST.reset(token) + output_dir = _request_output_dir(parsed) + result_path = output_dir / "workflow.result.json" + if not result_path.exists(): + raise WorkflowExecutionError( + f"Workflow {workflow_name} did not emit the required workflow.result.json contract" + ) + payload = json.loads(result_path.read_text(encoding="utf-8")) + return WorkflowResult.model_validate(payload).model_dump(mode="json", exclude_none=True) + + +async def async_run_workflow(request: WorkflowRequest | dict[str, Any]) -> dict[str, Any]: + return await asyncio.to_thread(run_workflow, request) + + +def current_workflow_request() -> WorkflowRequest | None: + """Return the active request while a registered builder is materializing artifacts.""" + + return _CURRENT_WORKFLOW_REQUEST.get() + + +def _run_brand(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.brand import build_brand_pack + + options = request.options + return build_brand_pack( + _request_value(request), + email=_optional_str(options, "email"), + name=_optional_str(options, "name"), + ticker=_optional_str(options, "ticker"), + output_dir=_request_output_dir(request), + policy=_request_policy(request), + allow_free_email=bool(options.get("allow_free_email", False)), + download_assets=bool(options.get("download_assets", True)), + max_pages=_positive_int(options, "max_pages", 6), + ) + + +def _run_product(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.product import build_product_pack + + options = request.options + return build_product_pack( + _request_value(request), + mode=str(options.get("mode") or "page"), + output_dir=_request_output_dir(request), + policy=_request_policy(request), + max_pages=_positive_int(options, "max_pages", 8), + ) + + +def _run_styleguide(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.styleguide import build_styleguide_pack + + options = request.options + return build_styleguide_pack( + _request_value(request), + output_dir=_request_output_dir(request), + policy=_request_policy(request), + render=bool(options.get("render", False)), + max_stylesheets=_positive_int(options, "max_stylesheets", 12), + ) + + +def _run_visual(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.visuals import build_image_pack + + options = request.options + return build_image_pack( + _request_value(request), + output_dir=_request_output_dir(request), + policy=_request_policy(request), + download_assets=bool(options.get("download_assets", True)), + max_assets=_positive_int(options, "max_assets", 40), + ) + + +def _run_screenshot(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.visuals import capture_screenshot_pack + + options = request.options + return capture_screenshot_pack( + _request_value(request), + output_dir=_request_output_dir(request), + policy=_request_policy(request), + viewport=str(options.get("viewport") or "1280x720"), + full_page=bool(options.get("full_page", False)), + wait_for=str(options.get("wait_for") or "load"), + agent_browser_binary=_optional_str(options, "agent_browser_binary"), + ) + + +def _run_policy(request: WorkflowRequest) -> dict[str, Any]: + from .context_packs.policy_pack import build_policy_pack + + options = request.options + baseline = _optional_str(options, "baseline_pack") + return build_policy_pack( + _request_value(request), + output_dir=_request_output_dir(request), + policy=_request_policy(request), + max_pages=_positive_int(options, "max_pages", 16), + baseline_pack=Path(baseline) if baseline else None, + ) + + +WORKFLOW_REGISTRY: dict[str, PackWorkflow] = { + "brand-pack": FunctionPackWorkflow("brand-pack", _run_brand), + "product-pack": FunctionPackWorkflow("product-pack", _run_product), + "styleguide-pack": FunctionPackWorkflow("styleguide-pack", _run_styleguide), + "visual-pack": FunctionPackWorkflow("visual-pack", _run_visual), + "image-pack": FunctionPackWorkflow("image-pack", _run_visual), + "screenshot-pack": FunctionPackWorkflow("screenshot-pack", _run_screenshot), + "policy-pack": FunctionPackWorkflow("policy-pack", _run_policy), +} + + +def _normalize_workflow(value: str) -> str: + normalized = value.strip().lower().replace("_", "-") + aliases = { + "brand": "brand-pack", + "product": "product-pack", + "styleguide": "styleguide-pack", + "visual": "visual-pack", + "image": "image-pack", + "screenshot": "screenshot-pack", + "policy": "policy-pack", + } + return aliases.get(normalized, normalized) + + +def _request_value(request: WorkflowRequest) -> str: + for key in ("value", "url", "domain_or_url", "url_or_domain", "url_or_pack"): + value = request.input.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + raise WorkflowExecutionError("WorkflowRequest.input must include a non-empty value or URL") + + +def _request_output_dir(request: WorkflowRequest) -> Path: + value = request.output.get("directory") + if not isinstance(value, str) or not value.strip(): + raise WorkflowExecutionError("WorkflowRequest.output.directory must be a non-empty path") + return Path(value).expanduser().resolve() + + +def _request_policy(request: WorkflowRequest) -> PolicyConfig | None: + raw = request.options.get("policy") + if raw is None: + return None + if not isinstance(raw, dict): + raise WorkflowExecutionError("WorkflowRequest.options.policy must be an object") + return PolicyConfig.model_validate(raw) + + +def _positive_int(options: dict[str, Any], key: str, default: int) -> int: + value = options.get(key, default) + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise WorkflowExecutionError(f"WorkflowRequest.options.{key} must be a positive integer") + return value + + +def _optional_str(options: dict[str, Any], key: str) -> str | None: + value = options.get(key) + if value is None: + return None + if not isinstance(value, str): + raise WorkflowExecutionError(f"WorkflowRequest.options.{key} must be a string") + return value + + +__all__ = [ + "PackWorkflow", + "WORKFLOW_REGISTRY", + "WorkflowExecutionError", + "async_run_workflow", + "current_workflow_request", + "create_workflow_request", + "run_workflow", +] diff --git a/tests/fixtures/evidence_failure_modes.json b/tests/fixtures/evidence_failure_modes.json new file mode 100644 index 0000000..58983cb --- /dev/null +++ b/tests/fixtures/evidence_failure_modes.json @@ -0,0 +1,17 @@ +{ + "pricing": { + "url": "https://acme.test/pricing", + "html": "

We raised $2 million in 2024.

\"Pricing

Pro

$29 per user per month

14-day free trial

Advanced API included; SSO only on Enterprise

", + "markdown": "We raised $2 million in 2024.\n\n## Pro pricing plan\n$29 per user per month — 14-day free trial — Advanced API included; SSO only on Enterprise" + }, + "branding": { + "url": "https://acme.test/brand", + "html": "

Acme

Unofficial profile", + "markdown": "# Acme\nUnofficial profiles and unsafe embedded assets must not become authoritative brand claims." + }, + "legal": { + "url": "https://acme.test/legal/privacy", + "html": "

Privacy Policy

Last updated: July 1, 2026

Data

We collect account data.

Data

We retain billing records.

", + "markdown": "# Privacy Policy\n\nLast updated: July 1, 2026\n\n## Data\n\nWe collect account data.\n\n## Data\n\nWe retain billing records." + } +} diff --git a/tests/test_context_packs.py b/tests/test_context_packs.py index d71b61f..98f129a 100644 --- a/tests/test_context_packs.py +++ b/tests/test_context_packs.py @@ -476,8 +476,11 @@ async def test_mcp_dispatch_brand_pack( }, ) - assert result.is_error is True - assert result.text == "Unknown tool: brand_pack" + assert result.is_error is False + assert result.data is not None + assert result.data["contract_version"] == "workflow.result.v1" + assert result.data["workflow"] == "brand-pack" + assert result.data["status"] == "completed" def _html_for_url(url: str) -> str: diff --git a/tests/test_workflow_contracts.py b/tests/test_workflow_contracts.py new file mode 100644 index 0000000..8f7324f --- /dev/null +++ b/tests/test_workflow_contracts.py @@ -0,0 +1,249 @@ +"""Compatibility and contract tests for evidence acquisition workflows.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest +from jsonschema import Draft202012Validator + +from docpull.change_events import build_change_events +from docpull.context_packs.policy_pack import build_policy_pack +from docpull.contracts import ( + CONTRACT_MODELS, + ArtifactManifest, + ChangeEvent, + IntelligenceBundle, + WorkflowRequest, + WorkflowResult, + bundled_schema_path, +) +from docpull.models.document import DocumentRecord +from docpull.output_contract import validate_pack_contract +from docpull.pack_reader import load_pack +from docpull.pack_tools import build_intelligence_bundle, prepare_pack +from docpull.project import add_source, init_project, load_project_config +from docpull.workflows import create_workflow_request, run_workflow +from tests.pack_fixtures import write_context_pack + +FIXTURES = Path(__file__).with_name("fixtures") / "evidence_failure_modes.json" + + +class WorkflowFetcher: + pages: dict[str, dict[str, str]] = {} + + def __init__(self, _config: object) -> None: + pass + + async def __aenter__(self) -> WorkflowFetcher: + return self + + async def __aexit__(self, *_exc: object) -> None: + return None + + async def fetch_one(self, url: str, *, save: bool) -> SimpleNamespace: + assert save is False + page = self.pages[url] + return SimpleNamespace( + error=None, + should_skip=False, + skip_reason=None, + html=page["html"].encode(), + markdown=page["markdown"], + title=page.get("title", "Acme"), + metadata={}, + extraction_info={}, + source_type="fixture", + ) + + +@pytest.fixture +def failure_modes() -> dict[str, dict[str, str]]: + return json.loads(FIXTURES.read_text(encoding="utf-8")) + + +def test_product_workflow_emits_generic_contracts_and_eval_grade( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_modes: dict[str, dict[str, str]], +) -> None: + pricing = failure_modes["pricing"] + WorkflowFetcher.pages = {pricing["url"]: pricing} + monkeypatch.setattr("docpull.context_packs.common.Fetcher", WorkflowFetcher) + output = tmp_path / "product" + + request = create_workflow_request( + "product-pack", + pricing["url"], + output_dir=output, + options={"mode": "page", "max_pages": 1}, + ) + result_payload = run_workflow(request) + legacy = json.loads((output / "products.result.json").read_text(encoding="utf-8")) + manifest_payload = json.loads((output / "artifact.manifest.json").read_text(encoding="utf-8")) + + result = WorkflowResult.model_validate(result_payload) + manifest = ArtifactManifest.model_validate(manifest_payload) + stored_request = WorkflowRequest.model_validate_json( + (output / "workflow.request.json").read_text(encoding="utf-8") + ) + assert result.request_id == request.request_id == stored_request.request_id + assert result.progress_events[0].phase == "run" + assert result.budget_usage.estimated_usd == 0 + assert result.replay_configuration.local_first is True + assert manifest.aggregate_sha256 == result.hashes["pack"].digest + assert legacy["pricing_matrix"][0]["currency"] == "USD" + assert legacy["pricing_matrix"][0]["billing_interval"] == {"unit": "month", "count": 1} + assert legacy["pricing_matrix"][0]["trial"]["duration_days"] == 14 + assert legacy["pricing_matrix"][0]["price_source"]["medium"] == "page_text" + assert all(row["price"] != 2_000_000 for row in legacy["pricing_matrix"]) + span = legacy["pricing_matrix"][0]["evidence"]["evidence_span"] + assert pricing["markdown"][span["char_start"] : span["char_end"]] == span["exact_text"] + + assert validate_pack_contract(output, level="raw")["status"] == "pass" + prepare_pack(output, graph=False, eval_grade=True, markdown=False) + assert validate_pack_contract(output, level="agent")["status"] == "pass" + assert validate_pack_contract(output, level="eval")["status"] == "pass" + + +def test_policy_pack_discovers_types_dates_stable_clauses_and_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_modes: dict[str, dict[str, str]], +) -> None: + legal = failure_modes["legal"] + home_url = "https://acme.test/" + home = { + "title": "Acme", + "html": 'Privacy', + "markdown": "[Privacy](/legal/privacy)", + } + WorkflowFetcher.pages = {home_url: home, legal["url"]: legal} + monkeypatch.setattr("docpull.context_packs.common.Fetcher", WorkflowFetcher) + baseline = tmp_path / "baseline" + first = build_policy_pack("acme.test", output_dir=baseline) + + assert first["policies"][0]["document_type"] == "privacy" + assert first["policies"][0]["effective_date"] == "July 1, 2026" + assert len(first["clauses"]) == 3 + assert first["clauses"][1]["clause_id"] != first["clauses"][2]["clause_id"] + assert first["clauses"][1]["evidence"]["evidence_span"]["char_start"] >= 0 + + revised = dict(legal) + revised["markdown"] = legal["markdown"].replace( + "We retain billing records.", "We retain billing records for seven years." + ) + WorkflowFetcher.pages = {home_url: home, legal["url"]: revised} + current = build_policy_pack( + "acme.test", + output_dir=tmp_path / "current", + baseline_pack=baseline, + ) + assert current["summary"]["change_candidate_count"] == 1 + assert current["change_candidates"][0]["classification"] == "policy" + assert current["change_candidates"][0]["status"] == "candidate" + + +def test_change_events_are_idempotent_and_separate_change_layers() -> None: + old = DocumentRecord.from_page( + url="https://acme.test/pricing", + title="Pricing", + content="# Pricing\nPro is $20 per month.", + ).model_dump(mode="json", exclude_none=True) + new = DocumentRecord.from_page( + url="https://acme.test/pricing", + title="Plans and pricing", + content="# Plans\nPro is $29 per month with a 14-day trial.", + ).model_dump(mode="json", exclude_none=True) + + first = build_change_events({old["url"]: [old]}, {new["url"]: [new]}, workflow="product-pack") + second = build_change_events({old["url"]: [old]}, {new["url"]: [new]}, workflow="product-pack") + + assert first == second + event = ChangeEvent.model_validate(first[0]) + assert event.old_document_id == old["document_id"] + assert event.new_document_id == new["document_id"] + assert event.structural_changes + assert event.textual_changes + assert event.semantic_candidates + assert "pricing" in event.classifications + assert event.replay_configuration.scheduler is None + + +def test_intelligence_bundle_is_deterministic_and_keeps_company_brain_alias(tmp_path: Path) -> None: + pack = tmp_path / "pack" + content = "Acme Pro pricing is $29 per month and includes cited product evidence." + record = DocumentRecord.from_page( + url="https://acme.test/pricing", + title="Acme pricing", + content=content, + source_citation_id="S1", + record_citation_id="S1.1", + ).model_dump(mode="json", exclude_none=True) + write_context_pack(pack, records=[record], include_domains=["acme.test"]) + + first = build_intelligence_bundle( + pack, + objective="Track Acme pricing", + market="Developer tools", + search_queries=["pricing"], + ) + first_bytes = (pack / "intelligence.bundle.v1.json").read_bytes() + second = build_intelligence_bundle( + pack, + objective="Track Acme pricing", + market="Developer tools", + search_queries=["pricing"], + ) + second_bytes = (pack / "intelligence.bundle.v1.json").read_bytes() + + parsed = IntelligenceBundle.model_validate(second) + assert first["bundle_hash"] == second["bundle_hash"] + assert first_bytes == second_bytes + assert parsed.observations + assert parsed.observations[0].status == "observation" + assert parsed.observations[0].evidence[0].document_version == record["content_hash"] + assert (pack / "company_brain.bundle.json").read_bytes() == second_bytes + + +def test_all_schemas_are_draft_2020_12_and_accept_model_examples(tmp_path: Path) -> None: + for filename, model in CONTRACT_MODELS.items(): + schema = json.loads(bundled_schema_path(filename).read_text(encoding="utf-8")) + Draft202012Validator.check_schema(schema) + assert schema["$id"].endswith(filename) + assert model.model_json_schema(mode="serialization")["title"] == schema["title"] + + exported = tmp_path / "schemas" + from docpull.contracts import write_contract_schemas + + assert len(write_contract_schemas(exported)) == len(CONTRACT_MODELS) + + +def test_declarative_project_accepts_knowledge_workflow_source_types(tmp_path: Path) -> None: + init_project(name="tracker", root=tmp_path) + for index, source_type in enumerate(("brand", "product", "styleguide", "visual", "policy")): + add_source( + f"https://{source_type}-{index}.example.test", + name=source_type, + source_type=source_type, + root=tmp_path, + ) + config = load_project_config(tmp_path) + assert [source.type for source in config.sources] == [ + "brand", + "product", + "styleguide", + "visual", + "policy", + ] + + +def test_pre_v6_pack_contract_remains_readable(tmp_path: Path) -> None: + pack = tmp_path / "legacy" + records = write_context_pack(pack) + loaded = load_pack(pack) + assert loaded.documents[0].document_id == records[0]["document_id"] + assert hashlib.sha256(loaded.documents[0].content.encode()).hexdigest()