From cdf121573db3ac6b6e7bf87c523e3559571100be Mon Sep 17 00:00:00 2001 From: foxsplendid <62349701+foxsplendid@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:50:21 +0800 Subject: [PATCH 1/2] Advance the local research workflow --- .github/workflows/ci.yml | 196 +- .gitignore | 1 + CHANGELOG.md | 28 + README.md | 127 +- README.zh.md | 113 +- docs/architecture-and-acceptance.zh-CN.md | 443 +++++ docs/case-study.zh-CN.md | 15 +- pyproject.toml | 2 +- src/scriptorium/__init__.py | 2 +- src/scriptorium/assets/compatibility.toml | 4 +- src/scriptorium/assets/lineage-graph.v1.json | 42 + src/scriptorium/assets/parsed-paper.v1.json | 43 + src/scriptorium/assets/reading-note.v1.json | 30 + src/scriptorium/assets/review.v1.json | 31 + .../skills/scriptorium-research/SKILL.md | 10 +- src/scriptorium/cli.py | 312 +++- src/scriptorium/demo.py | 121 +- src/scriptorium/doctor.py | 79 +- src/scriptorium/migration.py | 1595 +++++++++++++++++ src/scriptorium/resume.py | 462 +++++ tests/_e2e_lectern_worker.py | 245 +++ tests/e2e_install_lifecycle.py | 722 ++++++++ tests/e2e_pull.py | 198 +- tests/e2e_slides.py | 479 +++++ tests/test_cli_migration.py | 370 ++++ tests/test_demo.py | 29 +- tests/test_doctor.py | 51 + tests/test_e2e_slides.py | 86 + tests/test_install_lifecycle.py | 120 ++ tests/test_migration.py | 940 ++++++++++ tests/test_pull.py | 30 +- tests/test_resume.py | 277 +++ uv.lock | 2 +- 33 files changed, 7117 insertions(+), 88 deletions(-) create mode 100644 docs/architecture-and-acceptance.zh-CN.md create mode 100644 src/scriptorium/assets/lineage-graph.v1.json create mode 100644 src/scriptorium/assets/parsed-paper.v1.json create mode 100644 src/scriptorium/assets/reading-note.v1.json create mode 100644 src/scriptorium/assets/review.v1.json create mode 100644 src/scriptorium/migration.py create mode 100644 src/scriptorium/resume.py create mode 100644 tests/_e2e_lectern_worker.py create mode 100644 tests/e2e_install_lifecycle.py create mode 100644 tests/e2e_slides.py create mode 100644 tests/test_cli_migration.py create mode 100644 tests/test_e2e_slides.py create mode 100644 tests/test_install_lifecycle.py create mode 100644 tests/test_migration.py create mode 100644 tests/test_resume.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f129dc..b93da0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,8 @@ jobs: - run: scriptorium --version - run: scriptorium init --help - run: scriptorium inventory --help + - run: scriptorium migrate --help + - run: scriptorium resume --help - run: scriptorium status --help - run: scriptorium doctor --help - run: scriptorium host install --help @@ -39,10 +41,15 @@ jobs: with: path: scriptorium persist-credentials: false + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: v0.1.0 + path: scriptorium-v0.1.0 + persist-credentials: false - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: scriptorium-suite/scriptorium-spec - ref: 66bfefea93758231491fadc4694708d1b8107ea0 + ref: 0439e7abe4dd307c8b97afa34f9fc3b45605d3b4 path: scriptorium-spec persist-credentials: false - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -54,12 +61,48 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: foxsplendid/Provenance - ref: 910efbc47e604c51314866174581cfdd7eac47b6 + ref: 6380bb0a8bfe8129a9edb2f903856a33b59dbcbb path: Provenance persist-credentials: false + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: foxsplendid/Academic-Slides-Agent + ref: 6371f1e6eac41f505aed39e58dafb689317cab65 + path: Academic-Slides-Agent + persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" + - name: Verify clean Windows install lifecycle + shell: pwsh + run: | + python .\scriptorium\tests\e2e_install_lifecycle.py ` + --scriptorium-root .\scriptorium ` + --spec-root .\scriptorium-spec ` + --steward-root .\steward ` + --provenance-root .\Provenance ` + --previous-scriptorium-root .\scriptorium-v0.1.0 ` + --require-windows ` + --report .\install-lifecycle-report.json + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + $report = Get-Content -LiteralPath .\install-lifecycle-report.json -Raw | ConvertFrom-Json + if ( + $report.status -ne "passed" -or + $report.lifecycle.clean_install -ne "passed" -or + $report.lifecycle.uninstall -ne "passed" -or + $report.lifecycle.reinstall -ne "passed" -or + $report.lifecycle.doctor_and_demo -ne "passed" -or + $report.lifecycle.version_transition.status -ne "passed" -or + $report.lifecycle.version_transition.previous_version -ne "0.1.0" + ) { + throw "Clean Windows install lifecycle did not pass" + } + - name: Install uv for Lectern + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.16" + python-version: "3.12.3" - name: Install source checkouts shell: pwsh run: | @@ -67,6 +110,8 @@ jobs: .\.demo-venv\Scripts\python.exe -m pip install --no-deps .\scriptorium .\.demo-venv\Scripts\python.exe -m pip install --no-deps .\steward .\.demo-venv\Scripts\python.exe -m pip install --no-deps .\Provenance + - name: Install Lectern workspace without provider extras + run: uv sync --project .\Academic-Slides-Agent --all-packages --locked --no-dev - name: Smoke installed public pull entries shell: pwsh run: | @@ -78,6 +123,13 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } & .\.demo-venv\Scripts\prov-sync-unresolved.exe --help | Out-Null if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\.demo-venv\Scripts\prov-ingest-research.exe --help | Out-Null + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $contextVersion = & .\.demo-venv\Scripts\prov-context.exe --version + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if (($contextVersion -join "`n").Trim() -ne '0.18.0') { + throw "Installed Provenance context runtime version mismatch" + } $raw = & .\.demo-venv\Scripts\prov-sync-pull.exe --capabilities --json $code = $LASTEXITCODE @@ -98,8 +150,12 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } & .\.demo-venv\Scripts\scriptorium.exe inventory --help | Out-Null if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\.demo-venv\Scripts\scriptorium.exe migrate --help | Out-Null + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } & .\.demo-venv\Scripts\scriptorium.exe status --help | Out-Null if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & .\.demo-venv\Scripts\scriptorium.exe resume --help | Out-Null + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $errorRaw = & .\.demo-venv\Scripts\scriptorium.exe pull --json 2>$null $errorCode = $LASTEXITCODE @@ -163,6 +219,132 @@ jobs: if ([IO.File]::ReadAllText((Join-Path $source 'private-note.md')) -ne 'PRIVATE_RESEARCH_SENTINEL') { throw "Inventory modified a source file" } + - name: Smoke installed synthetic migration lifecycle + shell: pwsh + run: | + $root = Join-Path $env:RUNNER_TEMP 'migration smoke' + $source = Join-Path $root 'selected source' + $workspace = Join-Path $root 'research workspace' + $env:LOCALAPPDATA = Join-Path $root 'canonical local state' + New-Item -ItemType Directory -Path $source, $workspace | Out-Null + + [IO.File]::WriteAllText( + (Join-Path $source 'synthetic-note.md'), + '# SYNTHETIC_MIGRATION_SENTINEL' + ) + [IO.File]::WriteAllText( + (Join-Path $source 'synthetic-paper.pdf'), + '%PDF-1.4 SYNTHETIC' + ) + + $planRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate plan ` + --source $source ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration plan failed" } + $planSerialized = $planRaw -join "`n" + $plan = $planSerialized | ConvertFrom-Json + if ( + $plan.operation -ne 'plan' -or + $plan.status -ne 'planned' -or + $plan.summary.files -ne 2 + ) { + throw "Installed migration plan report changed" + } + if (Test-Path (Join-Path $workspace 'Sources')) { + throw "Migration plan wrote into the workspace" + } + if (Test-Path $env:LOCALAPPDATA) { + throw "Migration plan persisted private state" + } + + $applyRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate apply ` + --source $source ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration apply failed" } + $applySerialized = $applyRaw -join "`n" + $apply = $applySerialized | ConvertFrom-Json + if ( + $apply.operation -ne 'apply' -or + $apply.status -ne 'applied' -or + $apply.summary.changed -ne 2 + ) { + throw "Installed migration apply report changed" + } + + $verifyRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate verify ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration verify failed" } + $verify = ($verifyRaw -join "`n") | ConvertFrom-Json + if ($verify.operation -ne 'verify' -or $verify.status -ne 'applied') { + throw "Installed migration verify report changed" + } + + $repeatRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate apply ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration reapply failed" } + $repeat = ($repeatRaw -join "`n") | ConvertFrom-Json + if ($repeat.status -ne 'unchanged') { + throw "Installed migration reapply was not idempotent" + } + + $rollbackRaw = & .\.demo-venv\Scripts\scriptorium.exe migrate rollback ` + --workspace $workspace ` + --batch-id synthetic-ci ` + --json + if ($LASTEXITCODE -ne 0) { throw "Installed migration rollback failed" } + $rollbackSerialized = $rollbackRaw -join "`n" + $rollback = $rollbackSerialized | ConvertFrom-Json + if ( + $rollback.operation -ne 'rollback' -or + $rollback.status -ne 'rolled-back' -or + $rollback.summary.changed -ne 2 + ) { + throw "Installed migration rollback report changed" + } + + $allReports = @( + $planSerialized, + $applySerialized, + ($verifyRaw -join "`n"), + ($repeatRaw -join "`n"), + $rollbackSerialized + ) -join "`n" + foreach ($privateValue in @( + $root, + $source, + $workspace, + 'synthetic-note.md', + 'synthetic-paper.pdf', + 'SYNTHETIC_MIGRATION_SENTINEL' + )) { + if ($allReports.Contains($privateValue)) { + throw "Migration report exposed a private source value" + } + } + if ( + [IO.File]::ReadAllText((Join-Path $source 'synthetic-note.md')) -ne + '# SYNTHETIC_MIGRATION_SENTINEL' + ) { + throw "Migration modified its source" + } + $remaining = @( + Get-ChildItem ` + (Join-Path $workspace 'Sources\Imported\synthetic-ci') ` + -File ` + -Recurse ` + -ErrorAction SilentlyContinue + ) + if ($remaining.Count -ne 0) { + throw "Migration rollback left owned files" + } - name: Run isolated public pull E2E shell: pwsh run: | @@ -170,6 +352,15 @@ jobs: .\scriptorium\tests\e2e_pull.py ` --provenance-root .\Provenance if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Run offline Steward to Lectern golden path + shell: pwsh + run: | + & .\.demo-venv\Scripts\python.exe ` + .\scriptorium\tests\e2e_slides.py ` + --steward-root .\steward ` + --spec-root .\scriptorium-spec ` + --lectern-root .\Academic-Slides-Agent + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Run credential-free golden path shell: pwsh run: | @@ -194,4 +385,5 @@ jobs: path: | demo 输出 doctor-report.json + install-lifecycle-report.json retention-days: 14 diff --git a/.gitignore b/.gitignore index 143e54a..8d8f117 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ htmlcov/ build/ dist/ scriptorium-demo/ +.tmp-*/ # Local configuration and credentials .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 7206b87..35658ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## 0.2.0 (candidate) — 2026-07-22 + +- Add `scriptorium resume`, backed by Provenance's bounded, read-only Context + Capsule. It restores approved project state, explicitly labels auto-applied + low-risk progress, and keeps literature artifacts reference-only. +- Extend the synthetic demo with all four literature-reading artifacts, atomic + idempotent ingestion, reference hints, and privacy assertions. +- Exercise two consecutive synthetic Agent sessions so the second session proves + that it can resume the first session's reviewed project state. +- Require compatible `prov-context` and `prov-ingest-research` commands in Public + Alpha diagnostics, including an actual runtime-version probe. +- Move the candidate compatibility baseline to `scriptorium-spec` 2.3.0 for the + `experiment-run/1.0` and `claim-evidence/1.0` contracts. Runtime registration, + persistence, query, human review, and claim linkage remain V0.3 work. +- Add the safety-reviewed `scriptorium migrate` CLI as a V0.3 candidate for + explicit Markdown/PDF copies: write-free plan, create-if-absent apply, + canonical private state, cross-process verify/reapply recovery, and rollback. + Reports and errors remain aggregate-only and path-free; this does not mark V0.3 + complete. +- Add an offline, synthetic Steward-to-Lectern acceptance path that validates a + two-paper handoff through Lectern's production graph, stops for outline approval + with no persistent pre-approval PPTX, compiles an editable deck, and + scans transferable artifacts for paths, email addresses, and credential shapes. +- Add a temporary clean-environment lifecycle gate that builds local wheels with + package indexes disabled, verifies install/uninstall/reinstall and the synthetic + demo, and exercises the public v0.1.0-to-current version transition. Live Agent, + PowerPoint, external-user, and fresh remote-CI acceptance remain explicit gates. + ## 0.1.0 — 2026-07-20 - Add `scriptorium inventory`, a deterministic, zero-write preview for explicitly diff --git a/README.md b/README.md index e47a4e2..1978c0c 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,26 @@ # Scriptorium -> **Public Alpha v0.1.0:** the umbrella repository ships safe initialization for +> **Public Alpha v0.2.0 candidate:** the umbrella repository ships safe initialization for > a real project, one synthetic vertical slice through `scriptorium demo`, the > read-only `scriptorium doctor`, a content-free `scriptorium status` control-plane > summary, a zero-write `scriptorium inventory` preview for explicitly selected > local sources, > and explicit project-scoped Codex and Claude Code skill installers. It also ships -> the accepted on-demand `scriptorium pull` entry through Provenance's -> machine-readable public command. GitHub-hosted clean Windows CI and an isolated -> Windows source-install acceptance run pass; live Agent-host parity and Lectern -> remain outside the credential-free golden path. +> the accepted on-demand `scriptorium pull` entry and bounded, read-only +> `scriptorium resume` Context Capsule through Provenance's machine-readable +> public commands. A safety-reviewed `scriptorium migrate` CLI is included as a +> V0.3 candidate, not as a completed V0.3 release. The previously published baseline +> has GitHub-hosted Windows evidence; this worktree also has local synthetic migration, +> install/uninstall/reinstall, v0.1.0-to-current version-transition, and +> Steward-to-Lectern checks, but still needs fresh remote CI. +> Live Agent-host parity and real-provider slide generation remain manual Alpha gates. Scriptorium is a local-first, agent-native research workflow suite. This repository is its thin control plane: it coordinates independently useful components through public commands and versioned files without importing their internals or owning their research data. -[中文说明](README.zh.md) · [中文产品案例](docs/case-study.zh-CN.md) · [Showcase evidence](docs/showcase/README.zh-CN.md) · [Contract source of truth](https://github.com/scriptorium-suite/scriptorium-spec) · [Design inspirations](ACKNOWLEDGEMENTS.md) +[中文说明](README.zh.md) · [架构、使用与分层验收](docs/architecture-and-acceptance.zh-CN.md) · [中文产品案例](docs/case-study.zh-CN.md) · [Showcase evidence](docs/showcase/README.zh-CN.md) · [Contract source of truth](https://github.com/scriptorium-suite/scriptorium-spec) · [Design inspirations](ACKNOWLEDGEMENTS.md) ![Scriptorium Public Alpha synthetic golden-path evidence](docs/showcase/demo-poster.svg) @@ -47,9 +51,11 @@ AI4Science literature workflow through the real public interfaces: 1. validate a synthetic `library-kb/1.1` with `scriptorium-spec`; 2. call Steward to scope two papers and assemble a review from a recorded agent draft; 3. call Provenance to ingest the library and Markdown project; -4. build and query the local search index; -5. verify portfolio, project context, and literature search through Provenance MCP; -6. write the human-readable artifacts and `demo-report.json`. +4. validate and atomically ingest four synthetic literature-reading artifacts; +5. build and query the local search index; +6. verify portfolio, bounded Context Capsule, reference-only literature hints, + and literature search through Provenance MCP; +7. write the human-readable artifacts and `demo-report.json`. The run needs no API key, Zotero, Obsidian, browser extension, or agent login. Once the source checkouts are installed, this demo path is designed to operate @@ -77,6 +83,14 @@ its fill in-session, then another pull applies the low-risk timeline and stages high-value claims in `Approvals.md`. A user tick is still required before a later pull commits those claims. +`scriptorium resume` is the bounded session-start view. It asks the compatible +`prov-context` runtime for one registered project and accepts only an allowlisted +capsule shape. Approved project fields are separated from auto-applied low-risk +progress; literature and reading artifacts are explicitly reference-only. Raw +conversations, drafts, rejected claims, component stderr, and local paths are not +forwarded. The V0.2 end-to-end fixture runs two consecutive synthetic sessions and +checks that the second session can recover the first session's reviewed state. + `scriptorium status` is the daily content-free control-plane summary. It first rebuilds the Public Alpha readiness result from `doctor`; only when that boundary is ready does it run a `pull` preview. The result contains allowlisted capability states, @@ -101,6 +115,31 @@ migration occurred. On Windows, selected objects are held through metadata-only bindings for the duration of the preview, so another process cannot rename, delete, or open them for data write until the command finishes. +`scriptorium migrate` is the V0.3 candidate copy boundary for explicitly selected +Markdown/PDF files. `plan` is write-free; first `apply` requires the selected +sources; later `apply`, `verify`, and `rollback` recover the batch from only its +workspace and batch identifier. The preview is advisory rather than a persisted +execution snapshot: first `apply` rescans and re-hashes the explicit sources. +Targets are created without overwrite through a +same-directory hard-link publication step. Apply uses a random, exclusively +created `.scriptorium-*.stage` file; only after one file descriptor has completed +copy, hash, flush, and identity checks is that stage sealed into the private +manifest and published as the target's same-file ownership anchor. A crash before +that seal may leave an unclaimed random stage. It is never adopted or deleted +automatically. + +Rollback records a random same-directory quarantine name before each target and +anchor transition, atomically moves without replacement, then re-verifies content +and stable file identity before deletion. A foreign replacement is restored to its +original path when possible, otherwise preserved in quarantine while rollback +fails closed. Automatic rollback requires Windows no-replace rename or Linux +`renameat2(RENAME_NOREPLACE)`; other platforms fail closed. Private +path manifests use the canonical per-user local state root outside the workspace; +sources may not overlap that private state root, and no arbitrary state-root flag +is exposed. Terminal +and JSON reports contain aggregate counts and states only. This command copies +files—it does not parse, index, approve, or ingest them into Provenance. + ## Source quickstart on Windows Prerequisites: Git and Python 3.11+. Clone the four repositories into one parent @@ -147,6 +186,53 @@ filenames, research text, hashes, sizes, or timestamps. An incomplete or unsafe fails closed with exit code `1`; malformed invocation or an internal boundary failure returns `2` without echoing the sensitive input. +### V0.3 candidate: copy selected Markdown/PDF files safely + +Use a stable batch identifier. Review the aggregate plan first, then repeat the +same explicit sources for the first apply: + +```powershell +$Legacy = 'D:\Research\Legacy Notes' +$MigrationWorkspace = 'D:\Research\Scriptorium Workspace' +$Batch = 'legacy-notes-001' + +.\.venv\Scripts\scriptorium.exe migrate plan ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +.\.venv\Scripts\scriptorium.exe migrate apply ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +# Recovery operations need no source path or saved plan. +.\.venv\Scripts\scriptorium.exe migrate verify ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate apply ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate rollback ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +``` + +The second `apply` is an idempotency/recovery check and should report +`unchanged`. The destination filesystem must support hard links; the command +fails closed instead of using an overwrite-capable fallback. Do not remove the +internal `.scriptorium-*.stage` ownership anchors or `.scriptorium-*.rollback` +quarantine entries while a batch is active; a successful rollback removes its +recorded entries. A process crash before a random stage is recorded can leave an +unclaimed stage for manual inspection; Scriptorium will not guess that it owns or +delete that path. Keep this candidate on synthetic or isolated copies until its +full V0.3 acceptance gate is complete. + ### Ten-minute path for a real project The following path creates a real Markdown project rather than using the synthetic @@ -298,6 +384,7 @@ Both paths are explicit and local; use `--json` for the stable machine report: ```powershell scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData --run +scriptorium resume --provenance-home D:\Research\ProvenanceData --project my-project ``` The paths must already exist. Scriptorium never falls back to the current directory @@ -345,6 +432,8 @@ reported as a successful live integration or provider check. Public Alpha worksp evidence requires at least one `Projects/*.md` note with complete `project/1.x` frontmatter; an arbitrary repository README is not treated as a research workspace. `entry.pull` passes only when the compatible machine-readable capability probe succeeds. +Public Alpha also requires compatible `prov-ingest-research` and `prov-context` +commands, and probes the actual Context Capsule runtime version. Codex provides the first executable session-capture path; a Claude-only installation remains a manual readiness item until its opt-in `SessionEnd` hook is live-verified. @@ -397,12 +486,12 @@ the current report. ## Compatibility baseline -The first golden path intentionally locks exact source versions as its coordinated -Public Alpha release targets: +The current V0.2 candidate intentionally locks exact source versions as its +coordinated compatibility baseline: -- `scriptorium-spec` 2.2.0 +- `scriptorium-spec` 2.3.0 - Steward 0.2.0 -- Provenance 0.17.0 +- Provenance 0.18.0 The demo and CI workflows continue to pin exact component commits. A range-based compatibility policy is intentionally deferred until external Alpha usage provides @@ -410,11 +499,13 @@ evidence for safe ranges. ## Next product increments -1. add a compact project context-capsule/resume entry over Provenance MCP, distinct - from the content-free control-plane `status`; -2. add an explicitly reviewed, adapter-specific migration manifest and apply path; -3. add schema-driven cross-repository E2E for Lectern handoff; -4. verify live Claude Code `SessionEnd` parity with the Codex capture path; +1. run the current migration, install lifecycle, and slide handoff gates on fresh + remote Windows CI and in an isolated user environment before promotion; +2. wire the approved `experiment-run/1.0` contract into local run registration, + persistence, and query, including failed runs, without embedding a compute engine; +3. wire `claim-evidence/1.0` into validation, human review, and explicit links + between run evidence and candidate claims; +4. verify real-provider Lectern output and live Claude Code `SessionEnd` parity; 5. run an external-user Alpha and use the evidence to shape packaging and compatibility ranges. diff --git a/README.zh.md b/README.zh.md index 7a018a0..faeb311 100644 --- a/README.zh.md +++ b/README.zh.md @@ -1,18 +1,23 @@ # Scriptorium -> **Public Alpha v0.1.0：** 当前 umbrella 仓库已交付真实项目的安全初始化、一个 +> **Public Alpha v0.2.0 候选版：** 当前 umbrella 仓库已交付真实项目的安全初始化、一个 > 通过 `scriptorium demo` 跑通的合成纵向切片、只读诊断入口 `scriptorium doctor`、 > 不暴露内容的 `scriptorium status` 控制面状态摘要、仅盘点显式本地来源且零写入的 > `scriptorium inventory` 预览，以及显式、项目级的 Codex / -> Claude Code 技能安装器。已确认的按需入口 `scriptorium pull` 也已通过 Provenance -> 的机器可读公共命令交付。GitHub-hosted 干净 Windows CI 与隔离的 Windows 源码 -> 安装验收已通过；Agent host 实时对等路径与 Lectern 仍不属于无凭据黄金路径。 +> Claude Code 技能安装器。已确认的按需入口 `scriptorium pull` 与有边界、只读的 +> `scriptorium resume` Context Capsule 均通过 Provenance 的机器可读公共命令交付。 +> 经过安全复审的 `scriptorium migrate` CLI 已作为 V0.3 候选能力加入，但不代表 +> V0.3 已经发布或验收完成。 +> 已发布基线具有 GitHub-hosted Windows 证据；当前工作树也已在本地通过合成迁移、 +> 安装、卸载、重装、v0.1.0 到当前候选版的版本切换，以及 Steward-to-Lectern +> 检查，但仍需重新运行远端 CI。Agent host 实时 +> 对等路径和真实 provider 幻灯片生成仍是人工 Alpha 门槛。 Scriptorium 是面向科研工作者的本地优先、Agent 原生研究工作流套件。本仓库是 它的薄控制面：通过公开 CLI 与版本化文件编排可独立使用的组件，不导入组件内部 模块，也不成为论文库、项目笔记或科研记忆的新数据主库。 -[English](README.md) · [中文产品案例](docs/case-study.zh-CN.md) · [展示与证据](docs/showcase/README.zh-CN.md) · [契约单一事实源](https://github.com/scriptorium-suite/scriptorium-spec) · [设计借鉴与致谢](ACKNOWLEDGEMENTS.zh.md) +[English](README.md) · [架构、使用与分层验收](docs/architecture-and-acceptance.zh-CN.md) · [中文产品案例](docs/case-study.zh-CN.md) · [展示与证据](docs/showcase/README.zh-CN.md) · [契约单一事实源](https://github.com/scriptorium-suite/scriptorium-spec) · [设计借鉴与致谢](ACKNOWLEDGEMENTS.zh.md) ![Scriptorium Public Alpha 合成黄金路径证据](docs/showcase/demo-poster.svg) @@ -39,9 +44,11 @@ workspace、数据根与项目选择。 1. 使用 `scriptorium-spec` 校验合成 `library-kb/1.1`； 2. 调用 Steward 选出两篇主题文献，并从录制的 Agent 草稿组装综述； 3. 调用 Provenance 摄取文献库与 Markdown 项目； -4. 构建和查询本地全文索引； -5. 通过 Provenance MCP 验证项目组合、当前上下文和文献检索； -6. 生成可阅读成果与机器可读的 `demo-report.json`。 +4. 校验并原子摄取四类合成文献阅读工件； +5. 构建和查询本地全文索引； +6. 通过 Provenance MCP 验证项目组合、有边界的 Context Capsule、仅供参考的 + 文献线索和文献检索； +7. 生成可阅读成果与机器可读的 `demo-report.json`。 该流程不需要 API Key、Zotero、Obsidian、浏览器扩展或 Agent 登录。源码安装 完成后，此 demo 路径被设计为不申请网络动作；它不会调用在线模型，Agent 产出是 @@ -63,6 +70,12 @@ doctor 才会通过 host adapter 检查。它会真实探测 `prov-sync-pull --c 才会提交低风险 timeline，并把高价值声明放入 `Approvals.md`。这些声明仍必须由用户 勾选，之后的 pull 才会提交。 +`scriptorium resume` 是会话开始时读取的精炼视图。它只接受兼容 `prov-context` +运行时返回的白名单 Capsule：把已批准项目状态、自动写入的低风险进度和仅供参考的 +文献/阅读工件明确分开；不会透传原始对话、草稿、已拒绝声明、组件 stderr 或本地路径。 +V0.2 端到端夹具会连续运行两个合成 Agent 会话，并验证第二次能恢复第一次经过审阅的 +项目状态。 + `scriptorium status` 是日常使用且不暴露内容的控制面状态摘要。它先重建 `doctor` 的 Public Alpha 就绪结论；只有该边界就绪时，才执行一次 `pull` 预览。结果只包含 白名单化的能力状态、工作流聚合计数和固定审阅提示，不会透传本地路径、项目或会话 @@ -79,6 +92,24 @@ Provenance 导入审阅、Steward 审阅四类路由。该预览不验证文件 资料，也不会声称迁移已经发生。在 Windows 上，命令会在预览期间以仅元数据句柄绑定 选中对象，因此其他进程需等命令结束后才能重命名、删除或以数据写权限打开这些对象。 +`scriptorium migrate` 是 V0.3 候选版的显式 Markdown/PDF 复制边界。`plan` +零写入；首次 `apply` 必须再次给出已核对的来源；后续 `apply`、`verify` 和 +`rollback` 只凭 workspace 与批次标识恢复。预览只是 advisory，不是持久化执行快照； +首次 `apply` 会重新扫描并哈希显式来源。目标文件通过同目录 hard link 以 +create-if-absent 方式发布，绝不覆盖。`apply` 使用随机、独占创建的 +`.scriptorium-*.stage`：同一个文件句柄完成复制、哈希、刷盘和文件身份检查后，才把 +stage 与身份封入私有 manifest，并发布为目标的同文件所有权锚点。若进程在封存前崩溃， +可能留下未认领的随机 stage；系统不会猜测归属，也不会自动删除它。 + +回滚会先持久化随机的同目录 quarantine 名，再用不覆盖的原子移动分别隔离目标和锚点， +移动后重新检查内容与稳定文件身份，确认归属后才删除。错误替代文件会尽量原子恢复到 +原路径；恢复受阻时保留在 quarantine 并 fail closed。自动回滚要求 Windows +no-replace rename 或 Linux `renameat2(RENAME_NOREPLACE)`；其他平台不降级执行。 +含绝对路径的私有 manifest 固定存放在 workspace 外的用户级 canonical 本地状态根， +迁移来源不得与该私有状态根重叠，CLI 不提供任意 `state-root` 参数。终端和 JSON +报告只含聚合计数与状态。迁移只复制 +文件，不会解析、索引、批准科研内容，也不会自动导入 Provenance。 + ## Windows 源码快速体验 前置条件：Git 与 Python 3.11+。先把四个仓库克隆到同一个父目录，使源码发现 @@ -122,6 +153,49 @@ python -m venv .venv 时会 fail closed 并返回退出码 `1`；参数错误或入口边界内部失败返回 `2`，且不会回显 敏感输入。 +### V0.3 候选：安全复制显式 Markdown/PDF + +先给批次一个稳定标识并检查聚合预览；首次执行时再次传入相同来源： + +```powershell +$Legacy = 'D:\Research\Legacy Notes' +$MigrationWorkspace = 'D:\Research\Scriptorium Workspace' +$Batch = 'legacy-notes-001' + +.\.venv\Scripts\scriptorium.exe migrate plan ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +.\.venv\Scripts\scriptorium.exe migrate apply ` + --source $Legacy ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json + +# 恢复操作不再需要来源路径或旧 plan。 +.\.venv\Scripts\scriptorium.exe migrate verify ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate apply ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +.\.venv\Scripts\scriptorium.exe migrate rollback ` + --workspace $MigrationWorkspace ` + --batch-id $Batch ` + --json +``` + +第二次 `apply` 是幂等/恢复检查，应返回 `unchanged`。目标文件系统必须支持 hard +link；不支持时命令会 fail closed，不会降级到可能覆盖目标的方式。批次生效期间不要 +删除内部 `.scriptorium-*.stage` 所有权锚点或 `.scriptorium-*.rollback` quarantine； +成功回滚会清理已登记条目。若进程在随机 stage 登记前崩溃，未认领 stage 只供人工核查， +Scriptorium 不会认领或自动删除。在完整 V0.3 +验收完成前，只应对合成资料或隔离副本使用该候选能力。 + ### 真实项目的 10 分钟路径 以下命令创建真实 Markdown 项目，而不是使用合成 demo。workspace 与 Provenance @@ -262,6 +336,7 @@ scriptorium host install claude-code --workspace D:\Research\MyProject ```powershell scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData scriptorium pull --workspace D:\Research\Workspace --provenance-home D:\Research\ProvenanceData --run +scriptorium resume --provenance-home D:\Research\ProvenanceData --project my-project ``` 两个路径都必须已经存在；Scriptorium 永不把当前目录隐式当作科研数据根。已登记的 @@ -301,7 +376,9 @@ scriptorium doctor --json ` 与真实网络行为均明确标为未测试。检测到应用或命令，不等于对应 live integration、 provider 或真实产出已经验证。Public Alpha 的 workspace 证据至少要求一个带完整 `project/1.x` frontmatter 的 `Projects/*.md` 笔记；普通仓库 README 不会被误判为科研 -workspace。`entry.pull` 只有在兼容的机器可读能力探针通过时才通过。Codex 是当前第一条 +workspace。`entry.pull` 只有在兼容的机器可读能力探针通过时才通过。Public Alpha +还要求兼容的 `prov-ingest-research` 与 `prov-context` 命令，并探测真实 Context Capsule +运行时版本。Codex 是当前第一条 可执行会话捕获路径；仅安装 Claude 的配置在其 opt-in `SessionEnd` hook 完成 live 验证前， 仍会被诚实标为人工就绪项。 @@ -350,21 +427,19 @@ scriptorium-demo/ ## 当前兼容基线 -- `scriptorium-spec` 2.2.0 +- `scriptorium-spec` 2.3.0 - Steward 0.2.0 -- Provenance 0.17.0 +- Provenance 0.18.0 -首个 golden path 将以上版本作为协同 Public Alpha 的发布目标，并在 demo 与 CI 中 -继续固定精确组件提交。版本范围兼容策略暂不靠猜测定义，而是在获得外部 Alpha -使用证据后再确定。 +当前 V0.2 候选版将以上版本作为协同兼容基线，并在 demo 与 CI 中继续固定精确组件 +提交。版本范围兼容策略暂不靠猜测定义，而是在获得外部 Alpha 使用证据后再确定。 ## 紧邻的产品增量 -1. 在 Provenance MCP 之上增加精炼的项目 context-capsule/resume 入口，并与不暴露 - 内容的控制面 `status` 明确区分； -2. 增加需要显式人审的适配器级迁移清单与执行路径； -3. 补齐 Lectern handoff 的 schema 驱动跨仓 E2E； -4. 验证 Claude Code `SessionEnd` 与 Codex 捕获路径的实时对等性； +1. 在全新的远端 Windows CI 与隔离用户环境中复跑当前迁移、安装生命周期和幻灯片交接门禁后，再升级其发布状态； +2. 将已批准的 `experiment-run/1.0` 契约接入本地运行登记、持久化和查询，保留失败运行，但不把计算引擎内置进套件； +3. 将 `claim-evidence/1.0` 接入验证、人工审阅，以及运行证据与候选 Claim 的显式关联； +4. 验证 Lectern 真实 provider 输出，以及 Claude Code `SessionEnd` 与 Codex 捕获路径的实时对等性； 5. 开展外部用户 Alpha，用实测结果决定安装打包与兼容版本范围。 Apache-2.0，无遥测。 diff --git a/docs/architecture-and-acceptance.zh-CN.md b/docs/architecture-and-acceptance.zh-CN.md new file mode 100644 index 0000000..43fe7e4 --- /dev/null +++ b/docs/architecture-and-acceptance.zh-CN.md @@ -0,0 +1,443 @@ +# Scriptorium 架构、使用与分层验收说明 + +> 文档状态：开发验收说明，依据 2026-07-23 的公开代码工作区编写。 +> +> 当前已发布的 umbrella 基线仍是 Public Alpha v0.1.0；工作树正在形成 V0.2.0 候选版。本文中的 V0.2、V0.3、V0.4 是分层验收目标，不等同于已经发布。 + +## 1. 一句话理解这个产品 + +**Scriptorium 是本地优先、模型中立的科研项目控制层。它连接研究者、AI Agent、文献工具和计算执行器，把研究问题、会话、实验、证据和交付物沉淀为可恢复、可审查、可追溯的项目状态。** + +通俗地说，它不是替研究者自动“做完科研”，也不是另一个聊天机器人。它负责让科研过程不断线： + +- 新一轮 AI 协作能知道项目做到哪里； +- 文献、会话、判断和交付物能找到来源； +- AI 建议与研究者已经确认的结论不会混在一起； +- 关键写入先预览、再确认，必要时可以回滚； +- Codex、Claude Code 或其他 Agent 可以更换，项目状态仍留在本地文件和版本化契约中。 + +“实验进入项目状态”是产品方向。`experiment-run/1.0` 与 `claim-evidence/1.0` +已经作为正式文件契约存在，但当前还没有运行登记命令、持久化与查询路径、结果与 Claim +的关联，也没有统一执行器接口。现阶段实验仍由用户在 Jupyter、Python 或领域软件中运行， +再把经过审阅的结果接回项目。 + +## 2. 各组件分别做什么 + +| 组件 | 面向用户的职责 | 不负责什么 | +|---|---|---| +| **Scriptorium（umbrella）** | 统一入口；初始化项目；检查环境；盘点显式选中的资料；触发会话同步；读取恢复上下文；安装项目级 Agent Skill | 不直接拥有 Zotero、记忆库或 PPT 编译器；不在后台偷偷运行 | +| **scriptorium-spec** | 定义各组件交换的 JSON Schema 和约定，是跨仓库数据契约的唯一事实来源 | 没有运行时产品，也不保存用户研究数据 | +| **Steward** | 管理文献资料；导出文献 KB；解析论文；生成阅读、脉络、综述和 `handoff/1.x` 交接包 | 不替用户确认科学结论；不作为项目记忆所有者 | +| **Provenance** | 保存会话、项目状态、审批候选、文献研究产物和检索索引；生成有边界的 Context Capsule | 不替 Agent 做开放式推理；不会把 `reference_only` 资料自动升级为正式结论 | +| **外部 Agent** | 阅读项目上下文，辅助梳理问题、查文献、设计分析、填写待审候选 | 不是项目事实的最终裁决者 | +| **外部计算执行器** | 运行 Python、Jupyter、领域软件或其他可验证实验 | 当前尚未通过统一的 Scriptorium 实验账本接线 | +| **Lectern** | 消费论文或 `handoff/1.x`，先生成可审阅提纲，再编译为可编辑 `.pptx` | 离线跨仓验收使用 `FakeLLM`，不等同于真实 provider、真实论文或 PowerPoint 人工 UAT | + +Obsidian、Zotero 和 PowerPoint 是可选应用。核心交换面是本地 Markdown 与版本化 JSON；没有这些应用时，项目仍应能够以文件方式工作，只是体验会降级。 + +## 3. 当前整体架构 + +```mermaid +flowchart TD + U["研究者：目标、审批、科学判断"] --> S["Scriptorium 统一入口"] + A["Codex / Claude Code / 其他 Agent"] <--> S + S <--> W["本地 Markdown 工作区"] + S <--> P["Provenance：会话、项目状态、候选审批、Context Capsule"] + S <--> T["Steward：文献 KB、解析、阅读、综述、handoff"] + T --> C["scriptorium-spec：版本化文件契约"] + P --> C + W --> C + E["Jupyter / Python / 领域计算工具"] -. "当前由用户显式运行并人工接回" .-> W + T --> H["handoff/1.x"] + H --> L["Lectern：提纲审阅 -> 可编辑 PPTX"] +``` + +组件之间的基本原则是：**通过公开命令、只读 MCP 和文件契约协作，不直接调用彼此内部实现。** + +数据边界如下： + +- 默认保存在本机； +- 本地命令不应自行上传资料； +- 联网文献检索只能发送公开检索词、题名、标识符或用户明确批准的材料； +- 使用外部模型或云解析器时，由对应 Agent、Steward 或 Lectern 的显式配置决定，不能把它描述成离线； +- 私有笔记、未发表结果、原始会话、凭据和本地路径不得进入公开测试数据或公开报告。 + +## 4. 整体怎么使用 + +### 4.0 先跑公开 synthetic golden flow + +在接触真实资料前，先用仓库自带的完全合成数据检查组件接线： + +```powershell +scriptorium demo ` + --output ` + --spec-root ` + --steward-root ` + --provenance-root +``` + +当前 demo 会验证版本化示例、Steward 文献综述、Provenance 项目与研究产物导入、重复导入幂等、Context Capsule、检索和只读 MCP。会话的 `init -> pull -> approval -> resume` 双轮闭环由独立的合成 E2E 测试验证。 + +主 `scriptorium demo` 不读取真实 Zotero、真实 Agent 日志或模型凭据，也不调用 Lectern +和外部实验。另有 `tests/e2e_slides.py` 使用程序生成的两篇合成 PDF，验证 +`Steward -> handoff/1.1 -> Spec -> Lectern -> 提纲审批 -> 可编辑 PPTX`；这条测试同样 +不调用真实 provider。两条链路都只能证明软件按契约协作，**不能证明真实科研结论、 +实时模型路径或外部用户安装已经通过**。 + +### 4.1 首次建立项目 + +先检查组件是否可用： + +```powershell +scriptorium doctor --target public-alpha --json +``` + +然后预览初始化计划。默认不写入： + +```powershell +scriptorium init ` + --workspace ` + --provenance-home ` + --project-id ` + --title "" ` + --host codex ` + --idea "<research-intuition>" ` + --json +``` + +确认工作区、数据根目录、项目标识和初始研究直觉都正确后，才增加 `--run`。初始化会建立最小 Markdown 工作区和套件配置，不会自动读取 Zotero、模型密钥或整台电脑。 + +### 4.2 盘点已有资料 + +`inventory` 只扫描用户明确给出的 Markdown、PDF、会话导出或 Zotero 导出位置，并输出聚合预览： + +```powershell +scriptorium inventory --source <selected-file-or-directory> --json +``` + +它当前是**零写入盘点边界**：不复制、不解析、不索引、不迁移。报告用于回答“有多少资料、可以路由到哪里、哪些类型暂不支持”，而不是展示私人正文。 + +### 4.3 迁移资料：V0.3 候选流程 + +Markdown/PDF 迁移引擎已经接入公开 `scriptorium migrate` CLI，但仍定位为 +**V0.3 candidate**，不代表 V0.3 整体已经验收。入口只接受用户显式给出的来源， +默认不联网、不调用模型、不解析正文，也不会把副本自动升级为 Provenance 事实。 + +```powershell +# 零写入预览；只输出聚合计数。 +scriptorium migrate plan ` + --source <selected-file-or-directory> ` + --workspace <isolated-workspace> ` + --batch-id <stable-batch-id> ` + --json + +# 新批次首次 apply 必须再次给出来源。 +scriptorium migrate apply ` + --source <selected-file-or-directory> ` + --workspace <isolated-workspace> ` + --batch-id <stable-batch-id> ` + --json + +# 已有批次仅凭 workspace + batch ID 恢复。 +scriptorium migrate verify --workspace <isolated-workspace> --batch-id <stable-batch-id> --json +scriptorium migrate apply --workspace <isolated-workspace> --batch-id <stable-batch-id> --json +scriptorium migrate rollback --workspace <isolated-workspace> --batch-id <stable-batch-id> --json +``` + +其中第二次 `apply` 应返回 `unchanged`。私有 manifest 固定写到 workspace 外的 +canonical 用户级本地状态根，CLI 不允许分叉到任意 `state-root`。`plan` 是零写入 +advisory preview，不是持久化执行快照；首次 `apply` 会重新扫描并哈希显式来源。 +目标文件先以随机 +O_EXCL 名在目标同目录暂存；同一 fd 完成复制、哈希、刷盘与 `fstat` 后，才把随机 +stage 名与稳定文件身份封入 manifest，再以 hard link 原子执行 create-if-absent。 +文件系统不支持时 fail closed，不会降级为可覆盖发布。封存前崩溃可能留下未认领随机 +stage；后续重试不会扫描、认领或自动删除它，因此不会把竞争者文件误当成自己的锚点。 + +回滚采用 `target -> target-quarantined -> anchor -> anchor-quarantined -> deleted` +持久化状态机。每一步先记录 128-bit 随机同目录 quarantine 名，再用 Windows +no-replace rename 或 Linux `renameat2(RENAME_NOREPLACE)` 原子移动，移动后重新验证 +哈希、大小和持久文件身份；错误替代文件只恢复或保留，不删除。平台不支持原子 +no-replace move、恢复路径被占用或身份不一致时均 fail closed。active batch 期间不得 +删除已登记的 `.scriptorium-*.stage` / `.scriptorium-*.rollback`；成功回滚最后清理 +已登记条目，未认领 orphan 只供人工核查。终端与 +JSON 报告、参数错误和运行时错误均不回显路径、文件名或正文。 + +当前已由合成测试覆盖显式输入、无覆盖、拒绝链接/重解析点、进程退出释放锁、字节哈希 +一致、随机 stage 抢占保护、封存前 crash 重试、四阶段 quarantine crash 恢复、 +来源丢失后的幂等重跑、可恢复回滚与私有状态外置；还需要把干净 +Windows CI、隔离用户 UAT 和更广泛文件系统兼容结果作为升级候选状态的发布证据。 + +### 4.4 日常会话闭环 + +每次继续项目时，先读取有边界的项目摘要： + +```powershell +scriptorium resume --project <project-id> --json +``` + +`resume` 是只读操作。它返回项目目标、已批准状态、近期进展、下一步和参考资料提示，不把所有历史对话塞进上下文。文献研究产物固定标为 `reference_only`，只能作为导航，不能直接当成已确认的科学结论。 + +Agent 完成本轮工作后，先预览会话同步： + +```powershell +scriptorium pull --project <project-id> --json +``` + +只有在用户明确同意推进同步后才运行： + +```powershell +scriptorium pull --project <project-id> --run --json +``` + +需要注意： + +- 未解析到项目的会话先进入 project resolution，不能生成正式 `session-summary/1.0`； +- Agent 生成的总结填充是候选内容，不能自行写成项目事实； +- 低风险时间线与高价值结论采用不同的审批边界； +- `Approvals.md` 只能由用户审阅和勾选，Agent 不得代替用户批准； +- 重复执行不应重复添加同一事件。 + +### 4.5 文献工作 + +有 Zotero 时，可以由 Steward 对本地文献库做备份、只读盘点和导出；没有 Zotero 时，也可以从显式文件和已经导出的 KB 开始。典型产物包括： + +- `library-kb/1.x`：文献目录； +- `parsed-paper/1.0`：结构化论文； +- `reading-note/1.0`：分阶段阅读笔记； +- `lineage-graph/1.0`：库内引用脉络； +- `review/1.0`：综述产物； +- `handoff/1.x`：交给 Lectern 的论文与元数据包。 + +这些文件应先通过 scriptorium-spec 验证，再导入 Provenance。导入后仍保持参考资料身份，直到研究者把具体论断与原始证据核对并批准。 + +### 4.6 外部实验 + +当前建议这样做： + +1. 从 Context Capsule 和项目笔记中选出一个可证伪问题； +2. 在隔离的 Jupyter、Python 或领域软件环境运行最小实验； +3. 保存代码、输入版本、参数、随机种子、日志和输出文件； +4. 对关键数字做独立复算； +5. 由研究者审阅后，把结论、限制和下一步写回项目。 + +**尚未完成：**对 `experiment-run/1.0` 的运行登记、持久化和查询，对 +`claim-evidence/1.0` 的人工审阅与 Claim 关联，以及失败实验的可检索经验库。两个 Schema +只定义跨组件交换形状；因此“脚本成功退出”目前不能被 Scriptorium 自动解释为 +“科学结论成立”。 + +### 4.7 生成汇报 PPT + +Steward 先生成 `handoff/1.x`，Lectern 再按“先提纲、后编译”的方式工作： + +```powershell +lectern outline <handoff-directory-or-pdf> --out outline.json +``` + +研究者检查叙事、页数、图表、引用和保密边界后，明确批准该提纲，再运行： + +```powershell +lectern build --from-outline outline.json --out deck.pptx +``` + +不建议 Agent 使用一条命令直接从来源生成 PPT，因为 one-shot 路径会自动跨过提纲审批。 +套件跨仓合成验收已经证明：审批前输出目录没有 PPTX，批准后生成的标题和原生表格可在 +`python-pptx` 中重新打开、修改、保存并再次打开。真实模型路径与本机 PowerPoint 打开编辑 +仍属于 V0.4 私有 UAT。 + +## 5. 分层验收标准 + +### 5.1 V0.2：证明“下一次协作接得上” + +目标是把文献研究产物、两轮会话和项目状态接入同一个可恢复闭环。 + +公开 synthetic 验收至少应满足： + +1. `init` 预览零写入，执行后再次运行保持幂等； +2. `doctor` 能识别精确兼容的 Spec、Steward 和 Provenance； +3. 两轮合成 Agent 会话经过 `pull`、项目解析、候选填充和人工审批边界； +4. 第二轮 `resume` 能看到两轮已经接受的进展，而不是原始私聊全文； +5. 四类合成研究产物——`parsed-paper`、`reading-note`、`review`、`lineage-graph`——通过 Schema 验证并导入； +6. 相同产物逆序重复导入后，新增数为零且存储字节稳定； +7. Context Capsule 有固定大小上限，不含本地路径，研究产物均为 `reference_only`； +8. 未解析项目不生成 `session-summary/1.0`； +9. 测试输出和公开 artifact 通过邮箱、凭据、绝对路径与私人关键词扫描； +10. Windows 为阻断平台，Ubuntu 至少保持回归通过。 + +当前判断：V0.2 的 `resume`、研究产物导入、合成 demo 扩展和双会话 E2E 已通过本地 +自动化验证；它们还未形成正式 V0.2 发布。远端 Windows 与 Ubuntu CI、干净 diff、最终 +版本提交和 release tag 仍是发布结论的必要证据。 + +### 5.2 V0.3：证明“资料、实验和交付物走得通” + +V0.3 应在 V0.2 基础上新增三条可验收能力： + +1. **安全迁移** + - `inventory -> plan -> apply -> verify -> rollback` 有公开 CLI； + - 只接受显式选中的 Markdown/PDF； + - 不覆盖、不跟随链接、哈希一致、重复执行幂等； + - 私有路径只存在本机 manifest，公开报告只含计数和状态。 +2. **最小实验账本** + - 有经过批准的版本化契约； + - 能记录运行身份、代码版本、输入、参数、环境、状态、指标和产物哈希； + - 失败运行也保留； + - “运行完成”与“结论被证据支持”是两个独立状态。 +3. **汇报交付闭环** + - Steward 生成合法 `handoff/1.x`； + - Lectern 生成提纲并停在人工审批； + - 批准后生成可打开、可编辑的 `.pptx`； + - 每个关键结论能回到来源或明确标为推测。 + +当前判断：安全迁移 CLI 与 Lectern 跨仓合成 E2E 已在开发工作区通过； +`experiment-run/1.0` 与 `claim-evidence/1.0` 正式契约也已存在，但 umbrella 与 +Provenance 尚未接入运行登记、持久化、查询和 Claim 关联，远端 Windows CI 也还没有运行。 +因此 V0.3 目前仍不能整体验收通过。 + +### 5.3 V0.4：证明“别人能在 Windows 上独立用起来” + +V0.4 是面向外部 Alpha 的产品验收，而不只是开发者测试： + +1. 一台干净 Windows 环境能按文档完成安装、升级和卸载； +2. Codex 与 Claude Code 各完成至少一轮真实但隔离的会话恢复与收尾； +3. Zotero、Obsidian、PowerPoint、Lectern 缺失时给出明确降级，不破坏 Markdown 核心路径； +4. 可选组件存在时，能完成文献接入和 PPT 交付； +5. 所有默认动作本地优先、预览优先、无后台常驻、无隐式网络； +6. 至少一名未参与开发的目标用户，在不依赖开发者代操作的情况下完成： + - 建立或迁入一个项目； + - 恢复上下文； + - 完成一轮有证据的研究推进； + - 关闭会话并在下一轮恢复； + - 导出一个可审阅交付物； +7. 记录任务完成率、首次成功耗时、阻塞点、误操作和用户是否理解审批边界； +8. 发布包、CI artifact、截图和文档经过隐私与凭据扫描。 + +当前判断：本地隔离环境中的安装、卸载、重装和 v0.1.0 到 v0.2.0 的版本切换已经通过； +全新远端或独立干净 Windows 环境、真实双 Agent 对等验证和外部用户 Alpha 仍未完成， +因此 V0.4 不能宣称完成。 + +## 6. 验收数据如何分级 + +| 等级 | 可以包含什么 | 可以放到哪里 | 规则 | +|---|---|---|---| +| **S0：公开合成数据** | 明确标注 `[SYNTHETIC]` 的虚构论文、项目、会话、指标和 PPT 内容 | 仓库、CI、公开 demo、截图 | 不得从真实研究材料改写后假装合成 | +| **S1：生成的无害化数据** | 由 S0 运行生成的聚合报告、哈希、状态、匿名 ID、可公开产物 | 隔离测试目录；通过扫描后可作为 CI artifact | 不含邮箱、用户名、绝对路径、密钥、原始会话或真实研究内容 | +| **S2：本机真实研究数据** | 用户自己的论文、笔记、会话、实验和项目状态 | 仅限隔离的本机 UAT | 永不提交、永不上传、永不进入公开截图或 CI | +| **S3：秘密与环境身份** | API key、token、Cookie、邮箱、本机用户名、绝对路径、环境变量、私有远程地址 | 仅在必要的本机私有配置中 | 禁止进入测试输入、报告、日志、artifact、提交和文档示例 | + +公开验收以 S0 和合规的 S1 为主。S2 只能证明真实使用场景可行，不能复制到公开仓库来“展示效果”。S3 不属于产品数据，任何发现都应视为发布阻断问题。 + +## 7. 本机有哪些数据可以做私有 UAT + +本机已知有以下数据类别可供验证，但这里只说明用途，不记录任何真实正文、题名、人员或绝对路径： + +| 本机数据类别 | 适合验证什么 | 建议首轮规模 | +|---|---|---| +| Zotero 本地论文库 | 文献盘点、KB 导出、单篇解析、阅读状态 | 只选 1–2 篇 | +| 论文写作目录中的 PDF、Markdown、Word、表格和演示文稿 | `inventory` 类型识别、Markdown/PDF 安全迁移、交付物关联 | 1 个 Markdown + 1 个 PDF | +| Codex 本地会话日志 | 项目解析、候选总结、审批、`pull -> resume` | 1 个已知项目会话 | +| Claude Code 本地会话日志 | 与 Codex 路径的行为差异和 SessionEnd 接线 | 1 个已知项目会话 | +| 现有 Markdown 研究库 | 项目笔记识别、进展恢复、人工内容保护 | 复制 1 个最小项目到隔离区 | +| 现有 Provenance 收件箱与记忆 | 与旧状态的只读对照、重复导入和恢复检查 | 只读对照，不直接写入 | +| 小型、可重复的本地科研计算项目 | 未来实验账本的运行、指标、失败记录和哈希验证 | 1 个分钟级确定性实验 | +| 大型科研数据库或重型计算数据 | 后续性能与容量测试 | 不用于首轮 UAT | + +上述数据全部属于 S2。**不得上传到 GitHub、CI、第三方模型、网页工具或公开 issue。** 如果实验需要联网模型或云解析器，应改用 S0，或者先逐项确认允许发送的最小材料。 + +## 8. 隔离 UAT 操作步骤 + +1. 新建一个专用 UAT 根目录，内部再分开建立 `workspace`、`provenance-home`、`inputs` 和 `outputs`；不要把测试目录嵌套在当前真实库中。 +2. 保持原始来源只读。首轮只复制一个 Markdown、一个 PDF、一个 Agent 会话和一个小实验所需的最小输入。 +3. 在执行前记录输入文件清单、字节大小和 SHA-256；记录可以保存在本机私有 UAT 报告中，但不要公开绝对路径。 +4. 先用 S0 完成相同命令的 synthetic golden flow，再切换到 S2。 +5. 对所有会写入的命令先运行 preview 或 dry-run，确认目标根目录、数量和动作类型。 +6. 明确把 `workspace`、`provenance-home`、临时目录和配置目录指向 UAT 根目录；不得复用真实 Provenance home、真实 vault 或默认用户配置。 +7. 关闭非必要网络。若某一步必须联网，先确认只发送公开题名、标识符或已经批准的测试材料。 +8. 每一步后检查： + - 原始输入哈希未变化； + - 没有写到 UAT 根目录以外； + - 报告没有正文、邮箱、密钥和绝对路径； + - 待审批内容没有自动变成正式状态。 +9. 重复运行相同步骤，验证没有重复记录、重复复制或字节漂移。 +10. 用 `migrate verify` 检查批次后，在隔离副本上执行 `migrate rollback`；回滚只能删除本批次确认创建且身份、字节均未变的副本，不会删除或覆盖替代文件；恢复受阻时必须保留 quarantine 并 fail closed。 +11. 用新会话运行 `resume`，确认只恢复已接受的状态，拒绝项、原始聊天和未审批草稿没有进入胶囊。 +12. 最终 UAT 报告只记录版本、操作系统、通过/失败、计数、匿名 ID、耗时和已知限制。公开前再做一次秘密、路径和私人关键词扫描。 + +## 9. “流程跑通”不等于“科学有效” + +这是整个产品验收中最重要的边界。 + +**工作流成功**可以证明： + +- 文件符合 Schema； +- 会话能恢复； +- 引用和产物有来源； +- 命令可重复； +- 哈希一致； +- 审批边界没有被绕过； +- PPT 能生成并打开。 + +**科学有效**还需要证明： + +- 研究问题可证伪； +- 数据和方法适合该问题； +- 基线比较公平； +- 统计和误差分析成立； +- 结果能复现； +- 反例和冲突证据被认真处理； +- 引用原文确实支持该论断； +- 领域专家或研究者完成最终判断。 + +因此，一条实验记录至少应区分： + +```text +execution_succeeded +result_reproduced +evidence_reviewed +claim_supported +``` + +前一项为真，不能自动推出后一项为真。Context Capsule 中的 `reference_only` 也不能自动升级为 `claim_supported`。 + +## 10. 清理历史垃圾文件时的红线 + +可以列为候选、但必须在测试结束后逐项确认再删除： + +- 本轮明确创建的 synthetic demo 目录； +- 可重新生成的测试缓存、覆盖率文件、构建目录和临时日志； +- 已确认不再需要的隔离 UAT 副本； +- 过期的发布临时克隆或 CI artifact 下载副本。 + +不要因为“体积大”就当成垃圾。虚拟环境和前端依赖通常可重建，但在版本测试完成前应保留；它们与历史隐私垃圾不是同一类问题。 + +绝不能按通配符或整目录清理： + +- 原始私有仓库和保留的私有历史归档； +- Zotero 数据目录及附件； +- 论文、数据集、实验输出和项目笔记； +- Provenance 的 inbox、memory、vault、sync state、profile、搜索索引和输出； +- `.env`、凭据配置和用户级 Agent 配置； +- Codex、Claude Code 或其他 Agent 原始日志； +- 未确认来源的 untracked 文件； +- 任何备份。 + +安全清理流程必须是： + +```text +列出确切绝对目标 + -> 说明为什么可删除、是否可重建、占用大小 + -> 用户逐项确认 + -> 再执行删除 + -> 复查 Git 状态和关键数据仍在 +``` + +在未得到精确目标确认前，只做审计，不做删除。 + +## 11. 最终发布验收的判定方式 + +每个版本都应同时具备四类证据： + +1. **自动化证据**：单元测试、跨仓 E2E、Windows CI、Ubuntu 回归、Schema 验证； +2. **产品证据**：一条从项目建立到下一轮恢复的可观察用户旅程； +3. **隐私证据**：公开仓库、release、artifact、截图和日志的扫描结果； +4. **人工证据**：真实用户完成任务并理解“参考资料、候选结论、已批准状态”三者的区别。 + +只有代码存在，不算完成；只有 demo 跑通，也不算产品完成。V0.4 的真正结束条件是：一个目标科研用户能在干净 Windows 环境独立完成核心闭环，同时没有泄露私有数据，也没有把流程成功误报成科学发现。 diff --git a/docs/case-study.zh-CN.md b/docs/case-study.zh-CN.md index a8de1d9..695d3fa 100644 --- a/docs/case-study.zh-CN.md +++ b/docs/case-study.zh-CN.md @@ -78,8 +78,11 @@ flowchart LR 6. `demo-report.json` 固化版本、阶段、断言、产物和限制，使结果可重复核验。 这里没有 API Key、真实 Zotero、真实对话、在线模型或私人研究资料。Agent 草稿是仓库内 -明确标注的 fixture。Lectern 也没有被算入本条黄金路径，因为生成幻灯片需要独立配置的 -provider 路径。重复运行在功能上幂等，但带摄取时间戳的记录不承诺逐字节一致。 +明确标注的 fixture。主 `scriptorium demo` 仍不调用 Lectern；另有一条离线跨仓验收使用 +`FakeLLM` 和程序生成的合成 PDF，专门验证 `Steward -> handoff/1.1 -> Spec -> +Lectern 生产图 -> 人工提纲审批 -> 可编辑 PPTX`；临时质量检查渲染在进入审批前清理， +不会在输出树中留下未审批 PPTX。重复运行在功能上幂等，但带摄取时间戳的记录 +不承诺逐字节一致。 ## 安全机制如何进入产品细节 @@ -103,7 +106,7 @@ provider 路径。重复运行在功能上幂等，但带摄取时间戳的记 | 远端候选证据链 | commit `8069cbf1` / run `29818934741` / artifact `8490533423` | 通过；旧悬空对象待 GitHub Support 清理 | | 完整 Agent 工作流用户价值 | 尚无外部 beta 行为数据 | 未验证 | | Claude Code SessionEnd 对等路径 | 仍缺 live golden-path 证据 | 未验证 | -| Lectern 跨仓生成 | 不属于当前 credential-free demo | 未验证 | +| Lectern 跨仓生成 | 两篇程序生成的合成 PDF，经 Steward、Spec 和 Lectern 生产图；审批前输出树无持久 PPTX，审批后重新打开并修改原生标题和表格 | 本地自动验收通过；真实 provider 与 PowerPoint 人工打开仍待 UAT | “未验证”不是隐藏在脚注里的免责声明，而是下一阶段产品判断的输入。 @@ -127,10 +130,10 @@ provider 路径。重复运行在功能上幂等，但带摄取时间戳的记 - 当前以 GitHub 源码安装为主，还没有 PyPI/winget 或图形安装器； - editable 源码安装可能从已配置的包索引下载 Python 构建依赖；“运行期不申请网络动作” 不是离线安装承诺； -- `inventory` 当前只做不读取正文的安全盘点与路由，实际迁移仍缺受审阅的 - manifest/apply 闭环； +- `inventory` 仍只做安全盘点与路由；Markdown/PDF 的 + `migrate plan/apply/verify/rollback` 已形成开发候选，但尚未经过远端 Windows CI 和正式发布； - 完整 Public Alpha 需要用户已经安装 Codex 或 Claude Code，并主动安装项目级 Skill； -- Claude Code `SessionEnd` live 对等验证、Lectern schema-driven E2E 与外部用户 beta 仍待完成； +- Claude Code `SessionEnd` live 对等验证、Lectern 真实 provider 路径和外部用户 beta 仍待完成； - demo 的“无网络动作”来自实现与环境白名单，尚未由操作系统级网络沙箱观测； - 尚无真实用户效率数据，因此不宣称已经证明科研质量或节省了确定比例的时间。 diff --git a/pyproject.toml b/pyproject.toml index 9da3767..e20317a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scriptorium-suite" -version = "0.1.0" +version = "0.2.0" description = "Thin, agent-native entry point for the Scriptorium research workflow suite" readme = "README.md" requires-python = ">=3.11" diff --git a/src/scriptorium/__init__.py b/src/scriptorium/__init__.py index a062522..b234f47 100644 --- a/src/scriptorium/__init__.py +++ b/src/scriptorium/__init__.py @@ -1,3 +1,3 @@ """Scriptorium suite entry point.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/scriptorium/assets/compatibility.toml b/src/scriptorium/assets/compatibility.toml index a87787a..a40f9f7 100644 --- a/src/scriptorium/assets/compatibility.toml +++ b/src/scriptorium/assets/compatibility.toml @@ -1,6 +1,6 @@ manifest_version = 1 [components] -"scriptorium-spec" = "2.2.0" +"scriptorium-spec" = "2.3.0" steward = "0.2.0" -provenance = "0.17.0" +provenance = "0.18.0" diff --git a/src/scriptorium/assets/lineage-graph.v1.json b/src/scriptorium/assets/lineage-graph.v1.json new file mode 100644 index 0000000..5379e7d --- /dev/null +++ b/src/scriptorium/assets/lineage-graph.v1.json @@ -0,0 +1,42 @@ +{ + "schema_version": "lineage-graph/1.0", + "generated_by": "scriptorium synthetic fixture", + "direction": { + "query": "[SYNTHETIC] Physics-informed active learning for catalyst discovery", + "scope_method": "synthetic own-library citation match", + "created": "2026-07-15T10:15:00Z" + }, + "nodes": [ + { + "citekey": "chen2022PhysicsInformed", + "title": "[SYNTHETIC] Physics-informed learning for materials discovery", + "year": "2022", + "cluster": "synthetic-foundations" + }, + { + "citekey": "liu2024ActiveLearning", + "title": "[SYNTHETIC] Active learning for closed-loop materials experiments", + "year": "2024", + "cluster": "synthetic-closed-loop" + } + ], + "edges": [ + { + "from": "liu2024ActiveLearning", + "to": "chen2022PhysicsInformed", + "relation": "cites", + "evidence": "[SYNTHETIC] Generated reference recorded in parsed-paper.v1.json." + } + ], + "clusters": [ + { + "id": "synthetic-foundations", + "label": "[SYNTHETIC] Physics-informed foundations" + }, + { + "id": "synthetic-closed-loop", + "label": "[SYNTHETIC] Closed-loop experiment selection" + } + ], + "timeline": ["chen2022PhysicsInformed", "liu2024ActiveLearning"] +} diff --git a/src/scriptorium/assets/parsed-paper.v1.json b/src/scriptorium/assets/parsed-paper.v1.json new file mode 100644 index 0000000..3621ffe --- /dev/null +++ b/src/scriptorium/assets/parsed-paper.v1.json @@ -0,0 +1,43 @@ +{ + "schema_version": "parsed-paper/1.0", + "id": "liu2024ActiveLearning", + "created": "2026-07-15T09:00:00Z", + "parser": "scriptorium synthetic fixture", + "source_pdf": "fixtures/synthetic-active-learning.pdf", + "metadata": { + "title": "[SYNTHETIC] Active learning for closed-loop materials experiments", + "authors": ["Synthetic Author"], + "year": "2024", + "doi": "10.5555/demo.0002", + "abstract": "[SYNTHETIC] This generated abstract exists only to test the Scriptorium workflow." + }, + "sections": [ + { + "heading": "Synthetic method", + "level": 1, + "text": "[SYNTHETIC] A deterministic fixture ranks fictional catalyst candidates by generated uncertainty values." + }, + { + "heading": "Synthetic result", + "level": 1, + "text": "[SYNTHETIC] The fixture demonstrates traceable evidence flow and makes no scientific performance claim." + } + ], + "references": [ + { + "raw": "Chen (2022). [SYNTHETIC] Physics-informed learning for materials discovery.", + "title": "[SYNTHETIC] Physics-informed learning for materials discovery", + "authors": ["Chen"], + "year": "2022", + "doi": "10.5555/demo.0001" + } + ], + "figures": [ + { + "id": "fig-synthetic-loop", + "caption": "[SYNTHETIC] Generated closed-loop selection diagram.", + "page": 2 + } + ], + "tables": [] +} diff --git a/src/scriptorium/assets/reading-note.v1.json b/src/scriptorium/assets/reading-note.v1.json new file mode 100644 index 0000000..955f993 --- /dev/null +++ b/src/scriptorium/assets/reading-note.v1.json @@ -0,0 +1,30 @@ +{ + "schema_version": "reading-note/1.0", + "id": "liu2024ActiveLearning", + "zotero_key": "DEMO0002", + "doi": "10.5555/demo.0002", + "read_status": "In Progress", + "created": "2026-07-15T09:30:00Z", + "generated_by": "scriptorium synthetic fixture", + "stages": { + "glance": { + "tldr": "[SYNTHETIC] Generated note about combining calibrated active learning with physics-informed priors.", + "tags": ["synthetic", "active-learning", "calibration"], + "key_findings": [ + "[SYNTHETIC] The fixture records uncertainty calibration as a candidate design concern." + ] + }, + "close_read": { + "question": "[SYNTHETIC] Can a closed loop prioritize informative fictional catalyst experiments?", + "method": "[SYNTHETIC] Deterministic ranking over generated candidate rows.", + "data": "[SYNTHETIC] Twelve generated catalyst candidates with no real-world meaning.", + "results": "[SYNTHETIC] The workflow records a reviewable result; it does not establish scientific validity.", + "figures": ["fig-synthetic-loop"] + } + }, + "sources": { + "annotations": ["synthetic-note#calibration"], + "parsed_file": "fixtures/parsed-paper.v1.json", + "zotero_uri": "zotero://select/library/items/DEMO0002" + } +} diff --git a/src/scriptorium/assets/review.v1.json b/src/scriptorium/assets/review.v1.json new file mode 100644 index 0000000..a88f7ad --- /dev/null +++ b/src/scriptorium/assets/review.v1.json @@ -0,0 +1,31 @@ +{ + "schema_version": "review/1.0", + "direction": { + "query": "[SYNTHETIC] Physics-informed active learning for catalyst discovery", + "created": "2026-07-15T10:00:00Z" + }, + "sections": [ + { + "heading": "[SYNTHETIC] Candidate direction", + "prose": "[SYNTHETIC] The generated comparison suggests testing whether physics-informed priors and calibrated acquisition can be evaluated together. This is a workflow fixture, not a scientific conclusion." + } + ], + "comparison_table": { + "columns": ["Synthetic role", "Review state"], + "rows": [ + { + "citekey": "chen2022PhysicsInformed", + "cells": ["Generated prior", "Read"] + }, + { + "citekey": "liu2024ActiveLearning", + "cells": ["Generated acquisition loop", "In Progress"] + } + ] + }, + "gaps": [ + "[SYNTHETIC] No deterministic experiment yet links calibration quality to candidate-selection efficiency." + ], + "priority_reads": ["liu2024ActiveLearning"], + "lineage_ref": "fixtures/lineage-graph.v1.json" +} diff --git a/src/scriptorium/assets/skills/scriptorium-research/SKILL.md b/src/scriptorium/assets/skills/scriptorium-research/SKILL.md index 572f2ff..ff2133d 100644 --- a/src/scriptorium/assets/skills/scriptorium-research/SKILL.md +++ b/src/scriptorium/assets/skills/scriptorium-research/SKILL.md @@ -14,11 +14,15 @@ CLI, read-only MCP tools, and owned files as its stable interface. 1. Orient to the workspace. - Identify the workspace, project, research question, current stage, and requested deliverable. Run `scriptorium doctor --json --workspace <path>` when available. + - At the start of a continuing project, prefer + `scriptorium resume --project <project-id> --json` or Provenance MCP + `get_context_capsule`. Use the bounded capsule as navigation; retrieve older + sessions or source text only when the current task needs them. - Read the relevant `Projects/*.md` note and existing contract artifacts before proposing changes. - - Query Provenance through `get_current_context`, `get_portfolio`, or - `search_brain` when available. Treat memory as retrieval context, never as a - replacement for source files or primary evidence. + - Fall back to `get_current_context`, `get_portfolio`, or `search_brain` when the + capsule entry is unavailable. Treat memory and `reference_only` capsule items as + retrieval context, never as approved claims or replacements for primary evidence. 2. Frame the next decision. - Restate the question, evidence already available, assumptions, constraints, and a concrete success check. diff --git a/src/scriptorium/cli.py b/src/scriptorium/cli.py index 92b0c7a..b79f6ca 100644 --- a/src/scriptorium/cli.py +++ b/src/scriptorium/cli.py @@ -20,6 +20,18 @@ ) from .inventory import format_inventory_report, run_inventory from .init import InitError, format_init_report, run_init +from .migration import ( + MIGRATION_LIMITATIONS, + REPORT_VERSION as MIGRATION_REPORT_VERSION, + MigrationError, + apply_migration, + format_migration_report, + load_migration, + plan_migration, + reapply_migration, + rollback_migration, + verify_migration, +) from .path_selection import ( attach_path_selection, codex_home_selection, @@ -28,6 +40,7 @@ selection_warnings, ) from .pull import PullError, format_pull_report, run_pull +from .resume import ResumeError, format_resume_report, run_resume from .status import StatusError, format_status_report, run_status @@ -173,6 +186,29 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: help="configuration family root used when workspace/data paths are omitted", ) + resume = commands.add_parser( + "resume", + help="read a bounded, reviewable project context capsule", + ) + resume.add_argument( + "--provenance-home", + type=Path, + help="existing Provenance data root (or use suite config)", + ) + resume.add_argument("--provenance-root", type=Path, help="source checkout of Provenance") + resume.add_argument( + "--project", + help="registered project id (or use the suite default project)", + ) + resume.add_argument( + "--json", action="store_true", dest="json_output", help="write JSON to stdout" + ) + resume.add_argument( + "--config-dir", + type=Path, + help="configuration family root used when data root/project are omitted", + ) + status = commands.add_parser( "status", help="show content-free readiness and pending research workflow counts", @@ -233,6 +269,71 @@ def build_parser(*, json_errors: bool = False) -> argparse.ArgumentParser: "--json", action="store_true", dest="json_output", help="write JSON to stdout" ) + migrate = commands.add_parser( + "migrate", + help="plan, apply, verify, or roll back an explicit Markdown/PDF migration", + ) + migrate_commands = migrate.add_subparsers( + dest="migrate_command", required=True + ) + + def add_migration_identity(command: argparse.ArgumentParser) -> None: + command.add_argument( + "--workspace", + type=Path, + required=True, + help="existing research workspace", + ) + command.add_argument( + "--batch-id", + required=True, + help="stable local migration batch identifier", + ) + command.add_argument( + "--json", + action="store_true", + dest="json_output", + help="write aggregate JSON to stdout", + ) + + migrate_plan = migrate_commands.add_parser( + "plan", + help="preview an explicit migration without writing", + ) + add_migration_identity(migrate_plan) + migrate_plan.add_argument( + "--source", + action="append", + type=Path, + required=True, + help="selected Markdown/PDF file or directory; repeat as needed", + ) + + migrate_apply = migrate_commands.add_parser( + "apply", + help="apply selected sources or resume an existing batch", + ) + add_migration_identity(migrate_apply) + migrate_apply.add_argument( + "--source", + action="append", + type=Path, + default=[], + help="required for a new batch; omit to resume an existing batch", + ) + + migrate_verify = migrate_commands.add_parser( + "verify", + help="verify an existing batch by workspace and batch identifier", + ) + add_migration_identity(migrate_verify) + + migrate_rollback = migrate_commands.add_parser( + "rollback", + help="remove unchanged files owned by an existing batch", + ) + add_migration_identity(migrate_rollback) + host = commands.add_parser( "host", help="manage explicit, project-scoped agent host adapters", @@ -376,6 +477,30 @@ def _pull_error_report(*, run: bool) -> dict[str, object]: } +def _resume_error_report() -> dict[str, object]: + return { + "format_version": 1, + "generated_by": {"name": "scriptorium", "version": __version__}, + "operation": "resume", + "status": "error", + "exit_code": 2, + "capsule": None, + "egress": { + "suite_managed": "not-requested", + "host_managed": "not-invoked", + "optional_connectors": "not-invoked", + }, + "entry": { + "public_command": "prov-context", + "component_exit_code": None, + "stdout": "suppressed", + "stderr": "suppressed", + }, + "errors": [{"code": "entry_error"}], + "limitations": ["No trusted context capsule was available."], + } + + def _status_error_report() -> dict[str, object]: return { "format_version": 1, @@ -445,15 +570,54 @@ def _inventory_error_report() -> dict[str, object]: } +def _migration_error_report( + *, operation: str, code: str = "entry_error" +) -> dict[str, object]: + if operation not in {"plan", "apply", "verify", "rollback"}: + operation = "migration" + return { + "schema_version": MIGRATION_REPORT_VERSION, + "operation": operation, + "status": "error", + "summary": { + "sources_requested": 0, + "files": 0, + "markdown": 0, + "pdf": 0, + "bytes": 0, + "changed": 0, + "unchanged": 0, + }, + "errors": [{"code": code}], + "limitations": list(MIGRATION_LIMITATIONS), + } + + def main(argv: list[str] | None = None) -> int: _configure_output() raw_argv = list(sys.argv[1:] if argv is None else argv) json_command = bool( raw_argv - and raw_argv[0] in {"init", "pull", "status", "inventory"} + and raw_argv[0] in { + "init", + "pull", + "resume", + "status", + "inventory", + "migrate", + } and "--json" in raw_argv ) - private_usage_command = bool(raw_argv and raw_argv[0] == "inventory") + private_usage_command = bool( + raw_argv and raw_argv[0] in {"inventory", "resume", "migrate"} + ) + migration_operation = ( + raw_argv[1] + if len(raw_argv) > 1 + and raw_argv[0] == "migrate" + and raw_argv[1] in {"plan", "apply", "verify", "rollback"} + else "migration" + ) try: args = build_parser( json_errors=json_command or private_usage_command @@ -461,7 +625,8 @@ def main(argv: list[str] | None = None) -> int: except _JsonUsageError: if private_usage_command and not json_command: print( - "ERROR: invalid inventory invocation; review scriptorium inventory --help.", + f"ERROR: invalid {raw_argv[0]} invocation; " + f"review scriptorium {raw_argv[0]} --help.", file=sys.stderr, ) return 2 @@ -469,8 +634,14 @@ def main(argv: list[str] | None = None) -> int: error_report = _init_error_report(run="--run" in raw_argv) elif raw_argv and raw_argv[0] == "pull": error_report = _pull_error_report(run="--run" in raw_argv) + elif raw_argv and raw_argv[0] == "resume": + error_report = _resume_error_report() elif raw_argv and raw_argv[0] == "inventory": error_report = _inventory_error_report() + elif raw_argv and raw_argv[0] == "migrate": + error_report = _migration_error_report( + operation=migration_operation + ) else: error_report = _status_error_report() print( @@ -666,6 +837,59 @@ def main(argv: list[str] | None = None) -> int: else: print(format_pull_report(pull_report)) return int(pull_report["exit_code"]) + if args.command == "resume": + selections = { + "provenance_root": root_selection( + args.provenance_root, "SCRIPTORIUM_PROVENANCE_ROOT" + ), + } + warnings: list[dict[str, object]] = [] + try: + needs_config = ( + args.config_dir is not None + or args.provenance_home is None + or args.project is None + ) + suite_config = _load_suite_config(args.config_dir, needed=needs_config) + provenance_home, data_root_selection = select_configured_path( + args.provenance_home, + ("PROVENANCE_HOME",), + suite_config.provenance_home if suite_config else None, + ) + selections = { + "data_root": data_root_selection, + **selections, + } + warnings = selection_warnings(selections) + project = args.project or ( + suite_config.default_project if suite_config is not None else None + ) + if provenance_home is None or project is None: + raise ResumeError( + "Provenance home and project are required via flags, environment, or suite config" + ) + resume_report = run_resume( + provenance_home=provenance_home, + provenance_root=args.provenance_root, + project=project, + ) + attach_path_selection(resume_report, selections, warnings) + except (ConfigError, ResumeError): + if args.json_output: + error_report = _resume_error_report() + attach_path_selection(error_report, selections, warnings) + print(json.dumps(error_report, ensure_ascii=False, indent=2)) + else: + print( + "ERROR: context capsule unavailable; run scriptorium doctor for local diagnostics.", + file=sys.stderr, + ) + return 2 + if args.json_output: + print(json.dumps(resume_report, ensure_ascii=False, indent=2)) + else: + print(format_resume_report(resume_report)) + return int(resume_report["exit_code"]) if args.command == "status": selections = { "spec_root": root_selection(args.spec_root, "SCRIPTORIUM_SPEC_ROOT"), @@ -779,6 +1003,88 @@ def main(argv: list[str] | None = None) -> int: return 2 print(inventory_output) return inventory_exit_code + if args.command == "migrate": + try: + if args.migrate_command == "plan": + migration_result = plan_migration( + args.source, + workspace=args.workspace, + batch_id=args.batch_id, + ) + elif args.migrate_command == "apply": + migration_result = ( + apply_migration( + plan_migration( + args.source, + workspace=args.workspace, + batch_id=args.batch_id, + ) + ) + if args.source + else reapply_migration( + workspace=args.workspace, + batch_id=args.batch_id, + ) + ) + elif args.migrate_command == "verify": + migration_result = verify_migration( + workspace=args.workspace, + batch_id=args.batch_id, + ) + else: + migration_result = rollback_migration( + load_migration( + workspace=args.workspace, + batch_id=args.batch_id, + ) + ) + except MigrationError as exc: + if args.json_output: + print( + json.dumps( + _migration_error_report( + operation=args.migrate_command, + code=exc.code, + ), + ensure_ascii=False, + indent=2, + ) + ) + else: + print( + f"ERROR: migrate {args.migrate_command} failed ({exc.code}).", + file=sys.stderr, + ) + return 2 + # Migration reports are a privacy boundary; unexpected details stay local. + except Exception: + if args.json_output: + print( + json.dumps( + _migration_error_report( + operation=args.migrate_command, + ), + ensure_ascii=False, + indent=2, + ) + ) + else: + print( + f"ERROR: migrate {args.migrate_command} failed (entry_error).", + file=sys.stderr, + ) + return 2 + if args.json_output: + print( + json.dumps( + migration_result.report, + ensure_ascii=False, + indent=2, + ) + ) + else: + print(format_migration_report(migration_result.report)) + return 0 if args.command == "host" and args.host_command == "install": try: needs_config = ( diff --git a/src/scriptorium/demo.py b/src/scriptorium/demo.py index 73fdffd..c95080b 100644 --- a/src/scriptorium/demo.py +++ b/src/scriptorium/demo.py @@ -356,12 +356,18 @@ def run_demo( "artifacts": [], "expected_artifacts": [ "fixtures/library-kb.v1.1.json", + "fixtures/parsed-paper.v1.json", + "fixtures/reading-note.v1.json", + "fixtures/review.v1.json", + "fixtures/lineage-graph.v1.json", "workspace/Projects/synthetic-catalyst-discovery.md", "workspace/Reviews/ai4science-materials.md", "workspace/Reports/provenance-search.txt", + "workspace/Reports/context-capsule.json", "workspace/Reports/provenance-mcp.jsonl", "provenance/memory/library.json", "provenance/memory/projects.json", + "provenance/memory/research-artifacts.json", "provenance/search-index.db", "demo-report.json", ], @@ -381,13 +387,21 @@ def run_demo( draft_path = fixtures / "review-draft.v1.json" review_path = workspace / "Reviews" / "ai4science-materials.md" kb_path = fixtures / "library-kb.v1.1.json" + research_paths = [ + fixtures / "parsed-paper.v1.json", + fixtures / "reading-note.v1.json", + fixtures / "review.v1.json", + fixtures / "lineage-graph.v1.json", + ] project_path = workspace / "Projects" / "synthetic-catalyst-discovery.md" search_path = workspace / "Reports" / "provenance-search.txt" + context_path = workspace / "Reports" / "context-capsule.json" mcp_path = workspace / "Reports" / "provenance-mcp.jsonl" workspace_readme = workspace / "README.md" prompt_path = input_path.parent / "REVIEW-PROMPT.md" library_memory_path = provenance_home / "memory" / "library.json" project_memory_path = provenance_home / "memory" / "projects.json" + research_memory_path = provenance_home / "memory" / "research-artifacts.json" search_index_path = provenance_home / "search-index.db" config_dir = root / "config" @@ -397,15 +411,18 @@ def run_demo( report_path, kb_path, draft_path, + *research_paths, project_path, workspace_readme, input_path, prompt_path, review_path, search_path, + context_path, mcp_path, library_memory_path, project_memory_path, + research_memory_path, search_index_path, ], ) @@ -446,10 +463,19 @@ def run_demo( steward = find_script(roots["steward"], "steward") prov_ingest_library = find_script(roots["provenance"], "prov-ingest-library") + prov_ingest_research = find_script(roots["provenance"], "prov-ingest-research") prov_ingest_vault = find_script(roots["provenance"], "prov-ingest-vault") prov_search = find_script(roots["provenance"], "prov-search") prov_mcp = find_script(roots["provenance"], "prov-mcp") - provenance_scripts = [prov_ingest_library, prov_ingest_vault, prov_search, prov_mcp] + prov_context = find_script(roots["provenance"], "prov-context") + provenance_scripts = [ + prov_ingest_library, + prov_ingest_research, + prov_ingest_vault, + prov_search, + prov_mcp, + prov_context, + ] _assert( len({path.parent.resolve() for path in provenance_scripts}) == 1, "Provenance commands resolve to different script environments; install one compatible source checkout", @@ -458,6 +484,8 @@ def run_demo( _assert(validator.is_file(), f"spec validator not found: {validator}") _write_asset("library-kb.v1.1.json", kb_path) + for path in research_paths: + _write_asset(path.name, path) _write_asset("review-draft.v1.json", draft_path) _write_asset("synthetic-project.md", project_path) _write_asset("workspace-readme.md", workspace_readme) @@ -486,6 +514,15 @@ def run_demo( ) report["assertions"]["contract_validated"] = True + _run_stage( + "validate literature research contracts", + [sys.executable, str(validator), *[str(path) for path in research_paths]], + cwd=root, + env=env, + stages=stages, + ) + report["assertions"]["research_contracts_validated"] = True + version_result = _run_stage( "check Steward version", [str(steward), "--version"], @@ -589,6 +626,67 @@ def run_demo( ) report["assertions"]["memory_ingested"] = True + first_ingest = _run_stage( + "Provenance research artifact ingest", + [str(prov_ingest_research), *[str(path) for path in research_paths]], + cwd=root, + env=env, + stages=stages, + ) + _assert( + "added=4 updated=0 total=4" in first_ingest.stdout, + "Provenance did not ingest exactly four new synthetic research artifacts", + ) + research_memory = _load_json(research_memory_path) + research_artifacts = research_memory.get("artifacts") or [] + _assert( + research_memory.get("schema_version") == "research-artifacts/1.0" + and len(research_artifacts) == 4 + and {item.get("artifact_type") for item in research_artifacts} + == {"parsed-paper", "reading-note", "review", "lineage-graph"}, + "Provenance research memory does not contain the four expected artifact types", + ) + first_memory_bytes = research_memory_path.read_bytes() + second_ingest = _run_stage( + "Provenance research artifact idempotency", + [str(prov_ingest_research), *[str(path) for path in reversed(research_paths)]], + cwd=root, + env=env, + stages=stages, + ) + _assert( + "added=0 updated=0 total=4" in second_ingest.stdout + and research_memory_path.read_bytes() == first_memory_bytes, + "repeating research artifact ingestion was not byte-stable and idempotent", + ) + report["assertions"]["research_artifacts_ingested"] = True + report["assertions"]["research_ingest_idempotent"] = True + + context_result = _run_stage( + "Provenance context capsule", + [str(prov_context), "--project", "synthetic-catalyst-discovery", "--json"], + cwd=root, + env=env, + stages=stages, + ) + try: + context_capsule = json.loads(context_result.stdout) + except json.JSONDecodeError as exc: + raise DemoError(f"Provenance context capsule is not valid JSON: {exc}") from exc + capsule_artifacts = context_capsule.get("research_artifacts") or [] + _assert( + context_capsule.get("project", {}).get("project_id") + == "synthetic-catalyst-discovery" + and {item.get("kind") for item in capsule_artifacts} + == {"parsed-paper", "reading-note", "review", "lineage-graph"} + and all(item.get("trust") == "reference_only" for item in capsule_artifacts) + and context_capsule.get("trust", {}).get("research_artifacts") + == "reference_only_not_approved_claims", + "context capsule did not expose all research artifacts through the reference-only boundary", + ) + _write_managed_bytes(root, context_path, context_result.stdout.encode("utf-8")) + report["assertions"]["context_capsule_reference_only"] = True + _run_stage( "Provenance search build", [str(prov_search), "--build"], @@ -626,6 +724,15 @@ def run_demo( "method": "tools/call", "params": {"name": "search_brain", "arguments": {"query": "calibration", "limit": 5}}, }, + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": "get_context_capsule", + "arguments": {"project_id": "synthetic-catalyst-discovery"}, + }, + }, ] request_text = "\n".join(json.dumps(item, separators=(",", ":")) for item in requests) + "\n" mcp_result = _run_stage( @@ -645,7 +752,7 @@ def run_demo( continue if isinstance(response, dict) and isinstance(response.get("id"), int): responses[response["id"]] = response - _assert(set(responses) == {0, 1, 2, 3}, "Provenance MCP did not return all four responses") + _assert(set(responses) == {0, 1, 2, 3, 4}, "Provenance MCP did not return all five responses") initialize = responses[0] _assert( initialize.get("jsonrpc") == "2.0" @@ -664,6 +771,7 @@ def run_demo( portfolio_text = _mcp_text(responses[1], 1) context_text = _mcp_text(responses[2], 2) search_text = _mcp_text(responses[3], 3) + capsule_text = _mcp_text(responses[4], 4) _assert( "Synthetic Catalyst Discovery" in portfolio_text and "Physics-informed learning for materials discovery" in context_text @@ -671,6 +779,15 @@ def run_demo( and "id=lit:DEMO0002" in search_text, "Provenance MCP responses did not surface the expected project and literature context", ) + _assert( + "Context Capsule: Synthetic Catalyst Discovery" in capsule_text + and "Research artifact hints (reference only)" in capsule_text + and "[parsed-paper]" in capsule_text + and "[reading-note]" in capsule_text + and "[review]" in capsule_text + and "[lineage-graph]" in capsule_text, + "Provenance MCP context capsule did not preserve the reference-only research hints", + ) mcp_path.write_bytes(mcp_result.stdout.encode("utf-8")) report["assertions"]["mcp_context_verified"] = True diff --git a/src/scriptorium/doctor.py b/src/scriptorium/doctor.py index c51bfd8..e2efe27 100644 --- a/src/scriptorium/doctor.py +++ b/src/scriptorium/doctor.py @@ -30,6 +30,7 @@ PROBE_TIMEOUT_SECONDS = 5 PROVENANCE_BASE_COMMANDS = ( "prov-ingest-library", + "prov-ingest-research", "prov-ingest-vault", "prov-search", "prov-mcp", @@ -421,7 +422,7 @@ def _provenance_check( required_for=TARGETS, passed=passed, summary=( - f"Provenance source/runtime {expected_version} and four public commands matched" + f"Provenance source/runtime {expected_version} and five public commands matched" if passed else "; ".join(errors) ), @@ -434,7 +435,7 @@ def _provenance_check( }, remediation=( "Install the compatible Provenance checkout in one environment " - "and expose all four public commands." + "and expose all five public commands." ), ), root, @@ -876,6 +877,77 @@ def _entry_pull_check( ) +def _entry_context_check( + target: str, + provenance_root: Path | None, + expected_version: str | None = None, +) -> dict[str, Any]: + command = None + command_parent = None + baseline_commands: dict[str, Path] = {} + baseline_parent = None + runtime_version = None + error = None + try: + if provenance_root is None: + raise DoctorError("compatible Provenance source checkout is unavailable") + command = find_script(provenance_root, "prov-context") + baseline_commands = { + name: find_script(provenance_root, name) + for name in PROVENANCE_BASE_COMMANDS + } + baseline_parents = {path.parent.resolve() for path in baseline_commands.values()} + if len(baseline_parents) != 1: + raise DoctorError( + "baseline Provenance commands resolve from different environments" + ) + baseline_parent = next(iter(baseline_parents)) + command_parent = command.parent.resolve() + if command_parent != baseline_parent: + raise DoctorError( + "public context command resolves from a different Provenance environment" + ) + runtime_version = _run_probe( + [str(command), "--version"], + cwd=provenance_root, + extra_env={"PROVENANCE_HOME": os.devnull}, + ).strip() + if expected_version is not None and runtime_version != expected_version: + raise DoctorError( + f"context runtime version {runtime_version} != {expected_version}" + ) + except (DemoError, DoctorError, OSError, RuntimeError) as exc: + error = str(exc) + passed = error is None + return _check( + "entry.resume", + target=target, + required_for=("public-alpha",), + passed=passed, + summary=( + "Read-only bounded context-capsule entry is available" + if passed + else "The context-capsule resume entry is unavailable or incompatible" + ), + details={ + "command": str(command) if command else None, + "command_parent": str(command_parent) if command_parent else None, + "baseline_commands": { + name: str(path) for name, path in baseline_commands.items() + }, + "baseline_command_parent": str(baseline_parent) if baseline_parent else None, + "implemented": passed, + "expected_version": expected_version, + "runtime_version": runtime_version, + "error": error, + }, + remediation=( + "Install the compatible Provenance build exposing `prov-context`, then " + "reinstall the Scriptorium suite entry in the same environment." + ), + ) + + def _registry_application(executable: str) -> Path | None: if os.name != "nt": return None @@ -1295,6 +1367,9 @@ def run_doctor( host_adapter, _agent_capture_check(target, host_adapter), _entry_pull_check(target, resolved_provenance, expected["provenance"]), + _entry_context_check( + target, resolved_provenance, expected["provenance"] + ), ] ) lectern, _ = _lectern_check(target, lectern_root) diff --git a/src/scriptorium/migration.py b/src/scriptorium/migration.py new file mode 100644 index 0000000..25c91f3 --- /dev/null +++ b/src/scriptorium/migration.py @@ -0,0 +1,1595 @@ +"""Explicit, local-only Markdown/PDF migration. + +Private manifests contain absolute paths and live outside the research +workspace. Public reports contain aggregate counts only. + +Threat boundary: this module coordinates cooperative processes for one local +user. It does not defend against a malicious local process changing files +during an operation. +""" + +from __future__ import annotations + +import copy +import ctypes +import errno +import hashlib +import json +import os +import re +import secrets +import stat +import sys +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping + + +MANIFEST_VERSION = "migration-manifest/1.0" +REPORT_VERSION = "migration-report/1.0" +MANIFEST_PRIVACY = "local-private" +MIGRATION_LIMITATIONS = ( + "Only explicitly selected Markdown and PDF files or directories are supported.", + "AI conversations, Zotero libraries, parsing, indexing, and network sources are not supported.", + "The private path manifest stays in the local state root outside the research workspace.", + "Coordination assumes one local user and cooperative Scriptorium processes.", + "Atomic publication requires hard-link support on the local destination filesystem.", + "Sources may be on another local volume because bytes are staged beside each destination.", + ( + "A random internal .scriptorium-*.stage hard-link ownership anchor " + "remains beside each migrated target until rollback." + ), + ( + "A crash before an anchor is recorded may leave an unclaimed random stage; " + "it is never adopted or deleted automatically." + ), + ( + "Automatic rollback requires Windows no-replace rename or Linux renameat2; " + "unsupported platforms fail closed." + ), + ( + "Rollback quarantines and re-verifies owned files; foreign replacements are " + "restored or preserved, and empty directories may remain." + ), +) + +_KINDS = {".md": "markdown", ".markdown": "markdown", ".pdf": "pdf"} +_BATCH_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z") +_ID_RE = re.compile(r"file-[0-9]{6}\Z") +_HASH_RE = re.compile(r"[0-9a-f]{64}\Z") +_TOKEN_RE = re.compile(r"[0-9a-f]{32}\Z") +_REPARSE_POINT = 0x400 +_DRIVE_REMOTE = 4 +_ROLLBACK_PHASES = { + "target", + "target-quarantined", + "anchor", + "anchor-quarantined", +} +_AT_FDCWD = -100 +_RENAME_NOREPLACE = 1 +_RUN_STATES = { + "planned", + "applying", + "applied", + "rolling-back", + "rolled-back", +} +_ENTRY_STATES = { + "planned", + "creating", + "created", + "delete-pending", + "deleted", +} +_RUN_ENTRY_STATES = { + "planned": {"planned"}, + "applying": {"planned", "creating", "created"}, + "applied": {"created"}, + "rolling-back": _ENTRY_STATES, + "rolled-back": {"deleted"}, +} +MIGRATION_ERROR_CODES = frozenset( + { + "applied_manifest_missing", + "atomic_publish_unavailable", + "atomic_quarantine_unavailable", + "batch_conflict", + "cross_volume_publish_unsupported", + "entry_error", + "file_identity_unavailable", + "invalid_batch_id", + "invalid_manifest", + "invalid_path", + "invalid_sources", + "invalid_state_transition", + "link_or_reparse_rejected", + "lock_invalid", + "lock_unavailable", + "manifest_integrity_failed", + "migration_already_rolled_back", + "migration_not_applied", + "migration_not_found", + "no_sources_selected", + "no_supported_files", + "noncanonical_state_root", + "owned_target_changed", + "path_unreadable", + "remote_path_rejected", + "rollback_failed", + "rollback_in_progress", + "rollback_restore_blocked", + "source_destination_overlap", + "source_hash_changed", + "source_missing", + "source_not_regular", + "source_state_overlap", + "source_unreadable", + "staging_cleanup_failed", + "staging_conflict", + "quarantine_conflict", + "state_root_in_source", + "state_root_invalid", + "state_workspace_overlap", + "state_write_failed", + "stored_manifest_invalid", + "stored_manifest_missing", + "target_escape", + "target_exists", + "target_unwritable", + "unsupported_source_type", + "workspace_identity_changed", + "workspace_missing", + } +) + + +class MigrationError(RuntimeError): + """A path-free migration failure.""" + + def __init__(self, code: str) -> None: + self.code = code if code in MIGRATION_ERROR_CODES else "entry_error" + super().__init__(f"migration failed: {self.code}") + + +@dataclass(frozen=True, repr=False) +class MigrationPlan: + manifest: dict[str, Any] = field(repr=False) + report: dict[str, Any] = field(repr=False) + + def __repr__(self) -> str: + return _safe_repr(type(self).__name__, self.report) + + +@dataclass(frozen=True, repr=False) +class MigrationResult: + manifest: dict[str, Any] = field(repr=False) + report: dict[str, Any] = field(repr=False) + + def __repr__(self) -> str: + return _safe_repr(type(self).__name__, self.report) + + +def _safe_repr(name: str, report: Mapping[str, Any]) -> str: + summary = report.get("summary", {}) + files = summary.get("files", 0) if isinstance(summary, Mapping) else 0 + return f"{name}(status={report.get('status')!r}, files={files!r})" + + +def _absolute(value: os.PathLike[str] | str) -> Path: + try: + return Path(os.path.abspath(os.fspath(Path(value).expanduser()))) + except (OSError, TypeError, ValueError) as exc: + raise MigrationError("invalid_path") from exc + + +def _inside(root: Path, path: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return path != root + + +def _linklike(metadata: os.stat_result) -> bool: + attributes = int(getattr(metadata, "st_file_attributes", 0)) + return stat.S_ISLNK(metadata.st_mode) or bool(attributes & _REPARSE_POINT) + + +def _metadata(path: Path) -> os.stat_result | None: + try: + return path.lstat() + except FileNotFoundError: + return None + except OSError as exc: + raise MigrationError("path_unreadable") from exc + + +def _check_components(path: Path) -> None: + current = path + components: list[Path] = [] + while True: + components.append(current) + if current.parent == current: + break + current = current.parent + for component in reversed(components): + metadata = _metadata(component) + if metadata is not None and _linklike(metadata): + raise MigrationError("link_or_reparse_rejected") + + +def _reject_remote(path: Path) -> None: + if os.fspath(path).startswith(("\\\\", "//")): + raise MigrationError("remote_path_rejected") + if os.name != "nt" or not path.anchor: + return + get_drive_type = ctypes.windll.kernel32.GetDriveTypeW + get_drive_type.argtypes = [ctypes.c_wchar_p] + get_drive_type.restype = ctypes.c_uint + if int(get_drive_type(path.anchor)) == _DRIVE_REMOTE: + raise MigrationError("remote_path_rejected") + + +def _existing_directory(value: os.PathLike[str] | str, code: str) -> Path: + path = _absolute(value) + _reject_remote(path) + _check_components(path) + metadata = _metadata(path) + if metadata is None or not stat.S_ISDIR(metadata.st_mode): + raise MigrationError(code) + try: + return path.resolve(strict=True) + except OSError as exc: + raise MigrationError(code) from exc + + +def _ensure_tree(root: Path, directory: Path) -> None: + if directory != root and not _inside(root, directory): + raise MigrationError("target_escape") + _check_components(directory) + try: + root.mkdir(parents=True, exist_ok=True) + directory.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise MigrationError("target_unwritable") from exc + _check_components(directory) + for candidate in (root, directory): + metadata = _metadata(candidate) + if metadata is None or not stat.S_ISDIR(metadata.st_mode): + raise MigrationError("target_unwritable") + + +def _default_state_root() -> Path: + if os.name == "nt" and os.environ.get("LOCALAPPDATA"): + return _absolute(Path(os.environ["LOCALAPPDATA"]) / "Scriptorium" / "state") + if os.environ.get("XDG_STATE_HOME"): + return _absolute(Path(os.environ["XDG_STATE_HOME"]) / "scriptorium") + return _absolute(Path.home() / ".local" / "state" / "scriptorium") + + +def _same_path(left: Path, right: Path) -> bool: + try: + left_value = os.fspath(left.resolve(strict=False)) + right_value = os.fspath(right.resolve(strict=False)) + except OSError as exc: + raise MigrationError("state_root_invalid") from exc + return os.path.normcase(left_value) == os.path.normcase(right_value) + + +def _private_state_root( + value: os.PathLike[str] | str | None, workspace: Path +) -> Path: + canonical = _default_state_root() + requested = canonical if value is None else _absolute(value) + if not _same_path(requested, canonical): + raise MigrationError("noncanonical_state_root") + root = canonical.resolve(strict=False) + _reject_remote(root) + _check_components(root) + metadata = _metadata(root) + if metadata is not None and not stat.S_ISDIR(metadata.st_mode): + raise MigrationError("state_root_invalid") + if root == workspace or _inside(workspace, root) or _inside(root, workspace): + raise MigrationError("state_workspace_overlap") + return root + + +def _kind(path: Path) -> str | None: + return _KINDS.get(path.suffix.lower()) + + +def _reject_state_source_overlap(source: Path, private_root: Path) -> None: + if source == private_root or _inside(source, private_root): + raise MigrationError("state_root_in_source") + if _inside(private_root, source): + raise MigrationError("source_state_overlap") + + +def _hash_file(path: Path, code: str) -> tuple[str, int]: + metadata = _metadata(path) + if metadata is None: + raise MigrationError(code) + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + raise MigrationError(code) + digest = hashlib.sha256() + size = 0 + try: + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + except OSError as exc: + raise MigrationError(code) from exc + return digest.hexdigest(), size + + +def _selected( + sources: Iterable[os.PathLike[str] | str] | os.PathLike[str] | str, +) -> list[Path]: + if isinstance(sources, (str, os.PathLike)): + values = [sources] + else: + try: + values = list(sources) + except TypeError as exc: + raise MigrationError("invalid_sources") from exc + if not values: + raise MigrationError("no_sources_selected") + if any( + isinstance(value, bytes) or not isinstance(value, (str, os.PathLike)) + for value in values + ): + raise MigrationError("invalid_sources") + return [_absolute(value) for value in values] + + +def _scan(root: Path) -> list[Path]: + found: list[Path] = [] + + def fail(error: OSError) -> None: + raise error + + try: + walker = os.walk(root, topdown=True, onerror=fail, followlinks=False) + for directory, names, files in walker: + names.sort(key=str.casefold) + files.sort(key=str.casefold) + for name in names: + metadata = _metadata(Path(directory) / name) + if metadata is not None and _linklike(metadata): + raise MigrationError("link_or_reparse_rejected") + for name in files: + path = Path(directory) / name + metadata = _metadata(path) + if metadata is not None and _linklike(metadata): + raise MigrationError("link_or_reparse_rejected") + if metadata is not None and stat.S_ISREG(metadata.st_mode) and _kind(path): + found.append(path) + except OSError as exc: + raise MigrationError("source_unreadable") from exc + return sorted(found, key=lambda path: os.path.normcase(os.fspath(path))) + + +def _label(index: int, path: Path, *, file: bool) -> str: + suffix = path.suffix.lower() if file else "" + raw = path.stem if file else path.name + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip(".-") or "source" + return f"{index:03d}-{slug[:64]}{suffix}" + + +def _relative(value: str | Path) -> str: + path = Path(value) + if path.is_absolute() or not path.parts: + raise MigrationError("target_escape") + if any(part in {"", ".", ".."} or ":" in part or "\x00" in part for part in path.parts): + raise MigrationError("target_escape") + return path.as_posix() + + +def _new_internal_name( + manifest: Mapping[str, Any], entry: Mapping[str, Any], suffix: str +) -> str: + return ( + f".scriptorium-{manifest['batch_id']}-{entry['id']}-" + f"{secrets.token_hex(16)}.{suffix}" + ) + + +def _valid_internal_name( + manifest: Mapping[str, Any], + entry: Mapping[str, Any], + value: Any, + suffix: str, +) -> bool: + if not isinstance(value, str) or Path(value).name != value: + return False + prefix = f".scriptorium-{manifest['batch_id']}-{entry['id']}-" + token, separator, actual_suffix = value.removeprefix(prefix).rpartition(".") + return ( + value.startswith(prefix) + and separator == "." + and actual_suffix == suffix + and _TOKEN_RE.fullmatch(token) is not None + ) + + +def _stage( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> Path | None: + name = entry.get("stage_name") + if name is None: + return None + return Path(entry["target"]).parent / name + + +def _quarantine( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> Path | None: + name = entry.get("quarantine_name") + if name is None: + return None + return Path(entry["target"]).parent / name + + +def _plan_data(manifest: Mapping[str, Any]) -> dict[str, Any]: + fixed = ( + "schema_version", + "privacy", + "batch_id", + "workspace", + "destination_root", + "state_root", + "source_roots", + "limitations", + ) + entry_fixed = ( + "id", + "source", + "target", + "relative_target", + "kind", + "sha256", + "size", + ) + data = {key: manifest.get(key) for key in fixed} + data["entries"] = [ + {key: entry.get(key) for key in entry_fixed} + for entry in manifest.get("entries", []) + ] + return data + + +def _digest(value: Mapping[str, Any]) -> str: + encoded = json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _plan_digest(manifest: Mapping[str, Any]) -> str: + return _digest(_plan_data(manifest)) + + +def _state_digest(manifest: Mapping[str, Any]) -> str: + data = copy.deepcopy(dict(manifest)) + data.pop("state_sha256", None) + return _digest(data) + + +def _seal_manifest(manifest: dict[str, Any], *, new_plan: bool = False) -> None: + if new_plan: + manifest["plan_sha256"] = _plan_digest(manifest) + manifest["state_sha256"] = _state_digest(manifest) + + +def _report( + operation: str, + status: str, + entries: list[Mapping[str, Any]], + *, + roots: int, + changed: int = 0, + unchanged: int = 0, +) -> dict[str, Any]: + return { + "schema_version": REPORT_VERSION, + "operation": operation, + "status": status, + "summary": { + "sources_requested": roots, + "files": len(entries), + "markdown": sum(entry["kind"] == "markdown" for entry in entries), + "pdf": sum(entry["kind"] == "pdf" for entry in entries), + "bytes": sum(entry["size"] for entry in entries), + "changed": changed, + "unchanged": unchanged, + }, + "limitations": list(MIGRATION_LIMITATIONS), + } + + +def format_migration_report(report: Mapping[str, Any]) -> str: + """Render a fixed, path-free aggregate report.""" + + operation = report.get("operation") + if operation not in {"plan", "apply", "load", "verify", "rollback"}: + operation = "migration" + status = report.get("status") + if status not in { + "planned", + "applying", + "applied", + "unchanged", + "rolling-back", + "rolled-back", + }: + status = "unknown" + summary = report.get("summary") + if not isinstance(summary, Mapping): + summary = {} + + def count(name: str) -> int: + value = summary.get(name) + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + return "\n".join( + ( + f"Migration {operation}: {status}", + ( + f"Files: {count('files')} " + f"(Markdown: {count('markdown')}, PDF: {count('pdf')})" + ), + f"Bytes: {count('bytes')}", + f"Changed: {count('changed')}; unchanged: {count('unchanged')}", + "Private state: canonical local state root; paths suppressed", + ) + ) + + +def plan_migration( + sources: Iterable[os.PathLike[str] | str] | os.PathLike[str] | str, + *, + workspace: os.PathLike[str] | str, + batch_id: str, + state_root: os.PathLike[str] | str | None = None, +) -> MigrationPlan: + """Build a read-only plan from explicit local inputs.""" + + if not isinstance(batch_id, str) or not _BATCH_RE.fullmatch(batch_id): + raise MigrationError("invalid_batch_id") + workspace_path = _existing_directory(workspace, "workspace_missing") + private_root = _private_state_root(state_root, workspace_path) + destination = workspace_path / "Sources" / "Imported" / batch_id + requested = _selected(sources) + entries: list[dict[str, Any]] = [] + seen: set[str] = set() + + for index, source_root in enumerate(requested, start=1): + _reject_remote(source_root) + _check_components(source_root) + metadata = _metadata(source_root) + if metadata is None: + raise MigrationError("source_missing") + if stat.S_ISDIR(metadata.st_mode): + source_root = source_root.resolve(strict=True) + _reject_state_source_overlap(source_root, private_root) + if ( + source_root == destination + or _inside(source_root, destination) + or _inside(destination, source_root) + ): + raise MigrationError("source_destination_overlap") + routes = [ + (source, Path(_label(index, source_root, file=False)) / source.relative_to(source_root)) + for source in _scan(source_root) + ] + elif stat.S_ISREG(metadata.st_mode): + source_root = source_root.resolve(strict=True) + _reject_state_source_overlap(source_root, private_root) + if not _kind(source_root): + raise MigrationError("unsupported_source_type") + if source_root == destination or _inside(destination, source_root): + raise MigrationError("source_destination_overlap") + routes = [(source_root, Path(_label(index, source_root, file=True)))] + else: + raise MigrationError("source_not_regular") + + for source, relative_path in routes: + identity = os.path.normcase(os.path.normpath(os.fspath(source))) + if identity in seen: + continue + seen.add(identity) + relative_target = _relative(relative_path) + target = _absolute(destination / relative_target) + digest, size = _hash_file(source, "source_unreadable") + entry = { + "id": f"file-{len(entries) + 1:06d}", + "source": os.fspath(source), + "target": os.fspath(target), + "relative_target": relative_target, + "kind": _kind(source), + "sha256": digest, + "size": size, + "state": "planned", + "file_identity": None, + "stage_name": None, + "rollback_phase": None, + "quarantine_name": None, + } + _check_components(target.parent) + if _metadata(target) is not None: + raise MigrationError("target_exists") + entries.append(entry) + + if not entries: + raise MigrationError("no_supported_files") + manifest = { + "schema_version": MANIFEST_VERSION, + "privacy": MANIFEST_PRIVACY, + "batch_id": batch_id, + "run_state": "planned", + "workspace": os.fspath(workspace_path), + "destination_root": os.fspath(destination), + "state_root": os.fspath(private_root), + "source_roots": len(requested), + "entries": entries, + "limitations": list(MIGRATION_LIMITATIONS), + } + _seal_manifest(manifest, new_plan=True) + return MigrationPlan( + manifest=manifest, + report=_report("plan", "planned", entries, roots=len(requested)), + ) + + +def _unwrap(value: MigrationPlan | MigrationResult | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(value, (MigrationPlan, MigrationResult)): + value = value.manifest + if not isinstance(value, Mapping): + raise MigrationError("invalid_manifest") + return copy.deepcopy(dict(value)) + + +def _validate_manifest(value: dict[str, Any]) -> dict[str, Any]: + manifest = copy.deepcopy(value) + required = { + "schema_version", "privacy", "batch_id", "run_state", "workspace", + "destination_root", "state_root", "source_roots", "entries", + "limitations", "plan_sha256", "state_sha256", + } + if set(manifest) != required: + raise MigrationError("invalid_manifest") + if ( + manifest["schema_version"] != MANIFEST_VERSION + or manifest["privacy"] != MANIFEST_PRIVACY + or manifest["run_state"] not in _RUN_STATES + or not isinstance(manifest["batch_id"], str) + or not _BATCH_RE.fullmatch(manifest["batch_id"]) + or manifest["limitations"] != list(MIGRATION_LIMITATIONS) + or not isinstance(manifest["source_roots"], int) + or isinstance(manifest["source_roots"], bool) + or manifest["source_roots"] < 1 + or not isinstance(manifest["entries"], list) + or not manifest["entries"] + ): + raise MigrationError("invalid_manifest") + if not all( + isinstance(manifest[key], str) + for key in ("workspace", "destination_root", "state_root") + ): + raise MigrationError("invalid_manifest") + + workspace = _absolute(manifest["workspace"]) + destination = _absolute(manifest["destination_root"]) + state_root = _absolute(manifest["state_root"]) + if destination != workspace / "Sources" / "Imported" / manifest["batch_id"]: + raise MigrationError("target_escape") + if ( + state_root == workspace + or _inside(workspace, state_root) + or _inside(state_root, workspace) + ): + raise MigrationError("state_workspace_overlap") + + entry_keys = { + "id", "source", "target", "relative_target", "kind", "sha256", + "size", "state", "file_identity", "stage_name", "rollback_phase", + "quarantine_name", + } + identifiers: set[str] = set() + targets: set[str] = set() + for entry in manifest["entries"]: + if not isinstance(entry, dict) or set(entry) != entry_keys: + raise MigrationError("invalid_manifest") + if ( + not isinstance(entry["id"], str) + or not _ID_RE.fullmatch(entry["id"]) + or entry["id"] in identifiers + or not isinstance(entry["source"], str) + or not isinstance(entry["target"], str) + or not isinstance(entry["relative_target"], str) + or entry["kind"] not in {"markdown", "pdf"} + or not isinstance(entry["sha256"], str) + or not _HASH_RE.fullmatch(entry["sha256"]) + or not isinstance(entry["size"], int) + or isinstance(entry["size"], bool) + or entry["size"] < 0 + or entry["state"] not in _ENTRY_STATES + ): + raise MigrationError("invalid_manifest") + identity = entry["file_identity"] + if identity is not None and ( + not isinstance(identity, dict) + or set(identity) != {"device", "inode"} + or any( + not isinstance(identity[key], int) + or isinstance(identity[key], bool) + or identity[key] < 0 + for key in ("device", "inode") + ) + or identity["inode"] == 0 + ): + raise MigrationError("invalid_manifest") + stage_name = entry["stage_name"] + phase = entry["rollback_phase"] + quarantine_name = entry["quarantine_name"] + if stage_name is not None and not _valid_internal_name( + manifest, entry, stage_name, "stage" + ): + raise MigrationError("invalid_manifest") + if quarantine_name is not None and not _valid_internal_name( + manifest, entry, quarantine_name, "rollback" + ): + raise MigrationError("invalid_manifest") + if phase is not None and phase not in _ROLLBACK_PHASES: + raise MigrationError("invalid_manifest") + if ( + entry["state"] == "planned" + and any( + value is not None + for value in (identity, stage_name, phase, quarantine_name) + ) + ) or ( + entry["state"] in {"created", "delete-pending"} + and (identity is None or stage_name is None) + ) or ( + entry["state"] != "delete-pending" + and (phase is not None or quarantine_name is not None) + ) or ( + entry["state"] == "delete-pending" + and (phase is None or quarantine_name is None) + ) or ( + (identity is None) != (stage_name is None) + ): + raise MigrationError("invalid_state_transition") + source = Path(entry["source"]) + target = Path(entry["target"]) + relative = _relative(entry["relative_target"]) + expected = _absolute(destination / relative) + key = os.path.normcase(os.path.normpath(os.fspath(target))) + if ( + not source.is_absolute() + or not target.is_absolute() + or target != expected + or not _inside(destination, target) + or _kind(source) != entry["kind"] + or key in targets + ): + raise MigrationError("target_escape") + identifiers.add(entry["id"]) + targets.add(key) + + if ( + not isinstance(manifest["plan_sha256"], str) + or not _HASH_RE.fullmatch(manifest["plan_sha256"]) + or not isinstance(manifest["state_sha256"], str) + or not _HASH_RE.fullmatch(manifest["state_sha256"]) + ): + raise MigrationError("invalid_manifest") + if ( + manifest["plan_sha256"] != _plan_digest(manifest) + or manifest["state_sha256"] != _state_digest(manifest) + ): + raise MigrationError("manifest_integrity_failed") + states = {entry["state"] for entry in manifest["entries"]} + if not states <= _RUN_ENTRY_STATES[manifest["run_state"]]: + raise MigrationError("invalid_state_transition") + return manifest + + +def _workspace_key(workspace: str) -> str: + normalized = os.path.normcase(os.path.normpath(workspace)).encode() + return hashlib.sha256(normalized).hexdigest()[:20] + + +def _manifest_path(manifest: Mapping[str, Any]) -> Path: + return ( + Path(manifest["state_root"]) + / "migrations" + / _workspace_key(manifest["workspace"]) + / f"{manifest['batch_id']}.json" + ) + + +def _lock_path(manifest: Mapping[str, Any]) -> Path: + return _manifest_path(manifest).with_suffix(".lock") + + +def _persist_manifest(manifest: dict[str, Any]) -> None: + _seal_manifest(manifest) + path = _manifest_path(manifest) + _ensure_tree(Path(manifest["state_root"]), path.parent) + payload = ( + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + ).encode() + descriptor, name = tempfile.mkstemp( + prefix=".migration-", suffix=".tmp", dir=path.parent + ) + temporary = Path(name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except OSError as exc: + raise MigrationError("state_write_failed") from exc + finally: + try: + temporary.unlink() + except OSError: + pass + + +def _read_manifest(path: Path) -> dict[str, Any] | None: + metadata = _metadata(path) + if metadata is None: + return None + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + raise MigrationError("stored_manifest_invalid") + try: + value = json.loads(path.read_text(encoding="utf-8")) + stored = _validate_manifest(value) + except (OSError, UnicodeError, json.JSONDecodeError, MigrationError) as exc: + raise MigrationError("stored_manifest_invalid") from exc + return stored + + +def _load_stored(requested: Mapping[str, Any]) -> dict[str, Any] | None: + stored = _read_manifest(_manifest_path(requested)) + if stored is None: + return None + if stored["plan_sha256"] != requested["plan_sha256"]: + raise MigrationError("batch_conflict") + return stored + + +@contextmanager +def _kernel_lock(path: Path): + """Hold an advisory kernel lock released by close or process exit.""" + + _check_components(path.parent) + try: + path.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise MigrationError("lock_unavailable") from exc + _check_components(path.parent) + metadata = _metadata(path) + if metadata is not None and ( + _linklike(metadata) or not stat.S_ISREG(metadata.st_mode) + ): + raise MigrationError("lock_invalid") + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + try: + descriptor = os.open(path, flags, 0o600) + handle = os.fdopen(descriptor, "r+b") + except OSError as exc: + raise MigrationError("lock_unavailable") from exc + locked = False + try: + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write(b"\0") + handle.flush() + os.fsync(handle.fileno()) + handle.seek(0) + if os.name == "nt": + import msvcrt + + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + locked = True + yield + except OSError as exc: + raise MigrationError("lock_unavailable") from exc + finally: + if locked: + handle.seek(0) + if os.name == "nt": + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + handle.close() + + +def _stored_identity( + workspace: os.PathLike[str] | str, + batch_id: str, +) -> tuple[Path, dict[str, str]]: + if not isinstance(batch_id, str) or not _BATCH_RE.fullmatch(batch_id): + raise MigrationError("invalid_batch_id") + workspace_path = _existing_directory(workspace, "workspace_missing") + state_root = _private_state_root(None, workspace_path) + return workspace_path, { + "workspace": os.fspath(workspace_path), + "state_root": os.fspath(state_root), + "batch_id": batch_id, + } + + +def load_migration( + *, + workspace: os.PathLike[str] | str, + batch_id: str, +) -> MigrationResult: + """Load private migration state using only its stable public identity.""" + + workspace_path, identity = _stored_identity(workspace, batch_id) + stored = _read_manifest(_manifest_path(identity)) + if stored is None: + raise MigrationError("migration_not_found") + if ( + Path(stored["workspace"]) != workspace_path + or stored["batch_id"] != batch_id + or not _same_path(Path(stored["state_root"]), Path(identity["state_root"])) + ): + raise MigrationError("stored_manifest_invalid") + return MigrationResult( + stored, + _report( + "load", + stored["run_state"], + stored["entries"], + roots=stored["source_roots"], + unchanged=len(stored["entries"]), + ), + ) + + +def _content_status(path: Path | None, entry: Mapping[str, Any]) -> str: + if path is None: + return "missing" + metadata = _metadata(path) + if metadata is None: + return "missing" + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + return "changed" + digest, size = _hash_file(path, "path_unreadable") + return "exact" if (digest, size) == (entry["sha256"], entry["size"]) else "changed" + + +def _identity_from_metadata(metadata: os.stat_result) -> dict[str, int]: + """Return a stable local identity from already-bound file metadata.""" + + if _linklike(metadata) or not stat.S_ISREG(metadata.st_mode): + raise MigrationError("file_identity_unavailable") + device = getattr(metadata, "st_dev", None) + inode = getattr(metadata, "st_ino", None) + if ( + not isinstance(device, int) + or isinstance(device, bool) + or device < 0 + or not isinstance(inode, int) + or isinstance(inode, bool) + or inode <= 0 + ): + raise MigrationError("file_identity_unavailable") + return {"device": device, "inode": inode} + + +def _file_identity(path: Path | None) -> dict[str, int] | None: + """Return a stable local file identity or fail closed when unavailable.""" + + if path is None: + return None + metadata = _metadata(path) + if metadata is None: + return None + return _identity_from_metadata(metadata) + + +def _identity_matches(path: Path | None, entry: Mapping[str, Any]) -> bool: + expected = entry.get("file_identity") + return isinstance(expected, Mapping) and _file_identity(path) == dict(expected) + + +def _require_owned_anchor( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> None: + anchor = _stage(manifest, entry) + if ( + _content_status(anchor, entry) != "exact" + or not _identity_matches(anchor, entry) + ): + raise MigrationError("owned_target_changed") + + +def _require_owned_target( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> None: + """Require the published path to remain the anchor's unchanged hard link.""" + + target = Path(entry["target"]) + anchor = _stage(manifest, entry) + if ( + _content_status(target, entry) != "exact" + or _content_status(anchor, entry) != "exact" + or not _identity_matches(target, entry) + or not _identity_matches(anchor, entry) + ): + raise MigrationError("owned_target_changed") + try: + same_file = os.path.samefile(anchor, target) + except OSError as exc: + raise MigrationError("owned_target_changed") from exc + if not same_file: + raise MigrationError("owned_target_changed") + + +def _verify_source(entry: Mapping[str, Any]) -> None: + actual = _hash_file(Path(entry["source"]), "source_missing") + if actual != (entry["sha256"], entry["size"]): + raise MigrationError("source_hash_changed") + + +def _copy_stage(manifest: dict[str, Any], entry: dict[str, Any]) -> Path: + """Create or recover one random, manifest-owned same-directory stage.""" + + stage = _stage(manifest, entry) + if stage is not None: + _require_owned_anchor(manifest, entry) + return stage + if entry.get("file_identity") is not None: + raise MigrationError("owned_target_changed") + _verify_source(entry) + target_parent = Path(entry["target"]).parent + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0) + for _ in range(32): + name = _new_internal_name(manifest, entry, "stage") + candidate = target_parent / name + try: + descriptor = os.open(candidate, flags, 0o600) + except FileExistsError: + # A same-name entry is never adopted or removed. + continue + except OSError as exc: + raise MigrationError("target_unwritable") from exc + + digest = hashlib.sha256() + size = 0 + identity = None + try: + with os.fdopen(descriptor, "wb") as target: + with Path(entry["source"]).open("rb") as source: + while chunk := source.read(1024 * 1024): + target.write(chunk) + digest.update(chunk) + size += len(chunk) + target.flush() + os.fsync(target.fileno()) + identity = _identity_from_metadata(os.fstat(target.fileno())) + except OSError as exc: + # The random entry may no longer be ours after the handle closes. + # Preserve it for manual inspection instead of unlinking by path. + raise MigrationError("target_unwritable") from exc + if (digest.hexdigest(), size) != (entry["sha256"], entry["size"]): + # Never unlink a path whose ownership was not durably recorded. + raise MigrationError("source_hash_changed") + if identity is None: + raise MigrationError("file_identity_unavailable") + + entry["stage_name"] = name + entry["file_identity"] = identity + _persist_manifest(manifest) + _require_owned_anchor(manifest, entry) + return candidate + raise MigrationError("staging_conflict") + + +def _publish(stage: Path, target: Path) -> None: + if _metadata(target) is not None: + raise MigrationError("target_exists") + try: + os.link(stage, target) + except FileExistsError as exc: + raise MigrationError("target_exists") from exc + except OSError as exc: + code = ( + "cross_volume_publish_unsupported" + if exc.errno == errno.EXDEV + else "atomic_publish_unavailable" + ) + raise MigrationError(code) from exc + + +def _preflight_apply(manifest: Mapping[str, Any], *, new: bool) -> None: + for entry in manifest["entries"]: + state = entry["state"] + target = _content_status(Path(entry["target"]), entry) + stage = _content_status(_stage(manifest, entry), entry) + if new or state == "planned": + if target != "missing": + raise MigrationError("target_exists") + _verify_source(entry) + elif state == "created": + _require_owned_target(manifest, entry) + elif state == "creating": + if target == "changed": + raise MigrationError("owned_target_changed") + if target == "exact": + _require_owned_target(manifest, entry) + elif stage == "changed": + raise MigrationError("staging_conflict") + elif stage == "exact": + _require_owned_anchor(manifest, entry) + elif entry.get("stage_name") is not None: + raise MigrationError("owned_target_changed") + else: + _verify_source(entry) + else: + raise MigrationError("rollback_in_progress") + + +def apply_migration( + plan: MigrationPlan | MigrationResult | Mapping[str, Any], +) -> MigrationResult: + """Apply a plan with atomic create-if-absent publication.""" + + requested = _validate_manifest(_unwrap(plan)) + workspace = _existing_directory(requested["workspace"], "workspace_missing") + if workspace != Path(requested["workspace"]): + raise MigrationError("workspace_identity_changed") + _private_state_root(requested["state_root"], workspace) + _ensure_tree(Path(requested["state_root"]), _lock_path(requested).parent) + + with _kernel_lock(_lock_path(requested)): + stored = _load_stored(requested) + if stored is None: + if requested["run_state"] != "planned": + raise MigrationError("stored_manifest_missing") + current = requested + _preflight_apply(current, new=True) + current["run_state"] = "applying" + _persist_manifest(current) + else: + current = stored + if current["run_state"] == "applied": + _preflight_apply(current, new=False) + return MigrationResult( + current, + _report( + "apply", "unchanged", current["entries"], + roots=current["source_roots"], + unchanged=len(current["entries"]), + ), + ) + if current["run_state"] == "rolled-back": + raise MigrationError("migration_already_rolled_back") + if current["run_state"] == "rolling-back": + raise MigrationError("rollback_in_progress") + _preflight_apply(current, new=False) + + changed = 0 + for entry in current["entries"]: + if entry["state"] == "created": + continue + target = Path(entry["target"]) + _ensure_tree(workspace, target.parent) + if entry["state"] == "creating" and _content_status(target, entry) == "exact": + _require_owned_target(current, entry) + entry["state"] = "created" + _persist_manifest(current) + continue + if entry["state"] == "planned": + entry["state"] = "creating" + _persist_manifest(current) + stage = _copy_stage(current, entry) + _publish(stage, target) + _require_owned_target(current, entry) + entry["state"] = "created" + changed += 1 + _persist_manifest(current) + + current["run_state"] = "applied" + _persist_manifest(current) + return MigrationResult( + current, + _report( + "apply", "applied", current["entries"], + roots=current["source_roots"], + changed=changed, + unchanged=len(current["entries"]) - changed, + ), + ) + + +def _rename_noreplace(source: Path, target: Path) -> None: + """Atomically move one path without replacing an existing destination.""" + + if os.name == "nt": + try: + os.rename(source, target) + return + except FileExistsError as exc: + raise MigrationError("quarantine_conflict") from exc + except OSError as exc: + if getattr(exc, "winerror", None) in {80, 183}: + raise MigrationError("quarantine_conflict") from exc + raise MigrationError("rollback_failed") from exc + + if sys.platform.startswith("linux"): + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise MigrationError("atomic_quarantine_unavailable") + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + _AT_FDCWD, + os.fsencode(source), + _AT_FDCWD, + os.fsencode(target), + _RENAME_NOREPLACE, + ) + if result == 0: + return + error = ctypes.get_errno() + if error in {errno.EEXIST, errno.ENOTEMPTY}: + raise MigrationError("quarantine_conflict") + if error in { + errno.ENOSYS, + errno.EINVAL, + getattr(errno, "EOPNOTSUPP", errno.EINVAL), + }: + raise MigrationError("atomic_quarantine_unavailable") + raise MigrationError("rollback_failed") + + raise MigrationError("atomic_quarantine_unavailable") + + +def _require_owned_path(path: Path | None, entry: Mapping[str, Any]) -> None: + if ( + path is None + or _content_status(path, entry) != "exact" + or not _identity_matches(path, entry) + ): + raise MigrationError("owned_target_changed") + + +def _has_owned_identity(path: Path, entry: Mapping[str, Any]) -> bool: + try: + return _identity_matches(path, entry) + except MigrationError as exc: + if exc.code == "file_identity_unavailable": + return False + raise + + +def _restore_unowned_quarantine(quarantine: Path, source: Path) -> None: + if _metadata(source) is not None: + raise MigrationError("rollback_restore_blocked") + try: + _rename_noreplace(quarantine, source) + except MigrationError as exc: + if exc.code in {"quarantine_conflict", "rollback_failed"}: + raise MigrationError("rollback_restore_blocked") from exc + raise + raise MigrationError("owned_target_changed") + + +def _move_owned_to_quarantine( + manifest: Mapping[str, Any], + entry: Mapping[str, Any], + source: Path, +) -> None: + quarantine = _quarantine(manifest, entry) + if quarantine is None: + raise MigrationError("invalid_state_transition") + source_exists = _metadata(source) is not None + quarantine_exists = _metadata(quarantine) is not None + if quarantine_exists and source_exists: + raise MigrationError("quarantine_conflict") + if not quarantine_exists: + if not source_exists: + return + _rename_noreplace(source, quarantine) + quarantine_exists = True + if not quarantine_exists: + return + if ( + _content_status(quarantine, entry) != "exact" + or not _identity_matches(quarantine, entry) + ): + _restore_unowned_quarantine(quarantine, source) + if _metadata(source) is not None and _has_owned_identity(source, entry): + raise MigrationError("owned_target_changed") + + +def _delete_owned_quarantine( + manifest: Mapping[str, Any], entry: Mapping[str, Any] +) -> None: + quarantine = _quarantine(manifest, entry) + if quarantine is None or _metadata(quarantine) is None: + return + _require_owned_path(quarantine, entry) + try: + quarantine.unlink() + except OSError as exc: + raise MigrationError("rollback_failed") from exc + + +def _set_rollback_phase( + manifest: dict[str, Any], + entry: dict[str, Any], + phase: str, +) -> None: + entry["rollback_phase"] = phase + entry["quarantine_name"] = _new_internal_name(manifest, entry, "rollback") + _persist_manifest(manifest) + + +def _preflight_pending_rollback( + manifest: Mapping[str, Any], + entry: Mapping[str, Any], + *, + allow_recovery: bool, +) -> None: + phase = entry["rollback_phase"] + quarantine = _quarantine(manifest, entry) + quarantine_status = _content_status(quarantine, entry) + target = Path(entry["target"]) + anchor = _stage(manifest, entry) + + if phase == "target": + if quarantine_status == "exact" and _identity_matches(quarantine, entry): + _require_owned_anchor(manifest, entry) + return + if quarantine_status != "missing": + if allow_recovery: + return + raise MigrationError("owned_target_changed") + target_status = _content_status(target, entry) + if target_status == "exact": + _require_owned_target(manifest, entry) + elif target_status == "changed": + if not allow_recovery: + raise MigrationError("owned_target_changed") + else: + _require_owned_anchor(manifest, entry) + return + + if phase == "target-quarantined": + if quarantine_status != "missing": + _require_owned_path(quarantine, entry) + _require_owned_anchor(manifest, entry) + return + + if phase == "anchor": + if quarantine_status == "exact" and _identity_matches(quarantine, entry): + return + if quarantine_status != "missing": + if allow_recovery: + return + raise MigrationError("owned_target_changed") + if anchor is not None and _metadata(anchor) is not None: + _require_owned_anchor(manifest, entry) + return + + if phase == "anchor-quarantined": + if quarantine_status != "missing": + _require_owned_path(quarantine, entry) + return + + raise MigrationError("invalid_state_transition") + + +def _preflight_rollback( + manifest: Mapping[str, Any], *, allow_recovery: bool = False +) -> None: + for entry in manifest["entries"]: + state = entry["state"] + target = _content_status(Path(entry["target"]), entry) + anchor = _content_status(_stage(manifest, entry), entry) + if state == "created": + if target == "exact": + _require_owned_target(manifest, entry) + elif target == "missing": + _require_owned_anchor(manifest, entry) + else: + raise MigrationError("owned_target_changed") + elif state == "creating": + if entry["stage_name"] is None: + if target != "missing": + raise MigrationError("target_exists") + continue + if target == "changed": + raise MigrationError("owned_target_changed") + if target == "exact": + _require_owned_target(manifest, entry) + elif anchor == "exact": + _require_owned_anchor(manifest, entry) + elif anchor == "changed": + raise MigrationError("owned_target_changed") + elif state == "delete-pending": + _preflight_pending_rollback( + manifest, entry, allow_recovery=allow_recovery + ) + elif state == "planned": + if target != "missing": + raise MigrationError("target_exists") + + +def verify_migration( + *, + workspace: os.PathLike[str] | str, + batch_id: str, +) -> MigrationResult: + """Verify stored state and owned files without needing the original plan.""" + + loaded = load_migration(workspace=workspace, batch_id=batch_id) + requested = loaded.manifest + _ensure_tree(Path(requested["state_root"]), _lock_path(requested).parent) + with _kernel_lock(_lock_path(requested)): + current = _load_stored(requested) + if current is None: + raise MigrationError("migration_not_found") + if current["run_state"] == "planned": + _preflight_apply(current, new=True) + elif current["run_state"] in {"applying", "applied"}: + _preflight_apply(current, new=False) + else: + _preflight_rollback(current) + return MigrationResult( + current, + _report( + "verify", + current["run_state"], + current["entries"], + roots=current["source_roots"], + unchanged=len(current["entries"]), + ), + ) + + +def rollback_migration( + applied: MigrationPlan | MigrationResult | Mapping[str, Any], +) -> MigrationResult: + """Remove unchanged owned files with retryable per-entry state.""" + + requested = _validate_manifest(_unwrap(applied)) + workspace = _existing_directory(requested["workspace"], "workspace_missing") + if workspace != Path(requested["workspace"]): + raise MigrationError("workspace_identity_changed") + _private_state_root(requested["state_root"], workspace) + _ensure_tree(Path(requested["state_root"]), _lock_path(requested).parent) + + with _kernel_lock(_lock_path(requested)): + current = _load_stored(requested) + if current is None: + raise MigrationError("applied_manifest_missing") + if current["run_state"] == "rolled-back": + return MigrationResult( + current, + _report( + "rollback", "unchanged", current["entries"], + roots=current["source_roots"], + unchanged=len(current["entries"]), + ), + ) + if current["run_state"] not in {"applying", "applied", "rolling-back"}: + raise MigrationError("migration_not_applied") + _preflight_rollback(current, allow_recovery=True) + if current["run_state"] != "rolling-back": + current["run_state"] = "rolling-back" + _persist_manifest(current) + + changed = 0 + for entry in current["entries"]: + state = entry["state"] + target = Path(entry["target"]) + if state == "deleted": + continue + if state == "planned": + entry["state"] = "deleted" + _persist_manifest(current) + continue + if state == "creating" and entry["stage_name"] is None: + entry["state"] = "deleted" + _persist_manifest(current) + continue + if state != "delete-pending": + entry["state"] = "delete-pending" + _set_rollback_phase(current, entry, "target") + + while entry["state"] == "delete-pending": + phase = entry["rollback_phase"] + if phase == "target": + _move_owned_to_quarantine(current, entry, target) + entry["rollback_phase"] = "target-quarantined" + _persist_manifest(current) + continue + if phase == "target-quarantined": + _delete_owned_quarantine(current, entry) + _set_rollback_phase(current, entry, "anchor") + continue + if phase == "anchor": + anchor = _stage(current, entry) + if anchor is None: + raise MigrationError("invalid_state_transition") + _move_owned_to_quarantine(current, entry, anchor) + entry["rollback_phase"] = "anchor-quarantined" + _persist_manifest(current) + continue + if phase == "anchor-quarantined": + _delete_owned_quarantine(current, entry) + entry["state"] = "deleted" + entry["rollback_phase"] = None + entry["quarantine_name"] = None + changed += 1 + _persist_manifest(current) + continue + raise MigrationError("invalid_state_transition") + + current["run_state"] = "rolled-back" + _persist_manifest(current) + return MigrationResult( + current, + _report( + "rollback", "rolled-back", current["entries"], + roots=current["source_roots"], + changed=changed, + unchanged=len(current["entries"]) - changed, + ), + ) + + +def reapply_migration( + *, + workspace: os.PathLike[str] | str, + batch_id: str, +) -> MigrationResult: + """Resume or repeat apply using only workspace and batch identity.""" + + loaded = load_migration(workspace=workspace, batch_id=batch_id) + return apply_migration(loaded) + + +__all__ = [ + "MANIFEST_VERSION", + "MIGRATION_LIMITATIONS", + "MIGRATION_ERROR_CODES", + "MigrationError", + "MigrationPlan", + "MigrationResult", + "apply_migration", + "format_migration_report", + "load_migration", + "plan_migration", + "reapply_migration", + "rollback_migration", + "verify_migration", +] diff --git a/src/scriptorium/resume.py b/src/scriptorium/resume.py new file mode 100644 index 0000000..1fee4a0 --- /dev/null +++ b/src/scriptorium/resume.py @@ -0,0 +1,462 @@ +"""Read-only Scriptorium entry for a bounded Provenance context capsule.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from . import __version__ +from .demo import DemoError, find_script, load_compatibility, project_version, resolve_component_root +from .path_selection import format_path_selection + + +PUBLIC_COMMAND = "prov-context" +RESUME_TIMEOUT_SECONDS = 30 +CAPSULE_VERSION = "context-capsule/0.1" +ARTIFACT_KINDS = {"parsed-paper", "reading-note", "review", "lineage-graph"} +ARTIFACT_SCHEMAS = { + "parsed-paper/1.0", + "reading-note/1.0", + "review/1.0", + "lineage-graph/1.0", +} +PASSTHROUGH_ENV_NAMES = { + "APPDATA", + "COMSPEC", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "LOCALAPPDATA", + "PATH", + "PATHEXT", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TEMP", + "TMP", + "TMPDIR", + "USERPROFILE", + "WINDIR", +} +ENTRY_LIMITATIONS = ( + "The capsule contains human-authored or approved project state only.", + "Literature and research artifacts remain reference-only, not approved scientific claims.", + "The capsule is local research context and must be reviewed before sharing.", +) +_WINDOWS_PATH = re.compile(r"(?i)(?:[a-z]:[\\/]|\\\\[^\\\s]+[\\/])") +_PRIVATE_POSIX_PATH = re.compile(r"(?:^|[\s\"'])(?:/(?:home|Users|root|mnt|private|tmp)/)") + + +class ResumeError(RuntimeError): + """A trustworthy context capsule could not be produced.""" + + +def _resolve_directory(path: Path, *, label: str) -> Path: + try: + resolved = path.expanduser().resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ResumeError(f"{label} does not exist or cannot be resolved") from exc + if not resolved.is_dir(): + raise ResumeError(f"{label} is not a directory") + return resolved + + +def _compatibility_version() -> str: + try: + version = load_compatibility()["provenance"] + except (DemoError, KeyError, OSError, TypeError, ValueError) as exc: + raise ResumeError("packaged Provenance compatibility data is unavailable") from exc + if not version: + raise ResumeError("packaged Provenance compatibility version is empty") + return version + + +def _resolve_provenance( + explicit: Path | None, *, expected_version: str +) -> tuple[Path | None, Path]: + root: Path | None = None + if explicit is not None: + root = _resolve_directory(explicit, label="Provenance root") + else: + try: + root = resolve_component_root("provenance") + except (DemoError, OSError, RuntimeError) as exc: + configured = os.environ.get("SCRIPTORIUM_PROVENANCE_ROOT") + if configured and configured.strip(): + raise ResumeError("configured Provenance root is unavailable") from exc + + if root is not None: + try: + actual_version = project_version(root) + except DemoError as exc: + raise ResumeError("Provenance source metadata is unavailable") from exc + if actual_version != expected_version: + raise ResumeError( + "incompatible Provenance source version: " + f"expected {expected_version}, found {actual_version}" + ) + try: + return root, find_script(root, PUBLIC_COMMAND) + except DemoError as exc: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' is unavailable") from exc + + installed = shutil.which(PUBLIC_COMMAND) + if not installed: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' is unavailable") + return None, Path(installed).resolve() + + +def _resume_environment(provenance_home: Path) -> dict[str, str]: + env = { + name: value + for name, value in os.environ.items() + if name.upper() in PASSTHROUGH_ENV_NAMES + } + env.update( + { + "PROVENANCE_HOME": str(provenance_home), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + } + ) + return env + + +def _object(value: Any, *, fields: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != fields: + raise ResumeError(f"context capsule field '{label}' is incompatible") + return value + + +def _text(value: Any, *, label: str, maximum: int = 4000) -> str: + if not isinstance(value, str) or len(value) > maximum: + raise ResumeError(f"context capsule field '{label}' is incompatible") + if _WINDOWS_PATH.search(value) or _PRIVATE_POSIX_PATH.search(value): + raise ResumeError("context capsule contains a suppressed local path") + return value + + +def _text_list( + value: Any, *, label: str, maximum_items: int = 100, maximum_text: int = 2000 +) -> list[str]: + if not isinstance(value, list) or len(value) > maximum_items: + raise ResumeError(f"context capsule field '{label}' is incompatible") + return [ + _text(item, label=f"{label}[]", maximum=maximum_text) for item in value + ] + + +def _rebuild_project(value: Any) -> dict[str, str]: + fields = { + "project_id", + "title", + "status", + "stage", + "priority", + "updated", + "goal", + "conclusion", + } + project = _object(value, fields=fields, label="project") + return { + key: _text(project[key], label=f"project.{key}") + for key in ( + "project_id", + "title", + "status", + "stage", + "priority", + "updated", + "goal", + "conclusion", + ) + } + + +def _rebuild_recent_progress(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > 30: + raise ResumeError("context capsule field 'recent_progress' is incompatible") + rebuilt = [] + for item in value: + entry = _object(item, fields={"date", "items"}, label="recent_progress[]") + rebuilt.append( + { + "date": _text(entry["date"], label="recent_progress[].date", maximum=64), + "items": _text_list( + entry["items"], label="recent_progress[].items", maximum_items=30 + ), + } + ) + return rebuilt + + +def _rebuild_literature(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > 100: + raise ResumeError("context capsule field 'literature' is incompatible") + rebuilt = [] + fields = {"id", "citekey", "title", "year", "read_status", "tldr"} + for item in value: + entry = _object(item, fields=fields, label="literature[]") + year = entry["year"] + if year is not None and ( + isinstance(year, bool) or not isinstance(year, (int, str)) + ): + raise ResumeError("context capsule field 'literature[].year' is incompatible") + rebuilt.append( + { + "id": _text(entry["id"], label="literature[].id", maximum=256), + "citekey": _text( + entry["citekey"], label="literature[].citekey", maximum=256 + ), + "title": _text(entry["title"], label="literature[].title"), + "year": year, + "read_status": _text( + entry["read_status"], label="literature[].read_status", maximum=128 + ), + "tldr": _text(entry["tldr"], label="literature[].tldr"), + } + ) + return rebuilt + + +def _rebuild_artifacts(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > 100: + raise ResumeError("context capsule field 'research_artifacts' is incompatible") + rebuilt = [] + fields = { + "kind", + "schema_version", + "id", + "title", + "status", + "source_ids", + "summary", + "trust", + } + for item in value: + entry = _object(item, fields=fields, label="research_artifacts[]") + kind = _text(entry["kind"], label="research_artifacts[].kind", maximum=64) + schema_version = _text( + entry["schema_version"], + label="research_artifacts[].schema_version", + maximum=64, + ) + if kind not in ARTIFACT_KINDS or schema_version not in ARTIFACT_SCHEMAS: + raise ResumeError("context capsule research artifact type is incompatible") + if entry["trust"] != "reference_only": + raise ResumeError("context capsule research artifact trust is incompatible") + rebuilt.append( + { + "kind": kind, + "schema_version": schema_version, + "id": _text(entry["id"], label="research_artifacts[].id", maximum=256), + "title": _text(entry["title"], label="research_artifacts[].title"), + "status": _text( + entry["status"], label="research_artifacts[].status", maximum=128 + ), + "source_ids": _text_list( + entry["source_ids"], + label="research_artifacts[].source_ids", + maximum_items=100, + maximum_text=256, + ), + "summary": _text(entry["summary"], label="research_artifacts[].summary"), + "trust": "reference_only", + } + ) + return rebuilt + + +def _parse_capsule(stdout: str) -> dict[str, Any]: + try: + value = json.loads(stdout) + except json.JSONDecodeError as exc: + raise ResumeError("public context command returned non-JSON or extra stdout") from exc + fields = { + "capsule_version", + "project", + "next_actions", + "blocked_by", + "recent_progress", + "literature", + "research_artifacts", + "reference_leads", + "trust", + "limits", + } + capsule = _object(value, fields=fields, label="root") + if capsule["capsule_version"] != CAPSULE_VERSION: + raise ResumeError("public context command returned an incompatible capsule version") + + leads = _object( + capsule["reference_leads"], + fields={"gaps", "priority_reads"}, + label="reference_leads", + ) + trust = _object( + capsule["trust"], + fields={"project_state", "recent_progress", "research_artifacts"}, + label="trust", + ) + if trust != { + "project_state": "human_or_approved", + "recent_progress": "auto_applied_low_risk_not_approved_claims", + "research_artifacts": "reference_only_not_approved_claims", + }: + raise ResumeError("public context command returned incompatible trust semantics") + limits = _object( + capsule["limits"], fields={"max_markdown_chars", "truncated"}, label="limits" + ) + if limits.get("max_markdown_chars") != 8000 or not isinstance( + limits.get("truncated"), bool + ): + raise ResumeError("public context command returned incompatible limits") + + return { + "capsule_version": CAPSULE_VERSION, + "project": _rebuild_project(capsule["project"]), + "next_actions": _text_list(capsule["next_actions"], label="next_actions"), + "blocked_by": _text(capsule["blocked_by"], label="blocked_by"), + "recent_progress": _rebuild_recent_progress(capsule["recent_progress"]), + "literature": _rebuild_literature(capsule["literature"]), + "research_artifacts": _rebuild_artifacts(capsule["research_artifacts"]), + "reference_leads": { + "gaps": _text_list(leads["gaps"], label="reference_leads.gaps"), + "priority_reads": _text_list( + leads["priority_reads"], label="reference_leads.priority_reads" + ), + }, + "trust": dict(trust), + "limits": dict(limits), + } + + +def run_resume( + *, provenance_home: Path, project: str, provenance_root: Path | None = None +) -> dict[str, Any]: + """Read one project capsule through Provenance's public command.""" + resolved_home = _resolve_directory(provenance_home, label="Provenance home") + expected_version = _compatibility_version() + resolved_root, command = _resolve_provenance( + provenance_root, expected_version=expected_version + ) + if not isinstance(project, str) or not project.strip(): + raise ResumeError("project is required") + cwd = resolved_root or resolved_home + environment = _resume_environment(resolved_home) + try: + version_probe = subprocess.run( + [str(command), "--version"], + cwd=cwd, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=RESUME_TIMEOUT_SECONDS, + check=False, + ) + if ( + version_probe.returncode != 0 + or version_probe.stdout.strip() != expected_version + ): + raise ResumeError( + f"public command '{PUBLIC_COMMAND}' has an incompatible runtime version" + ) + argv = [str(command), "--project", project, "--json"] + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=RESUME_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' timed out") from exc + except (OSError, UnicodeError) as exc: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' could not run") from exc + if completed.returncode != 0: + raise ResumeError(f"public command '{PUBLIC_COMMAND}' did not return a capsule") + + capsule = _parse_capsule(completed.stdout) + return { + "format_version": 1, + "generated_by": {"name": "scriptorium", "version": __version__}, + "operation": "resume", + "status": "ready", + "exit_code": 0, + "capsule": capsule, + "egress": { + "suite_managed": "not-requested", + "host_managed": "not-invoked", + "optional_connectors": "not-invoked", + }, + "entry": { + "public_command": PUBLIC_COMMAND, + "component_generated_by": { + "name": "provenance", + "version": expected_version, + }, + "expected_component_version": expected_version, + "component_exit_code": completed.returncode, + "stdout": "parsed-and-suppressed", + "stderr": "suppressed" if completed.stderr else "empty", + }, + "limitations": list(ENTRY_LIMITATIONS), + } + + +def format_resume_report(report: dict[str, Any]) -> str: + """Render the trusted capsule without exposing component diagnostics or paths.""" + capsule = report["capsule"] + project = capsule["project"] + lines = [ + f"Scriptorium resume {report['generated_by']['version']}", + f"Project: {project['title']} ({project['project_id']})", + f"State: {project['status']} / {project['stage']}", + ] + if project["goal"]: + lines.extend(["", "Goal:", project["goal"]]) + if project["conclusion"]: + lines.extend(["", "Approved conclusion:", project["conclusion"]]) + if capsule["next_actions"]: + lines.extend(["", "Next actions:"]) + lines.extend(f"- {item}" for item in capsule["next_actions"]) + if capsule["blocked_by"]: + lines.extend(["", "Blocked by:", capsule["blocked_by"]]) + if capsule["recent_progress"]: + lines.extend(["", "Recent progress (auto-applied low-risk context):"]) + for entry in capsule["recent_progress"]: + lines.extend(f"- {entry['date']}: {item}" for item in entry["items"]) + if capsule["literature"] or capsule["research_artifacts"]: + lines.extend( + [ + "", + "Reference context:", + f"- Literature records: {len(capsule['literature'])}", + f"- Research artifacts: {len(capsule['research_artifacts'])}", + "- Trust: reference-only; verify against primary evidence.", + ] + ) + if capsule["limits"]["truncated"]: + lines.extend(["", "Note: the bounded capsule was truncated."]) + lines.extend( + format_path_selection( + report, + conflict_guidance="pass an explicit --provenance-home to select the intended local data root.", + ) + ) + return "\n".join(lines) diff --git a/tests/_e2e_lectern_worker.py b/tests/_e2e_lectern_worker.py new file mode 100644 index 0000000..2622794 --- /dev/null +++ b/tests/_e2e_lectern_worker.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Lectern-side worker for the synthetic cross-repository golden path.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from asa_agents import FakeLLM, build_outline +from asa_agents.graph import build_graph +from ingestion import ingest_handoff +from langgraph.types import Command +from pptx import Presentation +from slide_ir import GenerationState + + +class WorkerFailure(RuntimeError): + """Lectern violated a golden-path invariant.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise WorkerFailure(message) + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + require(isinstance(value, dict), "Handoff metadata must be an object") + return value + + +def scripted_deck(meta: dict[str, Any]) -> str: + papers = meta["papers"] + rows = [ + [ + str(paper["title"]), + f"SYNTHETIC-PAPER-{index}-EVIDENCE", + str(paper["doi"]), + ] + for index, paper in enumerate(papers, start=1) + ] + return json.dumps( + { + "deck_id": "synthetic-lectern-golden", + "slides": [ + { + "slide_id": "cover", + "layout_type": "title", + "title": "[SYNTHETIC] Dual-paper evidence review", + "subtitle": "Offline, credential-free golden path", + "blocks": [], + "speaker_notes": "This deck contains synthetic evidence only.", + "provenance": {"source": "report_basis"}, + }, + { + "slide_id": "evidence", + "layout_type": "two_column_table", + "title": "Evidence-backed comparison", + "blocks": [ + { + "type": "table", + "columns": ["Source", "Evidence marker", "DOI"], + "rows": rows, + "needs_human_check": False, + } + ], + "speaker_notes": "Each row remains traceable to the handoff.", + "provenance": { + "source": f"handoff:{meta['key']}", + "paper_count": len(papers), + }, + }, + ], + }, + ensure_ascii=True, + ) + + +def all_text(prs: Presentation) -> list[str]: + values: list[str] = [] + for slide in prs.slides: + for shape in slide.shapes: + if getattr(shape, "has_text_frame", False): + values.append(shape.text) + if getattr(shape, "has_table", False): + table = shape.table + for row in table.rows: + values.extend(cell.text for cell in row.cells) + return values + + +def field(value: Any, name: str, default: Any = None) -> Any: + if isinstance(value, dict): + return value.get(name, default) + return getattr(value, name, default) + + +def run(handoff: Path, output_dir: Path) -> dict[str, Any]: + meta = read_json(handoff / "meta.json") + papers = meta.get("papers") + require(meta.get("schema_version") == "handoff/1.1", "Expected handoff/1.1") + require(isinstance(papers, list) and len(papers) == 2, "Expected two papers") + + output_dir.mkdir(parents=True, exist_ok=True) + require(not list(output_dir.rglob("*.pptx")), "Output directory was not empty") + + ingested = ingest_handoff(handoff) + evidence_text = "\n".join( + str(asset.content_ref) + for asset in ingested.assets + if str(asset.kind.value) == "section_text" + ) + expected_evidence = [ + str(paper[field]) + for paper in papers + for field in ("title", "doi") + ] + [ + "SYNTHETIC-PAPER-1-EVIDENCE", + "SYNTHETIC-PAPER-2-EVIDENCE", + ] + for value in expected_evidence: + require(value in evidence_text, f"Lectern evidence omitted {value!r}") + + llm = FakeLLM(scripted_deck(meta)) + state = GenerationState( + job_id="synthetic-golden-path", + evidence=ingested.assets, + tables=ingested.tables, + max_retries=0, + ) + graph = build_graph( + llm, + out_dir=output_dir, + planner=build_outline, + ) + config = {"configurable": {"thread_id": state.job_id}} + graph.invoke(state.model_dump(), config) + snapshot = graph.get_state(config) + + require( + "approval" in snapshot.next, + "Production graph did not pause at the outline approval gate", + ) + require( + not field(snapshot.values, "user_approved_outline", False), + "Production graph bypassed human approval", + ) + require( + field(snapshot.values, "output_path") is None, + "Production graph assigned an output before approval", + ) + require( + not list(output_dir.rglob("*.pptx")), + "A persistent PPTX was created before outline approval", + ) + require(len(llm.calls) == 1, "FakeLLM was not called exactly once") + prompt = llm.calls[0]["prompt"] + for value in expected_evidence: + require(value in prompt, f"Planner prompt omitted {value!r}") + + final = graph.invoke(Command(resume={"approved": True}), config) + require(field(final, "user_approved_outline") is True, "Approval was not recorded") + approved_value = field(final, "output_path") + require(isinstance(approved_value, str), "Approval did not produce an output path") + approved_path = Path(approved_value) + require(approved_path.is_file(), "Approved deck was not generated") + + presentation = Presentation(str(approved_path)) + require(len(presentation.slides) == 2, "Generated deck has the wrong slide count") + table_shapes = [ + shape + for slide in presentation.slides + for shape in slide.shapes + if getattr(shape, "has_table", False) + ] + require(len(table_shapes) == 1, "Generated deck lacks one native table") + table = table_shapes[0].table + require(len(table.rows) == 3, "Native table lost a synthetic evidence row") + require(len(table.columns) == 3, "Native table lost a column") + + title_shape = next( + ( + shape + for slide in presentation.slides + for shape in slide.shapes + if getattr(shape, "has_text_frame", False) + and "[SYNTHETIC] Dual-paper evidence review" in shape.text + ), + None, + ) + require(title_shape is not None, "Generated deck lacks editable title text") + title_shape.text_frame.text = "[SYNTHETIC] Edited evidence review" + table.cell(1, 1).text = "SYNTHETIC-EDIT-CONFIRMED" + + edited_path = output_dir / "edited-synthetic-deck.pptx" + presentation.save(str(edited_path)) + reopened = Presentation(str(edited_path)) + reopened_text = all_text(reopened) + require( + "[SYNTHETIC] Edited evidence review" in reopened_text, + "Edited title did not survive reopening", + ) + require( + "SYNTHETIC-EDIT-CONFIRMED" in reopened_text, + "Edited table cell did not survive reopening", + ) + + return { + "status": "passed", + "handoff_schema": meta["schema_version"], + "paper_count": len(papers), + "evidence_asset_count": len(ingested.assets), + "warning_count": len(ingested.warnings), + "production_graph_exercised": True, + "planner_used_source_evidence": True, + "hard_stop_blocked_pptx": True, + "approved_slide_count": len(reopened.slides), + "native_table_count": len(table_shapes), + "editable_after_reopen": True, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--handoff", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + report = run(args.handoff.resolve(), args.output_dir.resolve()) + except (OSError, ValueError, WorkerFailure) as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + print(json.dumps(report, ensure_ascii=True, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e_install_lifecycle.py b/tests/e2e_install_lifecycle.py new file mode 100644 index 0000000..78e138d --- /dev/null +++ b/tests/e2e_install_lifecycle.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +"""Offline clean-install lifecycle acceptance for the Scriptorium runtime. + +Only explicitly supplied local source checkouts are used. The runner stages the +public package trees into a temporary directory, builds local wheels without an +index or build isolation, then verifies install, uninstall, reinstall, doctor, +and the credential-free demo in a brand-new virtual environment. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import venv +from pathlib import Path +from typing import Any + + +PACKAGES = { + "scriptorium-suite": Path("src") / "scriptorium", + "scriptorium-steward": Path("src") / "steward", + "provenance": Path("provenance"), +} +MODULES = { + "scriptorium-suite": "scriptorium", + "scriptorium-steward": "steward", + "provenance": "provenance", +} +ENTRIES = ("scriptorium", "steward", "prov-sync-pull") +SYSTEM_ENV = { + "COMSPEC", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NUMBER_OF_PROCESSORS", + "PATH", + "PATHEXT", + "PROCESSOR_ARCHITECTURE", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TERM", + "WINDIR", +} +RELEASE = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:[.+-][A-Za-z0-9.-]+)?$") + + +class LifecycleFailure(RuntimeError): + def __init__(self, stage: str, message: str): + super().__init__(message) + self.stage = stage + self.safe_message = message + + +def project_identity(root: Path, expected_name: str) -> dict[str, str]: + try: + resolved = root.resolve(strict=True) + except OSError as exc: + raise LifecycleFailure("validate-sources", "A required source is unavailable.") from exc + if root.is_symlink() or not resolved.is_dir(): + raise LifecycleFailure("validate-sources", "A source root is unsafe.") + pyproject = resolved / "pyproject.toml" + if pyproject.is_symlink() or not pyproject.is_file(): + raise LifecycleFailure("validate-sources", "A source lacks pyproject.toml.") + try: + project = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"] + name, version = project["name"], project["version"] + except (OSError, KeyError, TypeError, tomllib.TOMLDecodeError) as exc: + raise LifecycleFailure("validate-sources", "A package identity is invalid.") from exc + if name != expected_name or not isinstance(version, str) or not version: + raise LifecycleFailure("validate-sources", "A package identity does not match.") + return {"name": name, "version": version} + + +def release_relation(previous: str, current: str) -> str: + old, new = RELEASE.fullmatch(previous), RELEASE.fullmatch(current) + if old is None or new is None: + return "unsupported" + old_core = tuple(int(value) for value in old.groups()) + new_core = tuple(int(value) for value in new.groups()) + if old_core < new_core: + return "previous-is-older" + if old_core == new_core: + return "same" + return "previous-is-newer" + + +def isolated_environment(root: Path, scripts: Path | None = None) -> dict[str, str]: + profile = root / "profile" + runtime_temp = root / "runtime-temp" + appdata = profile / "AppData" / "Roaming" + localappdata = profile / "AppData" / "Local" + for directory in (profile, runtime_temp, appdata, localappdata, root / "pip-cache"): + directory.mkdir(parents=True, exist_ok=True) + env = { + name: value + for name, value in os.environ.items() + if name.upper() in SYSTEM_ENV + } + path = [str(scripts)] if scripts is not None else [] + if env.get("PATH"): + path.append(env["PATH"]) + env.update( + { + "HOME": str(profile), + "USERPROFILE": str(profile), + "APPDATA": str(appdata), + "LOCALAPPDATA": str(localappdata), + "TEMP": str(runtime_temp), + "TMP": str(runtime_temp), + "TMPDIR": str(runtime_temp), + "CODEX_HOME": str(profile / ".codex"), + "SCRIPTORIUM_CONFIG_DIR": str(root / "config"), + "PIP_CACHE_DIR": str(root / "pip-cache"), + "PIP_CONFIG_FILE": os.devnull, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INDEX": "1", + "PIP_NO_INPUT": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + "PATH": os.pathsep.join(path), + } + ) + return env + + +def run( + command: list[str | Path], + *, + stage: str, + cwd: Path, + env: dict[str, str], + timeout: int = 180, +) -> subprocess.CompletedProcess[str]: + try: + result = subprocess.run( + [str(value) for value in command], + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired, UnicodeError) as exc: + raise LifecycleFailure(stage, "A local subprocess could not complete.") from exc + if result.returncode != 0: + raise LifecycleFailure(stage, "A local subprocess returned a failure.") + return result + + +def copy_public_tree(source: Path, destination: Path) -> None: + if source.is_symlink() or not source.is_dir(): + raise LifecycleFailure("stage-sources", "A public source tree is unsafe.") + destination.mkdir(parents=True, exist_ok=False) + for item in sorted(source.rglob("*"), key=lambda path: path.as_posix()): + relative = item.relative_to(source) + if "__pycache__" in relative.parts or item.suffix in {".pyc", ".pyo"}: + continue + if item.is_symlink(): + raise LifecycleFailure("stage-sources", "A public source contains a link.") + target = destination / relative + if item.is_dir(): + target.mkdir(parents=True, exist_ok=True) + elif item.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(item, target) + else: + raise LifecycleFailure("stage-sources", "A public source contains a special file.") + + +def copy_metadata(source: Path, destination: Path) -> None: + for filename in ("pyproject.toml", "README.md", "LICENSE", "NOTICE"): + candidate = source / filename + if not candidate.exists(): + continue + if candidate.is_symlink() or not candidate.is_file(): + raise LifecycleFailure("stage-sources", "A metadata file is unsafe.") + shutil.copy2(candidate, destination / filename) + + +def stage_package(source: Path, destination: Path, distribution: str) -> Path: + package = PACKAGES.get(distribution) + if package is None or destination.exists() or destination.is_symlink(): + raise LifecycleFailure("stage-sources", "A package staging request is invalid.") + destination.mkdir(parents=True) + copy_metadata(source, destination) + if not (destination / "pyproject.toml").is_file(): + raise LifecycleFailure("stage-sources", "Staged metadata is incomplete.") + copy_public_tree(source / package, destination / package) + return destination + + +def stage_spec(source: Path, destination: Path) -> Path: + project_identity(source, "scriptorium-spec") + if destination.exists() or destination.is_symlink(): + raise LifecycleFailure("stage-sources", "The Spec staging target is occupied.") + destination.mkdir(parents=True) + copy_metadata(source, destination) + for directory in ("tools", "examples", "schemas"): + copy_public_tree(source / directory, destination / directory) + return destination + + +def build_wheel( + source: Path, + *, + distribution: str, + label: str, + root: Path, + env: dict[str, str], +) -> tuple[Path, Path]: + staged = stage_package( + source, + root / "staged-sources" / label, + distribution, + ) + wheel_dir = root / "wheels" / label + wheel_dir.mkdir(parents=True) + run( + [ + sys.executable, + "-m", + "pip", + "wheel", + "--no-index", + "--no-deps", + "--no-build-isolation", + "--wheel-dir", + wheel_dir, + staged, + ], + stage=f"build-{label}", + cwd=root, + env=env, + ) + wheels = list(wheel_dir.glob("*.whl")) + if len(wheels) != 1 or wheels[0].is_symlink(): + raise LifecycleFailure(f"build-{label}", "The local wheel build was ambiguous.") + return wheels[0], staged + + +def venv_paths(root: Path) -> tuple[Path, Path]: + scripts = root / ("Scripts" if os.name == "nt" else "bin") + return scripts / ("python.exe" if os.name == "nt" else "python"), scripts + + +def entry(scripts: Path, name: str) -> Path: + return scripts / (f"{name}.exe" if os.name == "nt" else name) + + +def install( + python: Path, + wheels: list[Path], + *, + cwd: Path, + env: dict[str, str], + stage: str, + force: bool = False, +) -> None: + command: list[str | Path] = [ + python, + "-m", + "pip", + "install", + "--no-index", + "--no-deps", + ] + if force: + command.append("--force-reinstall") + run([*command, *wheels], stage=stage, cwd=cwd, env=env) + + +def package_state( + python: Path, *, cwd: Path, env: dict[str, str], stage: str +) -> dict[str, dict[str, Any]]: + code = ( + "import importlib.metadata as m, importlib.util, json\n" + f"items={tuple(MODULES.items())!r}\n" + "result={}\n" + "for dist,module in items:\n" + " try: version=m.version(dist)\n" + " except m.PackageNotFoundError: version=None\n" + " result[dist]={'version':version,'importable':importlib.util.find_spec(module) is not None}\n" + "print(json.dumps(result,sort_keys=True))\n" + ) + output = run( + [python, "-I", "-c", code], + stage=stage, + cwd=cwd, + env=env, + ).stdout + try: + value = json.loads(output) + except json.JSONDecodeError as exc: + raise LifecycleFailure(stage, "Package verification did not return JSON.") from exc + if not isinstance(value, dict): + raise LifecycleFailure(stage, "Package verification returned an invalid value.") + return value + + +def verify_installed( + python: Path, + scripts: Path, + expected: dict[str, str], + *, + cwd: Path, + env: dict[str, str], + stage: str, +) -> None: + state = package_state(python, cwd=cwd, env=env, stage=stage) + for distribution, version in expected.items(): + item = state.get(distribution) + if ( + not isinstance(item, dict) + or item.get("version") != version + or item.get("importable") is not True + ): + raise LifecycleFailure(stage, "An installed package failed verification.") + for name in ENTRIES: + candidate = entry(scripts, name) + if candidate.is_symlink() or not candidate.is_file(): + raise LifecycleFailure(stage, "A required console entry is unavailable.") + if run( + [entry(scripts, "scriptorium"), "--version"], + stage=stage, + cwd=cwd, + env=env, + ).stdout.strip() != f"scriptorium {expected['scriptorium-suite']}": + raise LifecycleFailure(stage, "The Scriptorium version did not match.") + if run( + [entry(scripts, "steward"), "--version"], + stage=stage, + cwd=cwd, + env=env, + ).stdout.strip() != f"steward {expected['scriptorium-steward']}": + raise LifecycleFailure(stage, "The Steward version did not match.") + try: + capabilities = json.loads( + run( + [entry(scripts, "prov-sync-pull"), "--capabilities", "--json"], + stage=stage, + cwd=cwd, + env=env, + ).stdout + ) + except json.JSONDecodeError as exc: + raise LifecycleFailure(stage, "The Provenance capability probe was invalid.") from exc + if ( + not isinstance(capabilities, dict) + or capabilities.get("operation") != "pull.capabilities" + or capabilities.get("generated_by", {}).get("version") != expected["provenance"] + ): + raise LifecycleFailure(stage, "The Provenance version did not match.") + + +def uninstall( + python: Path, scripts: Path, *, cwd: Path, env: dict[str, str] +) -> None: + run( + [python, "-m", "pip", "uninstall", "--yes", *PACKAGES], + stage="uninstall", + cwd=cwd, + env=env, + ) + state = package_state(python, cwd=cwd, env=env, stage="verify-uninstalled") + if any( + not isinstance(item, dict) + or item.get("version") is not None + or item.get("importable") is not False + for item in state.values() + ): + raise LifecycleFailure("verify-uninstalled", "A package remained after uninstall.") + if any(entry(scripts, name).exists() for name in ENTRIES): + raise LifecycleFailure("verify-uninstalled", "A console entry remained after uninstall.") + + +def all_strings(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [ + text + for key, item in value.items() + for text in (*all_strings(key), *all_strings(item)) + ] + if isinstance(value, list): + return [text for item in value for text in all_strings(item)] + return [] + + +def verify_demo( + scripts: Path, + *, + root: Path, + spec: Path, + steward: Path, + provenance: Path, + env: dict[str, str], +) -> None: + command = entry(scripts, "scriptorium") + common = [ + "--spec-root", + spec, + "--steward-root", + steward, + "--provenance-root", + provenance, + ] + run( + [command, "doctor", "--target", "demo", "--json", *common], + stage="doctor-after-reinstall", + cwd=root, + env=env, + ) + output = root / "synthetic-demo" + run( + [command, "demo", "--output", output, *common], + stage="demo-after-reinstall", + cwd=root, + env=env, + ) + try: + report = json.loads((output / "demo-report.json").read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise LifecycleFailure("demo-after-reinstall", "The demo report was invalid.") from exc + if not isinstance(report, dict) or report.get("demo_status") != "passed": + raise LifecycleFailure("demo-after-reinstall", "The synthetic demo did not pass.") + forbidden = str(root).replace("\\", "/").casefold() + if any(forbidden in text.replace("\\", "/").casefold() for text in all_strings(report)): + raise LifecycleFailure("demo-after-reinstall", "The demo report exposed its local path.") + + +def write_report(path: Path, report: dict[str, Any]) -> None: + if path.exists() or path.is_symlink(): + raise LifecycleFailure("write-report", "The report destination already exists.") + try: + parent = path.parent.resolve(strict=True) + except OSError as exc: + raise LifecycleFailure("write-report", "The report parent is unavailable.") from exc + temporary = parent / f".{path.name}.{os.getpid()}.tmp" + if temporary.exists() or temporary.is_symlink(): + raise LifecycleFailure("write-report", "The temporary report target is occupied.") + try: + with temporary.open("x", encoding="utf-8", newline="\n") as handle: + json.dump(report, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + os.replace(temporary, path) + except OSError as exc: + temporary.unlink(missing_ok=True) + raise LifecycleFailure("write-report", "The lifecycle report could not be written.") from exc + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + value.add_argument("--scriptorium-root", type=Path, required=True) + value.add_argument("--spec-root", type=Path, required=True) + value.add_argument("--steward-root", type=Path, required=True) + value.add_argument("--provenance-root", type=Path, required=True) + value.add_argument("--previous-scriptorium-root", type=Path) + value.add_argument("--report", type=Path) + value.add_argument("--require-windows", action="store_true") + return value + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + if args.require_windows and os.name != "nt": + raise LifecycleFailure("platform", "This acceptance run requires Windows.") + roots = { + "scriptorium-suite": args.scriptorium_root, + "scriptorium-steward": args.steward_root, + "provenance": args.provenance_root, + } + identities = { + name: project_identity(root, name) for name, root in roots.items() + } + expected = {name: value["version"] for name, value in identities.items()} + stages = [{"name": "validate-sources", "status": "passed"}] + limitations = [ + "OS-level network egress is not instrumented; pip is constrained to local sources with no index.", + "Live Codex/Claude sessions and PowerPoint editing remain manual acceptance items.", + ] + transition: dict[str, Any] = {"status": "not-requested"} + if args.previous_scriptorium_root is None: + limitations.append( + "Upgrade/downgrade is not automated because no older local source was supplied." + ) + previous = None + relation = "not-requested" + else: + previous = project_identity( + args.previous_scriptorium_root, "scriptorium-suite" + ) + relation = release_relation(previous["version"], expected["scriptorium-suite"]) + transition.update( + { + "status": "pending", + "previous_version": previous["version"], + "current_version": expected["scriptorium-suite"], + "relation": relation, + } + ) + try: + import setuptools # noqa: F401 + import wheel # noqa: F401 + except ImportError as exc: + raise LifecycleFailure("build-tooling", "Local wheel build tooling is unavailable.") from exc + stages.append({"name": "build-tooling", "status": "passed"}) + + with tempfile.TemporaryDirectory(prefix="scriptorium-install-lifecycle-") as raw: + root = Path(raw) + build_env = isolated_environment(root / "build-environment") + wheels: list[Path] = [] + staged: dict[str, Path] = {} + for distribution, source in roots.items(): + wheel, staged_root = build_wheel( + source.resolve(strict=True), + distribution=distribution, + label=distribution, + root=root, + env=build_env, + ) + wheels.append(wheel) + staged[distribution] = staged_root + stages.append({"name": f"build-{distribution}", "status": "passed"}) + spec = stage_spec( + args.spec_root.resolve(strict=True), + root / "staged-sources" / "scriptorium-spec", + ) + stages.append({"name": "stage-scriptorium-spec", "status": "passed"}) + + previous_wheel = None + if previous is not None and relation == "previous-is-older": + previous_wheel, _ = build_wheel( + args.previous_scriptorium_root.resolve(strict=True), + distribution="scriptorium-suite", + label="previous-scriptorium-suite", + root=root, + env=build_env, + ) + stages.append( + {"name": "build-previous-scriptorium-suite", "status": "passed"} + ) + elif previous is not None: + transition["status"] = "not-covered" + limitations.append( + "Upgrade/downgrade is not claimed because the supplied versions are not an older-to-current pair." + ) + + environment = root / "clean-venv" + try: + venv.EnvBuilder(with_pip=True, symlinks=False).create(environment) + except OSError as exc: + raise LifecycleFailure("create-clean-venv", "The clean venv could not be created.") from exc + python, scripts = venv_paths(environment) + if not python.is_file(): + raise LifecycleFailure("create-clean-venv", "The clean venv lacks Python.") + stages.append({"name": "create-clean-venv", "status": "passed"}) + runtime_env = isolated_environment(root / "runtime-environment", scripts) + + lifecycle = { + "clean_install": "not-run", + "uninstall": "not-run", + "reinstall": "not-run", + "doctor_and_demo": "not-run", + "version_transition": transition, + } + install(python, wheels, cwd=root, env=runtime_env, stage="clean-install") + verify_installed( + python, scripts, expected, cwd=root, env=runtime_env, stage="verify-clean-install" + ) + lifecycle["clean_install"] = "passed" + stages.extend( + [ + {"name": "clean-install", "status": "passed"}, + {"name": "verify-clean-install", "status": "passed"}, + ] + ) + uninstall(python, scripts, cwd=root, env=runtime_env) + lifecycle["uninstall"] = "passed" + stages.extend( + [ + {"name": "uninstall", "status": "passed"}, + {"name": "verify-uninstalled", "status": "passed"}, + ] + ) + install(python, wheels, cwd=root, env=runtime_env, stage="reinstall") + verify_installed( + python, scripts, expected, cwd=root, env=runtime_env, stage="verify-reinstall" + ) + lifecycle["reinstall"] = "passed" + stages.extend( + [ + {"name": "reinstall", "status": "passed"}, + {"name": "verify-reinstall", "status": "passed"}, + ] + ) + if previous_wheel is not None and previous is not None: + install( + python, + [previous_wheel], + cwd=root, + env=runtime_env, + stage="verified-downgrade", + force=True, + ) + old_expected = dict(expected) + old_expected["scriptorium-suite"] = previous["version"] + verify_installed( + python, scripts, old_expected, cwd=root, env=runtime_env, stage="verify-downgrade" + ) + install( + python, + [wheels[0]], + cwd=root, + env=runtime_env, + stage="verified-upgrade", + force=True, + ) + verify_installed( + python, scripts, expected, cwd=root, env=runtime_env, stage="verify-upgrade" + ) + transition["status"] = "passed" + stages.extend( + [ + {"name": "verified-downgrade", "status": "passed"}, + {"name": "verify-downgrade", "status": "passed"}, + {"name": "verified-upgrade", "status": "passed"}, + {"name": "verify-upgrade", "status": "passed"}, + ] + ) + verify_demo( + scripts, + root=root, + spec=spec, + steward=staged["scriptorium-steward"], + provenance=staged["provenance"], + env=runtime_env, + ) + lifecycle["doctor_and_demo"] = "passed" + stages.extend( + [ + {"name": "doctor-after-reinstall", "status": "passed"}, + {"name": "demo-after-reinstall", "status": "passed"}, + ] + ) + + return { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "passed", + "components": identities, + "platform": { + "implementation": sys.implementation.name, + "python": ".".join(str(value) for value in sys.version_info[:3]), + "windows": os.name == "nt", + }, + "safety": { + "data": "synthetic-only", + "network_index": "disabled", + "build_isolation": "disabled", + "runtime_dependencies_added": False, + "temporary_workspace": True, + }, + "lifecycle": lifecycle, + "stages": stages, + "limitations": limitations, + } + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + report = execute(args) + exit_code = 0 + except LifecycleFailure as exc: + report = { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "failed", + "failure": {"stage": exc.stage, "message": exc.safe_message}, + } + exit_code = 1 + except Exception: + report = { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "failed", + "failure": { + "stage": "internal", + "message": "The lifecycle runner encountered an internal failure.", + }, + } + exit_code = 2 + if args.report is not None: + try: + write_report(args.report, report) + except LifecycleFailure as exc: + report = { + "format_version": 1, + "operation": "windows-install-lifecycle", + "status": "failed", + "failure": {"stage": exc.stage, "message": exc.safe_message}, + } + exit_code = 2 + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e_pull.py b/tests/e2e_pull.py index 42bfc58..559922a 100644 --- a/tests/e2e_pull.py +++ b/tests/e2e_pull.py @@ -32,6 +32,7 @@ HUMAN_TEXT = "Human-owned research rationale must remain byte-identical." TRAILING_HUMAN_TEXT = "Human-owned trailing notes must also remain untouched." TIMELINE_FACT = "Captured the synthetic Codex research session through the public pull entry." +SECOND_TIMELINE_FACT = "Continued the synthetic project from the approved context capsule." CONCLUSION = "The synthetic research loop is ready for product-level validation." SYSTEM_ENV_NAMES = { "COMSPEC", @@ -388,6 +389,15 @@ def status_arguments(*, provenance_root: Path) -> list[str]: ] +def resume_arguments(*, provenance_root: Path) -> list[str]: + return [ + "resume", + "--provenance-root", + str(provenance_root), + "--json", + ] + + def init_arguments( *, workspace: Path, @@ -422,7 +432,13 @@ def init_arguments( return arguments -def write_rollout(profile: Path, linked_repo: Path) -> Path: +def write_rollout( + profile: Path, + linked_repo: Path, + *, + session_id: str = "e2e-codex-session", + filename: str = "rollout-e2e-pull.jsonl", +) -> Path: rollout = ( profile / ".codex" @@ -430,7 +446,7 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: / "2026" / "07" / "16" - / "rollout-e2e-pull.jsonl" + / filename ) rollout.parent.mkdir(parents=True, exist_ok=True) now = datetime.now(timezone.utc) @@ -439,7 +455,7 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: "type": "session_meta", "timestamp": now.isoformat(), "payload": { - "session_id": "e2e-codex-session", + "session_id": session_id, "cwd": linked_repo.as_posix(), }, }, @@ -470,21 +486,31 @@ def write_rollout(profile: Path, linked_repo: Path) -> Path: def submit_fill( - summary_id: str, *, env: dict[str, str], provenance_home: Path + summary_id: str, + *, + env: dict[str, str], + provenance_home: Path, + timeline: list[str] | None = None, + include_high_value_claims: bool = True, ) -> None: fixture = { "schema_version": "summary-fill/1.0", - "timeline": [ + "timeline": timeline or [ TIMELINE_FACT, "Confirmed that preview, execution, and approval remain separate actions.", ], - "status": "active", - "stage": "product validation", - "next_actions": ["Review the isolated E2E evidence before release."], - "conclusion": CONCLUSION, - "blocked_by": "", - "confidence": "high", } + if include_high_value_claims: + fixture.update( + { + "status": "active", + "stage": "product validation", + "next_actions": ["Review the isolated E2E evidence before release."], + "conclusion": CONCLUSION, + "blocked_by": "", + "confidence": "high", + } + ) report = invoke_provenance_json( "provenance.synclayer.pending_fill", [summary_id, "--provenance-home", str(provenance_home), "--json"], @@ -1004,6 +1030,42 @@ def configured_pull(run: bool) -> list[str]: require(CONCLUSION in approved_text, "Approved conclusion did not reach the progress log") require(human_regions(note_after_approval) == baseline_human, "Approval changed human-owned prose") + before_first_resume = snapshot_tree(base) + first_resume = invoke_json( + resume_arguments(provenance_root=provenance_root), + env=env, + ) + require( + snapshot_tree(base) == before_first_resume, + "Context resume changed the isolated filesystem", + ) + first_capsule = first_resume.get("capsule", {}) + first_project = first_capsule.get("project", {}) + first_progress = [ + item + for block in first_capsule.get("recent_progress", []) + if isinstance(block, dict) + for item in block.get("items", []) + ] + require( + first_project.get("status") == "active" + and first_project.get("stage") == "product validation", + "First resumed session did not receive approved project state", + ) + require( + first_project.get("conclusion") == CONCLUSION, + "First resumed session did not receive the approved conclusion", + ) + require( + TIMELINE_FACT in first_progress, + "First resumed session did not receive prior low-risk progress", + ) + require_private_values_absent( + first_resume, + label="First resume capsule", + forbidden=report_private_values, + ) + approvals_after_approval = approvals_path.read_bytes() final = invoke_json( configured_pull(run=True), @@ -1018,6 +1080,120 @@ def configured_pull(run: bool) -> list[str]: require(approvals_path.read_bytes() == approvals_after_approval, "Final rerun changed Approvals.md") require(human_regions(project_note.read_bytes()) == baseline_human, "Final rerun changed human-owned prose") + second_session_id = "e2e-codex-session-two" + second_rollout = write_rollout( + profile, + agent_cwd, + session_id=second_session_id, + filename="rollout-e2e-pull-two.jsonl", + ) + second_private_values = [ + *report_private_values, + second_session_id, + second_rollout, + ] + second_run = invoke_json( + configured_pull(run=True), + env=env, + ) + require( + second_run.get("summary", {}).get("codex_enqueued") == 1, + "Second session was not enqueued exactly once", + ) + require( + second_run.get("summary", {}).get("scaffolded") == 1, + "Second session did not create one pending scaffold", + ) + require_private_values_absent( + second_run, + label="Second-session report", + forbidden=second_private_values, + ) + + second_scaffolds = read_pending_scaffolds( + env=env, provenance_home=provenance_home + ) + require( + len(second_scaffolds) == 1, + "Second session did not expose exactly one pending scaffold", + ) + second_summary_id = second_scaffolds[0].get("summary_id") + require( + isinstance(second_summary_id, str) and second_summary_id != summary_id, + "Second session did not receive a distinct stable summary id", + ) + submit_fill( + second_summary_id, + env=env, + provenance_home=provenance_home, + timeline=[SECOND_TIMELINE_FACT], + include_high_value_claims=False, + ) + second_applied = invoke_json( + configured_pull(run=True), + env=env, + ) + require( + second_applied.get("summary", {}).get("applied") == 1, + "Second session timeline was not applied", + ) + require( + second_applied.get("summary", {}).get("pending_approval") == 0, + "Timeline-only second session created a high-value approval", + ) + require_private_values_absent( + second_applied, + label="Second-session apply report", + forbidden=second_private_values, + ) + note_after_second = project_note.read_bytes() + require( + note_after_second.decode("utf-8").count(SECOND_TIMELINE_FACT) == 1, + "Second session timeline was not appended exactly once", + ) + require( + human_regions(note_after_second) == baseline_human, + "Second session changed human-owned prose", + ) + + before_second_resume = snapshot_tree(base) + second_resume = invoke_json( + resume_arguments(provenance_root=provenance_root), + env=env, + ) + require( + snapshot_tree(base) == before_second_resume, + "Second context resume changed the isolated filesystem", + ) + second_progress = [ + item + for block in second_resume.get("capsule", {}).get("recent_progress", []) + if isinstance(block, dict) + for item in block.get("items", []) + ] + require( + TIMELINE_FACT in second_progress and SECOND_TIMELINE_FACT in second_progress, + "Second resumed session could not recover both prior session increments", + ) + require_private_values_absent( + second_resume, + label="Second resume capsule", + forbidden=second_private_values, + ) + + second_idempotent = invoke_json( + configured_pull(run=True), + env=env, + ) + require( + second_idempotent.get("summary", {}).get("applied") == 0, + "Second-session idempotency rerun re-applied a summary", + ) + require( + project_note.read_bytes() == note_after_second, + "Second-session idempotency rerun changed the project note", + ) + before_final_status = snapshot_tree(base) final_status = invoke_json( status_arguments(provenance_root=provenance_root), diff --git a/tests/e2e_slides.py b/tests/e2e_slides.py new file mode 100644 index 0000000..175947b --- /dev/null +++ b/tests/e2e_slides.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +"""Synthetic, offline Steward -> Spec -> Lectern golden-path acceptance. + +The script launches each repository through a separate Python process, creates +only synthetic fixtures under a temporary directory, and deletes them at exit. +It never reads a user library, research directory, agent profile, or credential. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Any, Iterable + + +SCRIPTORIUM_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_STEWARD_ROOT = SCRIPTORIUM_ROOT.parent / "steward" +DEFAULT_SPEC_ROOT = SCRIPTORIUM_ROOT.parent / "scriptorium-spec" +DEFAULT_LECTERN_ROOT = SCRIPTORIUM_ROOT.parent / "Academic-Slides-Agent" +LECTERN_WORKER = Path(__file__).with_name("_e2e_lectern_worker.py") +SYSTEM_ENV_NAMES = { + "COMSPEC", + "LANG", + "LC_ALL", + "LC_CTYPE", + "NUMBER_OF_PROCESSORS", + "PATH", + "PATHEXT", + "PROCESSOR_ARCHITECTURE", + "SYSTEMDRIVE", + "SYSTEMROOT", + "TERM", + "WINDIR", +} +SYNTHETIC_PAPERS = ( + { + "key": "SYNTHA01", + "title": "[SYNTHETIC] Offline catalyst benchmark", + "authors": ["Synthetic Researcher A"], + "year": "2025", + "doi": "10.0000/scriptorium-golden-a", + "tldr": "A synthetic benchmark records catalyst signal A.", + "abstract": "No real research data is used in this generated fixture.", + "folders": ["Synthetic/GoldenPath"], + "marker": "SYNTHETIC-PAPER-1-EVIDENCE", + }, + { + "key": "SYNTHB02", + "title": "[SYNTHETIC] Offline catalyst replication", + "authors": ["Synthetic Researcher B"], + "year": "2026", + "doi": "10.0000/scriptorium-golden-b", + "tldr": "A synthetic replication records catalyst signal B.", + "abstract": "The values and narrative are fabricated for software testing.", + "folders": ["Synthetic/GoldenPath"], + "marker": "SYNTHETIC-PAPER-2-EVIDENCE", + }, +) +EMAIL_PATTERN = re.compile( + r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE +) +SECRET_PATTERN = re.compile( + r"\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9]{20,}|" + r"AKIA[0-9A-Z]{16})\b" +) +HOME_PATH_PATTERN = re.compile( + r"(?:[A-Z]:[\\/](?:Users|Documents and Settings)[\\/][^<>\"'\s]+|" + r"/(?:home|Users)/[^<>\"'\s]+)", + re.IGNORECASE, +) + + +class E2EFailure(RuntimeError): + """A cross-repository acceptance invariant failed.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise E2EFailure(message) + + +def _pdf_escape(value: str) -> str: + return value.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + + +def build_pdf(lines: Iterable[str]) -> bytes: + """Build a deterministic, one-page PDF with extractable Helvetica text.""" + rendered_lines = list(lines) + require(bool(rendered_lines), "A synthetic PDF needs at least one text line") + commands = ["BT", "/F1 10 Tf", "72 740 Td", "12 TL"] + for index, line in enumerate(rendered_lines): + if index: + commands.append("T*") + commands.append(f"({_pdf_escape(line)}) Tj") + commands.append("ET") + stream = ("\n".join(commands) + "\n").encode("ascii") + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ( + f"<< /Length {len(stream)} >>\nstream\n".encode("ascii") + + stream + + b"endstream" + ), + ] + + output = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") + offsets = [0] + for number, body in enumerate(objects, start=1): + offsets.append(len(output)) + output.extend(f"{number} 0 obj\n".encode("ascii")) + output.extend(body) + output.extend(b"\nendobj\n") + xref_offset = len(output) + output.extend(f"xref\n0 {len(objects) + 1}\n".encode("ascii")) + output.extend(b"0000000000 65535 f \n") + for offset in offsets[1:]: + output.extend(f"{offset:010d} 00000 n \n".encode("ascii")) + output.extend( + ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{xref_offset}\n%%EOF\n" + ).encode("ascii") + ) + return bytes(output) + + +def synthetic_pdf_lines(paper: dict[str, Any]) -> list[str]: + base = [ + str(paper["marker"]), + str(paper["title"]), + f"DOI {paper['doi']}", + "This document is fully synthetic and exists only for offline acceptance.", + "No private library, conversation, account, experiment, or credential is used.", + ] + base.extend( + f"Synthetic observation {index:02d}: catalyst signal remains reproducible " + "under a fabricated control condition." + for index in range(1, 14) + ) + return base + + +def isolated_environment(base: Path) -> dict[str, str]: + profile = base / "profile" + runtime_temp = base / "runtime-temp" + directories = { + "HOME": profile, + "USERPROFILE": profile, + "APPDATA": profile / "AppData" / "Roaming", + "LOCALAPPDATA": profile / "AppData" / "Local", + "TEMP": runtime_temp, + "TMP": runtime_temp, + "TMPDIR": runtime_temp, + "XDG_CACHE_HOME": profile / ".cache", + } + for directory in directories.values(): + directory.mkdir(parents=True, exist_ok=True) + env = { + name: os.environ[name] + for name in SYSTEM_ENV_NAMES + if name in os.environ + } + env.update({name: str(path) for name, path in directories.items()}) + env.update( + { + "ASA_PDF_PARSER": "pdfplumber", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONIOENCODING": "utf-8", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1", + } + ) + return env + + +def component_python(root: Path, *, require_venv: bool = False) -> Path: + candidates = ( + root / ".venv" / "Scripts" / "python.exe", + root / ".venv" / "bin" / "python", + ) + for candidate in candidates: + if candidate.is_file(): + return candidate + if require_venv: + raise E2EFailure( + "Lectern has no .venv; run `uv sync --all-packages --locked --no-dev`" + ) + return Path(sys.executable) + + +def run_command( + command: list[str], + *, + cwd: Path, + env: dict[str, str], + label: str, + timeout: int = 180, +) -> subprocess.CompletedProcess[str]: + try: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + shell=False, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired, UnicodeError) as exc: + raise E2EFailure(f"{label} could not execute") from exc + if completed.returncode != 0: + raise E2EFailure( + f"{label} failed with exit {completed.returncode}: " + f"stdout={completed.stdout!r} stderr={completed.stderr!r}" + ) + return completed + + +def read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError) as exc: + raise E2EFailure(f"Expected valid JSON artifact: {path.name}") from exc + require(isinstance(value, dict), f"Expected JSON object: {path.name}") + return value + + +def artifact_chunks(path: Path) -> Iterable[bytes]: + if path.suffix.lower() == ".pptx": + try: + with zipfile.ZipFile(path) as archive: + for name in sorted(archive.namelist()): + if not name.endswith("/"): + yield archive.read(name) + except (OSError, zipfile.BadZipFile) as exc: + raise E2EFailure(f"Generated PPTX is not a valid package: {path.name}") from exc + return + yield path.read_bytes() + + +def assert_public_artifacts_safe( + paths: Iterable[Path], *, forbidden_literals: Iterable[str | Path] +) -> None: + literals = [ + variant + for value in forbidden_literals + for variant in { + str(value), + str(value).replace("\\", "/"), + str(value).replace("/", "\\"), + } + if variant + ] + for path in paths: + for chunk in artifact_chunks(path): + for literal in literals: + if ( + literal.encode("utf-8", errors="ignore") in chunk + or literal.encode("utf-16le", errors="ignore") in chunk + ): + raise E2EFailure( + f"Transferable artifact exposed a local value: {path.name}" + ) + texts = ( + chunk.decode("utf-8", errors="ignore"), + chunk.decode("latin-1", errors="ignore"), + ) + for text in texts: + if "pdfPaths" in text: + raise E2EFailure( + f"Transferable artifact retained source paths: {path.name}" + ) + if EMAIL_PATTERN.search(text): + raise E2EFailure( + f"Transferable artifact contains an email: {path.name}" + ) + if SECRET_PATTERN.search(text): + raise E2EFailure( + f"Transferable artifact contains a credential pattern: {path.name}" + ) + if HOME_PATH_PATTERN.search(text): + raise E2EFailure( + f"Transferable artifact contains a home path: {path.name}" + ) + + +def create_library_kb(source_dir: Path) -> Path: + items: list[dict[str, Any]] = [] + for index, paper in enumerate(SYNTHETIC_PAPERS, start=1): + pdf_path = source_dir / f"synthetic-paper-{index}.pdf" + pdf_path.write_bytes(build_pdf(synthetic_pdf_lines(paper))) + item = { + key: value + for key, value in paper.items() + if key != "marker" + } + item["pdfPaths"] = [str(pdf_path)] + items.append(item) + kb_path = source_dir / "synthetic-library.json" + kb_path.write_text( + json.dumps( + {"schema_version": "library-kb/1.1", "items": items}, + ensure_ascii=True, + indent=2, + ), + encoding="utf-8", + ) + return kb_path + + +def validate_handoff(meta: dict[str, Any], handoff_dir: Path) -> None: + require(meta.get("schema_version") == "handoff/1.1", "Steward did not emit handoff/1.1") + require(meta.get("report_type") == "literature", "Handoff has the wrong report type") + require(meta.get("title") == "[SYNTHETIC] Offline evidence review", "Handoff title changed") + require( + isinstance(meta.get("key"), str) and len(meta["key"]) == 8, + "Handoff lacks an eight-character report key", + ) + papers = meta.get("papers") + require(isinstance(papers, list) and len(papers) == 2, "Handoff lost a paper") + for expected, actual in zip(SYNTHETIC_PAPERS, papers): + require(actual.get("title") == expected["title"], "Handoff changed a title") + require(actual.get("doi") == expected["doi"], "Handoff changed a DOI") + require("key" not in actual, "Handoff duplicated a per-paper key") + pdf_name = actual.get("pdfFilename") + require( + isinstance(pdf_name, str) and (handoff_dir / pdf_name).is_file(), + "Handoff omitted a staged PDF", + ) + + +def run_e2e(steward_root: Path, spec_root: Path, lectern_root: Path) -> dict[str, Any]: + for label, root in ( + ("Steward", steward_root), + ("Spec", spec_root), + ("Lectern", lectern_root), + ): + require(root.is_dir(), f"{label} checkout not found") + + with tempfile.TemporaryDirectory(prefix="scriptorium-slides-golden-") as raw_base: + base = Path(raw_base) + source_dir = base / "synthetic-input" + staging_dir = base / "handoff" + output_dir = base / "lectern-output" + source_dir.mkdir() + staging_dir.mkdir() + output_dir.mkdir() + env = isolated_environment(base) + kb_path = create_library_kb(source_dir) + + steward_python = component_python(steward_root) + run_command( + [ + str(steward_python), + "-m", + "steward.cli", + "pick", + *(str(paper["key"]) for paper in SYNTHETIC_PAPERS), + "--kb", + str(kb_path), + "--staging", + str(staging_dir), + "--report-type", + "literature", + "--report-title", + "[SYNTHETIC] Offline evidence review", + ], + cwd=steward_root, + env=env, + label="Steward pick", + ) + handoff_dirs = [path for path in staging_dir.iterdir() if path.is_dir()] + require(len(handoff_dirs) == 1, "Steward did not create exactly one handoff") + handoff_dir = handoff_dirs[0] + meta_path = handoff_dir / "meta.json" + meta = read_json(meta_path) + validate_handoff(meta, handoff_dir) + + spec_python = component_python(spec_root) + validated = run_command( + [str(spec_python), str(spec_root / "tools" / "validate.py"), str(meta_path)], + cwd=spec_root, + env=env, + label="Spec validation", + ) + require("ok" in validated.stdout.lower(), "Spec validator did not accept metadata") + + lectern_python = component_python(lectern_root, require_venv=True) + worker = run_command( + [ + str(lectern_python), + str(LECTERN_WORKER), + "--handoff", + str(handoff_dir), + "--output-dir", + str(output_dir), + ], + cwd=lectern_root, + env=env, + label="Lectern golden path", + ) + try: + report = json.loads(worker.stdout) + except json.JSONDecodeError as exc: + raise E2EFailure("Lectern worker did not emit one JSON object") from exc + require(isinstance(report, dict), "Lectern worker report is not an object") + require(report.get("status") == "passed", "Lectern worker did not pass") + require( + report.get("production_graph_exercised") is True, + "Lectern production-graph proof is absent", + ) + require(report.get("hard_stop_blocked_pptx") is True, "Hard-stop proof is absent") + require(report.get("editable_after_reopen") is True, "PPTX editability proof is absent") + + transferable = [ + path + for root in (handoff_dir, output_dir) + for path in root.rglob("*") + if path.is_file() + ] + generated_pptx = list(output_dir.rglob("*.pptx")) + require(len(generated_pptx) == 2, "Expected approved and edited PPTX outputs") + forbidden = [ + base, + steward_root, + spec_root, + lectern_root, + Path.home(), + os.environ.get("HOME", ""), + os.environ.get("USERPROFILE", ""), + ] + assert_public_artifacts_safe(transferable, forbidden_literals=forbidden) + return report + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steward-root", type=Path, default=DEFAULT_STEWARD_ROOT) + parser.add_argument("--spec-root", type=Path, default=DEFAULT_SPEC_ROOT) + parser.add_argument("--lectern-root", type=Path, default=DEFAULT_LECTERN_ROOT) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + report = run_e2e( + args.steward_root.resolve(), + args.spec_root.resolve(), + args.lectern_root.resolve(), + ) + except E2EFailure as exc: + print(f"FAIL: {exc}", file=sys.stderr) + return 1 + print( + "PASS: synthetic Steward handoff, Spec validation, Lectern evidence, " + f"approval hard-stop, and editable PPTX ({report['approved_slide_count']} slides)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cli_migration.py b/tests/test_cli_migration.py new file mode 100644 index 0000000..5306966 --- /dev/null +++ b/tests/test_cli_migration.py @@ -0,0 +1,370 @@ +from __future__ import annotations + +import contextlib +import io +import json +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from scriptorium import cli +from scriptorium.migration import MigrationError + + +def aggregate_report(operation: str, status: str) -> dict[str, object]: + return { + "schema_version": "migration-report/1.0", + "operation": operation, + "status": status, + "summary": { + "sources_requested": 1, + "files": 2, + "markdown": 1, + "pdf": 1, + "bytes": 24, + "changed": 0, + "unchanged": 0, + }, + "limitations": [], + } + + +class MigrationCliTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.workspace = self.root / "research workspace" + self.sources = self.root / "selected sources" + self.platform_state = self.root / "platform state" + self.workspace.mkdir() + self.sources.mkdir() + self.environment = mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(self.platform_state), + "XDG_STATE_HOME": str(self.platform_state), + }, + clear=False, + ) + self.environment.start() + self.addCleanup(self.environment.stop) + + def invoke( + self, arguments: list[str] + ) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = cli.main(arguments) + return exit_code, stdout.getvalue(), stderr.getvalue() + + def invoke_json( + self, arguments: list[str] + ) -> tuple[int, dict[str, object], str]: + exit_code, stdout, stderr = self.invoke(arguments) + return exit_code, json.loads(stdout), stderr + + def test_plan_forwards_repeated_explicit_sources(self): + first = self.sources / "first.md" + second = self.sources / "second.pdf" + expected = SimpleNamespace(report=aggregate_report("plan", "planned")) + with mock.patch( + "scriptorium.cli.plan_migration", return_value=expected + ) as plan: + exit_code, report, stderr = self.invoke_json( + [ + "migrate", + "plan", + "--source", + str(first), + "--source", + str(second), + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(stderr, "") + self.assertEqual(report, expected.report) + plan.assert_called_once_with( + [first, second], + workspace=self.workspace, + batch_id="batch-001", + ) + + def test_apply_uses_sources_for_first_run_and_identity_for_reapply(self): + source = self.sources / "idea.md" + planned = object() + applied = SimpleNamespace(report=aggregate_report("apply", "applied")) + with ( + mock.patch( + "scriptorium.cli.plan_migration", return_value=planned + ) as plan, + mock.patch( + "scriptorium.cli.apply_migration", return_value=applied + ) as apply, + mock.patch("scriptorium.cli.reapply_migration") as reapply, + ): + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "apply", + "--source", + str(source), + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["status"], "applied") + plan.assert_called_once_with( + [source], workspace=self.workspace, batch_id="batch-001" + ) + apply.assert_called_once_with(planned) + reapply.assert_not_called() + + unchanged = SimpleNamespace( + report=aggregate_report("apply", "unchanged") + ) + with ( + mock.patch("scriptorium.cli.plan_migration") as plan, + mock.patch("scriptorium.cli.apply_migration") as apply, + mock.patch( + "scriptorium.cli.reapply_migration", + return_value=unchanged, + ) as reapply, + ): + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "apply", + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["status"], "unchanged") + reapply.assert_called_once_with( + workspace=self.workspace, batch_id="batch-001" + ) + plan.assert_not_called() + apply.assert_not_called() + + def test_verify_and_rollback_need_only_workspace_and_batch(self): + verified = SimpleNamespace( + report=aggregate_report("verify", "applied") + ) + with mock.patch( + "scriptorium.cli.verify_migration", return_value=verified + ) as verify: + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "verify", + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["operation"], "verify") + verify.assert_called_once_with( + workspace=self.workspace, batch_id="batch-001" + ) + + loaded = object() + rolled_back = SimpleNamespace( + report=aggregate_report("rollback", "rolled-back") + ) + with ( + mock.patch( + "scriptorium.cli.load_migration", return_value=loaded + ) as load, + mock.patch( + "scriptorium.cli.rollback_migration", + return_value=rolled_back, + ) as rollback, + ): + exit_code, report, _stderr = self.invoke_json( + [ + "migrate", + "rollback", + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 0) + self.assertEqual(report["status"], "rolled-back") + load.assert_called_once_with( + workspace=self.workspace, batch_id="batch-001" + ) + rollback.assert_called_once_with(loaded) + + def test_real_cli_lifecycle_is_path_free_and_idempotent(self): + markdown = self.sources / "synthetic idea.md" + pdf = self.sources / "synthetic paper.pdf" + markdown.write_text("# Synthetic\n", encoding="utf-8") + pdf.write_bytes(b"%PDF-1.4\nsynthetic\n") + common = [ + "--workspace", + str(self.workspace), + "--batch-id", + "synthetic-batch", + "--json", + ] + + human_code, human_output, human_stderr = self.invoke( + [ + "migrate", + "plan", + "--source", + str(self.sources), + "--workspace", + str(self.workspace), + "--batch-id", + "synthetic-batch", + ] + ) + plan_code, planned, plan_stderr = self.invoke_json( + [ + "migrate", + "plan", + "--source", + str(self.sources), + *common, + ] + ) + apply_code, applied, apply_stderr = self.invoke_json( + [ + "migrate", + "apply", + "--source", + str(self.sources), + *common, + ] + ) + verify_code, verified, verify_stderr = self.invoke_json( + ["migrate", "verify", *common] + ) + repeat_code, repeated, repeat_stderr = self.invoke_json( + ["migrate", "apply", *common] + ) + rollback_code, rolled_back, rollback_stderr = self.invoke_json( + ["migrate", "rollback", *common] + ) + + self.assertEqual( + (plan_code, apply_code, verify_code, repeat_code, rollback_code), + (0, 0, 0, 0, 0), + ) + self.assertEqual(human_code, 0) + self.assertEqual(human_stderr, "") + self.assertIn("Migration plan: planned", human_output) + self.assertNotIn(str(self.root), human_output) + self.assertNotIn(markdown.name, human_output) + self.assertEqual( + ( + planned["status"], + applied["status"], + verified["status"], + repeated["status"], + rolled_back["status"], + ), + ("planned", "applied", "applied", "unchanged", "rolled-back"), + ) + self.assertEqual( + plan_stderr + + apply_stderr + + verify_stderr + + repeat_stderr + + rollback_stderr, + "", + ) + serialized = json.dumps( + [planned, applied, verified, repeated, rolled_back], + ensure_ascii=False, + ) + self.assertNotIn(str(self.root), serialized) + self.assertNotIn(markdown.name, serialized) + self.assertEqual(markdown.read_text(encoding="utf-8"), "# Synthetic\n") + self.assertEqual(pdf.read_bytes(), b"%PDF-1.4\nsynthetic\n") + + def test_usage_and_runtime_errors_never_echo_private_arguments(self): + sentinel = str(self.root / "private research") + arguments = [ + "migrate", + "plan", + "--source", + sentinel, + "--workspace", + sentinel, + "--batch-id", + "batch-001", + "--unknown", + sentinel, + ] + exit_code, report, stderr = self.invoke_json(arguments + ["--json"]) + self.assertEqual(exit_code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["operation"], "plan") + self.assertEqual(report["errors"], [{"code": "entry_error"}]) + self.assertNotIn(sentinel, json.dumps(report)) + + exit_code, stdout, stderr = self.invoke(arguments) + self.assertEqual(exit_code, 2) + self.assertEqual(stdout, "") + self.assertIn("invalid migrate invocation", stderr) + self.assertNotIn(sentinel, stderr) + + failures = ( + (MigrationError("target_exists"), "target_exists"), + (MigrationError(sentinel), "entry_error"), + (OSError(sentinel), "entry_error"), + ) + for failure, code in failures: + with ( + self.subTest(code=code), + mock.patch( + "scriptorium.cli.plan_migration", + side_effect=failure, + ), + ): + exit_code, report, stderr = self.invoke_json( + [ + "migrate", + "plan", + "--source", + sentinel, + "--workspace", + str(self.workspace), + "--batch-id", + "batch-001", + "--json", + ] + ) + self.assertEqual(exit_code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["errors"], [{"code": code}]) + self.assertNotIn(sentinel, json.dumps(report)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_demo.py b/tests/test_demo.py index b1c64ea..1efde90 100644 --- a/tests/test_demo.py +++ b/tests/test_demo.py @@ -39,9 +39,9 @@ def test_compatibility_manifest_locks_alpha_baseline(self): self.assertEqual( load_compatibility(), { - "scriptorium-spec": "2.2.0", + "scriptorium-spec": "2.3.0", "steward": "0.2.0", - "provenance": "0.17.0", + "provenance": "0.18.0", }, ) @@ -54,6 +54,31 @@ def test_demo_library_fixture_is_explicitly_synthetic_and_versioned(self): self.assertEqual(data["generated_by"], "scriptorium demo fixture") self.assertEqual([item["key"] for item in data["items"]], ["DEMO0001", "DEMO0002", "DEMO0003"]) + def test_demo_research_fixtures_are_synthetic_and_share_stable_citekeys(self): + expected = { + "parsed-paper.v1.json": "parsed-paper/1.0", + "reading-note.v1.json": "reading-note/1.0", + "review.v1.json": "review/1.0", + "lineage-graph.v1.json": "lineage-graph/1.0", + } + fixtures = {} + for name, schema_version in expected.items(): + payload = resources.files("scriptorium.assets").joinpath(name).read_text( + encoding="utf-8" + ) + self.assertIn("[SYNTHETIC]", payload) + self.assertNotRegex(payload, r"(?i)(?<![A-Z0-9])[A-Z]:[\\/]") + fixtures[name] = json.loads(payload) + self.assertEqual(fixtures[name]["schema_version"], schema_version) + + self.assertEqual(fixtures["parsed-paper.v1.json"]["id"], "liu2024ActiveLearning") + self.assertEqual(fixtures["reading-note.v1.json"]["id"], "liu2024ActiveLearning") + review_rows = fixtures["review.v1.json"]["comparison_table"]["rows"] + lineage_nodes = fixtures["lineage-graph.v1.json"]["nodes"] + expected_citekeys = {"chen2022PhysicsInformed", "liu2024ActiveLearning"} + self.assertEqual({row["citekey"] for row in review_rows}, expected_citekeys) + self.assertEqual({node["citekey"] for node in lineage_nodes}, expected_citekeys) + def test_prepare_output_reuses_only_owned_directory(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) / "demo" diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 63e9ff0..27e56ab 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -11,6 +11,7 @@ from scriptorium import cli from scriptorium.doctor import ( DoctorError, + PROVENANCE_BASE_COMMANDS, TARGETS, _agent_host_check, _browser_extension_check, @@ -19,6 +20,7 @@ _find_application, _host_adapter_check, _agent_capture_check, + _entry_context_check, _entry_pull_check, _project_metadata, _provenance_home_check, @@ -242,6 +244,9 @@ def test_human_output_is_stable_plain_text_with_fix(self): class DoctorProbeTests(unittest.TestCase): + def test_public_alpha_baseline_requires_research_artifact_ingest(self): + self.assertIn("prov-ingest-research", PROVENANCE_BASE_COMMANDS) + def test_invalid_lectern_environment_root_fails_closed(self): with tempfile.TemporaryDirectory() as temporary: missing = Path(temporary) / "missing-lectern" @@ -705,6 +710,52 @@ def locate(_root: Path, name: str) -> Path: self.assertIn("different Provenance environment", check["details"]["error"]) probe.assert_not_called() + def test_context_entry_requires_matching_runtime_version(self): + with mock.patch( + "scriptorium.doctor.find_script", + side_effect=lambda _root, name: Path("shared-environment") / name, + ), mock.patch( + "scriptorium.doctor._run_probe", return_value="0.17.0\n" + ) as probe: + ready = _entry_context_check( + "public-alpha", Path("Provenance"), "0.17.0" + ) + mismatch = _entry_context_check( + "public-alpha", Path("Provenance"), "0.18.0" + ) + + self.assertEqual(ready["status"], "pass") + self.assertEqual(ready["details"]["runtime_version"], "0.17.0") + self.assertEqual(mismatch["status"], "fail") + self.assertIn("runtime version", mismatch["details"]["error"]) + self.assertEqual(probe.call_args.args[0][-1], "--version") + + def test_context_entry_is_required_only_for_public_alpha(self): + public = _entry_context_check("public-alpha", None, "0.17.0") + demo = _entry_context_check("demo", None, "0.17.0") + self.assertEqual(public["status"], "fail") + self.assertEqual(demo["status"], "info") + + def test_context_command_must_share_the_provenance_environment(self): + def locate(_root: Path, name: str) -> Path: + directory = ( + "context-environment" + if name == "prov-context" + else "base-environment" + ) + return Path(directory) / name + + with mock.patch( + "scriptorium.doctor.find_script", side_effect=locate + ), mock.patch("scriptorium.doctor._run_probe") as probe: + check = _entry_context_check( + "public-alpha", Path("Provenance"), "0.17.0" + ) + + self.assertEqual(check["status"], "fail") + self.assertIn("different Provenance environment", check["details"]["error"]) + probe.assert_not_called() + def test_invalid_compatibility_shape_is_a_doctor_error(self): with mock.patch( "scriptorium.doctor.load_compatibility", side_effect=AttributeError("items") diff --git a/tests/test_e2e_slides.py b/tests/test_e2e_slides.py new file mode 100644 index 0000000..05307ca --- /dev/null +++ b/tests/test_e2e_slides.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import importlib.util +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +MODULE_PATH = Path(__file__).with_name("e2e_slides.py") +MODULE_SPEC = importlib.util.spec_from_file_location("e2e_slides", MODULE_PATH) +assert MODULE_SPEC and MODULE_SPEC.loader +e2e_slides = importlib.util.module_from_spec(MODULE_SPEC) +MODULE_SPEC.loader.exec_module(e2e_slides) + + +class SyntheticSlidesE2ETest(unittest.TestCase): + def test_pdf_has_valid_offsets_and_extractable_markers(self) -> None: + payload = e2e_slides.build_pdf( + ["SYNTHETIC-PAPER-1-EVIDENCE", r"escaped (value) \ test"] + ) + self.assertTrue(payload.startswith(b"%PDF-1.4")) + self.assertTrue(payload.endswith(b"%%EOF\n")) + self.assertIn(b"SYNTHETIC-PAPER-1-EVIDENCE", payload) + xref = payload.index(b"xref\n") + lines = payload[xref:].splitlines() + self.assertEqual(lines[1], b"0 6") + for object_number, row in enumerate(lines[3:8], start=1): + offset = int(row[:10]) + self.assertTrue( + payload[offset:].startswith(f"{object_number} 0 obj\n".encode("ascii")) + ) + + def test_isolated_environment_drops_credentials(self) -> None: + with tempfile.TemporaryDirectory() as raw: + with patch.dict( + os.environ, + { + "PATH": os.environ.get("PATH", ""), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + "OPENAI_API_KEY": "sk-not-allowed-in-child", + "ANTHROPIC_API_KEY": "also-not-allowed", + "GITHUB_TOKEN": "ghp_not_allowed_in_child", + }, + clear=True, + ): + env = e2e_slides.isolated_environment(Path(raw)) + self.assertNotIn("OPENAI_API_KEY", env) + self.assertNotIn("ANTHROPIC_API_KEY", env) + self.assertNotIn("GITHUB_TOKEN", env) + self.assertEqual(env["ASA_PDF_PARSER"], "pdfplumber") + self.assertEqual(env["PYTHONNOUSERSITE"], "1") + + def test_privacy_scan_accepts_synthetic_payload(self) -> None: + with tempfile.TemporaryDirectory() as raw: + artifact = Path(raw) / "meta.json" + artifact.write_text( + '{"schema_version":"handoff/1.1","title":"[SYNTHETIC] Safe"}', + encoding="utf-8", + ) + e2e_slides.assert_public_artifacts_safe( + [artifact], forbidden_literals=["C:/Users/private"] + ) + + def test_privacy_scan_rejects_paths_email_and_secrets(self) -> None: + unsafe_values = ( + "C:/Users/private/research.pdf", + "researcher@example.org", + "sk-0123456789ABCDEFGHIJ", + '"pdfPaths":["synthetic.pdf"]', + ) + for index, value in enumerate(unsafe_values): + with self.subTest(value=value): + with tempfile.TemporaryDirectory() as raw: + artifact = Path(raw) / f"unsafe-{index}.txt" + artifact.write_text(value, encoding="utf-8") + with self.assertRaises(e2e_slides.E2EFailure): + e2e_slides.assert_public_artifacts_safe( + [artifact], + forbidden_literals=["C:/Users/private"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_install_lifecycle.py b/tests/test_install_lifecycle.py new file mode 100644 index 0000000..f71fb4e --- /dev/null +++ b/tests/test_install_lifecycle.py @@ -0,0 +1,120 @@ +import importlib.util +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT = Path(__file__).with_name("e2e_install_lifecycle.py") +SPEC = importlib.util.spec_from_file_location("e2e_install_lifecycle", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +LIFECYCLE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(LIFECYCLE) + + +class IdentityTests(unittest.TestCase): + def test_reads_expected_identity(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "pyproject.toml").write_text( + '[project]\nname="scriptorium-suite"\nversion="0.4.0"\n', + encoding="utf-8", + ) + + self.assertEqual( + LIFECYCLE.project_identity(root, "scriptorium-suite"), + {"name": "scriptorium-suite", "version": "0.4.0"}, + ) + + def test_rejects_wrong_distribution(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "pyproject.toml").write_text( + '[project]\nname="other"\nversion="0.4.0"\n', + encoding="utf-8", + ) + + with self.assertRaises(LIFECYCLE.LifecycleFailure): + LIFECYCLE.project_identity(root, "scriptorium-suite") + + +class IsolationTests(unittest.TestCase): + def test_environment_drops_provider_and_index_credentials(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + scripts = root / "clean-venv" / "Scripts" + with mock.patch.dict( + os.environ, + { + "PATH": "system-path", + "OPENAI_API_KEY": "private", + "PIP_INDEX_URL": "https://user:secret@example.invalid/simple", + }, + clear=True, + ): + env = LIFECYCLE.isolated_environment(root, scripts) + + self.assertNotIn("OPENAI_API_KEY", env) + self.assertNotIn("PIP_INDEX_URL", env) + self.assertEqual(env["PIP_NO_INDEX"], "1") + self.assertEqual(env["PIP_NO_INPUT"], "1") + self.assertTrue(env["PATH"].startswith(str(scripts))) + + def test_staging_excludes_unselected_private_directories(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + source = root / "source" + package = source / "src" / "scriptorium" + package.mkdir(parents=True) + (source / "pyproject.toml").write_text( + '[project]\nname="scriptorium-suite"\nversion="0.4.0"\n', + encoding="utf-8", + ) + (source / "README.md").write_text("Public", encoding="utf-8") + (package / "__init__.py").write_text("__version__='0.4.0'\n", encoding="utf-8") + private = source / "private-memory" + private.mkdir() + (private / "session.jsonl").write_text("private", encoding="utf-8") + + staged = LIFECYCLE.stage_package( + source, + root / "staged", + "scriptorium-suite", + ) + + self.assertTrue((staged / "src" / "scriptorium" / "__init__.py").is_file()) + self.assertFalse((staged / "private-memory").exists()) + + +class TransitionTests(unittest.TestCase): + def test_only_older_release_pair_is_covered(self): + self.assertEqual( + LIFECYCLE.release_relation("0.3.0", "0.4.0"), + "previous-is-older", + ) + self.assertEqual(LIFECYCLE.release_relation("0.4.0", "0.4.0"), "same") + self.assertEqual( + LIFECYCLE.release_relation("0.5.0", "0.4.0"), + "previous-is-newer", + ) + self.assertEqual(LIFECYCLE.release_relation("local", "0.4.0"), "unsupported") + + +class ReportTests(unittest.TestCase): + def test_report_is_created_once_and_not_replaced(self): + with tempfile.TemporaryDirectory() as raw: + path = Path(raw) / "report.json" + payload = {"status": "passed"} + + LIFECYCLE.write_report(path, payload) + + self.assertEqual(json.loads(path.read_text(encoding="utf-8")), payload) + with self.assertRaises(LIFECYCLE.LifecycleFailure): + LIFECYCLE.write_report(path, {"status": "changed"}) + self.assertEqual(json.loads(path.read_text(encoding="utf-8")), payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_migration.py b/tests/test_migration.py new file mode 100644 index 0000000..8a017a9 --- /dev/null +++ b/tests/test_migration.py @@ -0,0 +1,940 @@ +import copy +import errno +import json +import os +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from scriptorium import migration as migration_module +from scriptorium.migration import ( + MANIFEST_PRIVACY, + MANIFEST_VERSION, + MigrationError, + apply_migration, + plan_migration, + rollback_migration, +) + + +def snapshot(root: Path) -> dict[str, bytes | None]: + result: dict[str, bytes | None] = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + result[relative] = path.read_bytes() if path.is_file() else None + return result + + +class MigrationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.base = Path(self.temporary.name).resolve() + self.workspace = self.base / "research workspace" + self.sources = self.base / "selected sources" + self.platform_state = self.base / "platform state" + self.environment = mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(self.platform_state), + "XDG_STATE_HOME": str(self.platform_state), + }, + clear=False, + ) + self.environment.start() + self.state_root = ( + self.platform_state / "Scriptorium" / "state" + if os.name == "nt" + else self.platform_state / "scriptorium" + ) + self.workspace.mkdir() + self.sources.mkdir() + + def tearDown(self): + self.environment.stop() + self.temporary.cleanup() + + def write_sources(self): + (self.sources / "notes").mkdir() + markdown = self.sources / "notes" / "idea.md" + pdf = self.sources / "paper.pdf" + ignored = self.sources / "raw.txt" + markdown.write_text("# Synthetic idea\n", encoding="utf-8") + pdf.write_bytes(b"%PDF-1.4\nsynthetic\n") + ignored.write_text("not selected by suffix", encoding="utf-8") + return markdown, pdf, ignored + + def plan(self, sources=None, *, batch_id="batch-001", state_root=None): + return plan_migration( + sources if sources is not None else [self.sources], + workspace=self.workspace, + batch_id=batch_id, + state_root=self.state_root if state_root is None else state_root, + ) + + def target(self, plan, index=0) -> Path: + return Path(plan.manifest["entries"][index]["target"]) + + def manifest_path(self, plan) -> Path: + return migration_module._manifest_path(plan.manifest) + + def test_plan_is_read_only_private_and_repr_is_path_free(self): + self.write_sources() + workspace_before = snapshot(self.workspace) + sources_before = snapshot(self.sources) + + planned = self.plan() + + self.assertEqual(snapshot(self.workspace), workspace_before) + self.assertEqual(snapshot(self.sources), sources_before) + self.assertFalse(self.state_root.exists()) + self.assertEqual(planned.report["status"], "planned") + self.assertEqual(planned.report["summary"]["files"], 2) + self.assertEqual(planned.report["summary"]["markdown"], 1) + self.assertEqual(planned.report["summary"]["pdf"], 1) + self.assertNotIn(str(self.base), json.dumps(planned.report)) + self.assertNotIn(str(self.base), repr(planned)) + self.assertEqual(planned.manifest["schema_version"], MANIFEST_VERSION) + self.assertEqual(planned.manifest["privacy"], MANIFEST_PRIVACY) + self.assertEqual(Path(planned.manifest["state_root"]), self.state_root) + + def test_default_state_root_uses_platform_local_state_outside_workspace(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + local_state = self.base / "platform state" + environment = { + "LOCALAPPDATA": str(local_state), + "XDG_STATE_HOME": str(local_state), + } + with mock.patch.dict(os.environ, environment, clear=False): + planned = plan_migration( + [source], workspace=self.workspace, batch_id="default-state" + ) + expected = ( + local_state / "Scriptorium" / "state" + if os.name == "nt" + else local_state / "scriptorium" + ) + self.assertEqual(Path(planned.manifest["state_root"]), expected) + self.assertFalse(expected.exists()) + + def test_noncanonical_explicit_state_root_is_rejected_without_path(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([source], state_root=self.workspace / ".private") + self.assertEqual(raised.exception.code, "noncanonical_state_root") + self.assertNotIn(str(self.base), str(raised.exception)) + + def test_canonical_state_root_inside_workspace_is_rejected(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + local_state = self.workspace / "platform state" + with mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(local_state), + "XDG_STATE_HOME": str(local_state), + }, + clear=False, + ): + with self.assertRaises(MigrationError) as raised: + plan_migration( + [source], workspace=self.workspace, batch_id="overlap" + ) + self.assertEqual(raised.exception.code, "state_workspace_overlap") + + def test_canonical_state_root_inside_source_root_is_rejected(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + with mock.patch.dict( + os.environ, + { + "LOCALAPPDATA": str(self.sources), + "XDG_STATE_HOME": str(self.sources), + }, + clear=False, + ): + with self.assertRaises(MigrationError) as raised: + plan_migration( + [self.sources], + workspace=self.workspace, + batch_id="source-overlap", + ) + self.assertEqual(raised.exception.code, "state_root_in_source") + + def test_source_directory_inside_canonical_state_root_is_rejected(self): + source = self.state_root / "candidate inputs" + source.mkdir(parents=True) + (source / "idea.md").write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([source]) + self.assertEqual(raised.exception.code, "source_state_overlap") + self.assertFalse((self.workspace / "Sources").exists()) + + def test_source_file_inside_canonical_state_root_is_rejected(self): + source = self.state_root / "candidate.md" + source.parent.mkdir(parents=True) + source.write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([source]) + self.assertEqual(raised.exception.code, "source_state_overlap") + self.assertFalse((self.workspace / "Sources").exists()) + + def test_apply_copies_supported_files_and_keeps_manifest_outside_workspace(self): + self.write_sources() + sources_before = snapshot(self.sources) + applied = apply_migration(self.plan()) + + self.assertEqual(applied.report["status"], "applied") + self.assertEqual(applied.report["summary"]["changed"], 2) + self.assertEqual(snapshot(self.sources), sources_before) + for entry in applied.manifest["entries"]: + self.assertEqual( + Path(entry["target"]).read_bytes(), Path(entry["source"]).read_bytes() + ) + self.assertEqual(entry["state"], "created") + self.assertEqual( + entry["file_identity"], + migration_module._file_identity(Path(entry["target"])), + ) + self.assertTrue(self.manifest_path(applied).is_file()) + self.assertFalse((self.workspace / ".scriptorium").exists()) + self.assertNotIn(str(self.base), json.dumps(applied.report)) + self.assertNotIn(str(self.base), repr(applied)) + + def test_publication_uses_same_directory_hard_link_create(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + target = self.target(planned) + + with mock.patch( + "scriptorium.migration.os.link", wraps=os.link + ) as link: + apply_migration(planned) + + self.assertEqual(link.call_count, 1) + stage, published = map(Path, link.call_args.args) + self.assertEqual(published, target) + self.assertEqual(stage.parent, target.parent) + self.assertTrue(stage.is_file()) + self.assertTrue(os.path.samefile(stage, target)) + + def test_existing_target_is_never_adopted_or_overwritten(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + destination = self.target(planned) + destination.parent.mkdir(parents=True) + destination.write_text("user owned", encoding="utf-8") + + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "target_exists") + self.assertEqual(destination.read_text(encoding="utf-8"), "user owned") + self.assertFalse(self.manifest_path(planned).exists()) + + def test_hard_link_failure_is_safe_and_retryable(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + target = self.target(planned) + + with mock.patch( + "scriptorium.migration.os.link", + side_effect=OSError(errno.EPERM, "synthetic"), + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "atomic_publish_unavailable") + self.assertFalse(target.exists()) + recovered = apply_migration(planned) + self.assertEqual(recovered.report["status"], "applied") + self.assertEqual(target.read_text(encoding="utf-8"), "synthetic") + + def test_cross_volume_publish_error_has_no_overwrite_fallback(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + planned = self.plan([source]) + + with mock.patch( + "scriptorium.migration.os.link", + side_effect=OSError(errno.EXDEV, "synthetic"), + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual( + raised.exception.code, "cross_volume_publish_unsupported" + ) + self.assertFalse(self.target(planned).exists()) + + def test_source_change_is_rejected_before_target_write(self): + markdown, _, _ = self.write_sources() + planned = self.plan() + markdown.write_text("changed after preview\n", encoding="utf-8") + + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "source_hash_changed") + self.assertFalse((self.workspace / "Sources").exists()) + self.assertFalse(self.manifest_path(planned).exists()) + + def test_repeat_apply_stays_idempotent_when_sources_are_gone(self): + self.write_sources() + planned = self.plan() + first = apply_migration(planned) + target_mtimes = { + Path(entry["target"]): Path(entry["target"]).stat().st_mtime_ns + for entry in first.manifest["entries"] + } + manifest_path = self.manifest_path(first) + manifest_mtime = manifest_path.stat().st_mtime_ns + for path in sorted(self.sources.rglob("*"), reverse=True): + if path.is_file(): + path.unlink() + else: + path.rmdir() + + second = apply_migration(planned) + + self.assertEqual(second.report["status"], "unchanged") + self.assertEqual(second.report["summary"]["changed"], 0) + self.assertEqual(second.report["summary"]["unchanged"], 2) + self.assertEqual(manifest_path.stat().st_mtime_ns, manifest_mtime) + self.assertEqual( + {path: path.stat().st_mtime_ns for path in target_mtimes}, target_mtimes + ) + + def test_new_process_can_load_verify_reapply_and_rollback_without_sources(self): + self.write_sources() + planned = self.plan(batch_id="process-recovery") + applied = apply_migration(planned) + targets = [Path(entry["target"]) for entry in applied.manifest["entries"]] + for path in sorted(self.sources.rglob("*"), reverse=True): + if path.is_file(): + path.unlink() + else: + path.rmdir() + + source_root = Path(__file__).resolve().parents[1] / "src" + code = """ +import json +import sys +from scriptorium.migration import ( + load_migration, + reapply_migration, + rollback_migration, + verify_migration, +) +workspace, batch_id = sys.argv[1:3] +loaded = load_migration(workspace=workspace, batch_id=batch_id) +verified = verify_migration(workspace=workspace, batch_id=batch_id) +reapplied = reapply_migration(workspace=workspace, batch_id=batch_id) +rolled_back = rollback_migration( + load_migration(workspace=workspace, batch_id=batch_id) +) +print(json.dumps({ + "load": loaded.report["status"], + "verify": verified.report["status"], + "reapply": reapplied.report["status"], + "rollback": rolled_back.report["status"], +})) +""" + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + [str(source_root), environment.get("PYTHONPATH", "")] + ) + completed = subprocess.run( + [ + sys.executable, + "-c", + code, + str(self.workspace), + "process-recovery", + ], + env=environment, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual( + json.loads(completed.stdout), + { + "load": "applied", + "verify": "applied", + "reapply": "unchanged", + "rollback": "rolled-back", + }, + ) + self.assertNotIn(str(self.base), completed.stdout) + self.assertNotIn(str(self.base), completed.stderr) + self.assertTrue(all(not target.exists() for target in targets)) + + def test_runtime_state_tampering_is_covered_by_integrity(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + manifest = copy.deepcopy(self.plan([source]).manifest) + manifest["run_state"] = "applied" + + with self.assertRaises(MigrationError) as raised: + apply_migration(manifest) + + self.assertEqual(raised.exception.code, "manifest_integrity_failed") + + def test_invalid_but_resealed_state_combination_is_rejected(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + manifest = copy.deepcopy(self.plan([source]).manifest) + manifest["run_state"] = "applied" + migration_module._seal_manifest(manifest) + + with self.assertRaises(MigrationError) as raised: + apply_migration(manifest) + + self.assertEqual(raised.exception.code, "invalid_state_transition") + + def test_tampered_escape_is_rejected_even_when_digests_are_recomputed(self): + source = self.sources / "idea.md" + source.write_text("synthetic", encoding="utf-8") + manifest = copy.deepcopy(self.plan([source]).manifest) + manifest["entries"][0]["relative_target"] = "../outside.md" + manifest["entries"][0]["target"] = str(self.base / "outside.md") + migration_module._seal_manifest(manifest, new_plan=True) + + with self.assertRaises(MigrationError) as raised: + apply_migration(manifest) + + self.assertEqual(raised.exception.code, "target_escape") + self.assertFalse((self.base / "outside.md").exists()) + + def test_rollback_removes_owned_files_only_and_is_idempotent(self): + self.write_sources() + planned = self.plan() + applied = apply_migration(planned) + sibling = self.workspace / "Sources" / "Imported" / "user-owned.md" + sibling.write_text("keep", encoding="utf-8") + anchors = [ + migration_module._stage(applied.manifest, entry) + for entry in applied.manifest["entries"] + ] + self.assertTrue(all(anchor.is_file() for anchor in anchors)) + + rolled_back = rollback_migration(planned) + + self.assertEqual(rolled_back.report["status"], "rolled-back") + self.assertEqual(rolled_back.report["summary"]["changed"], 2) + self.assertTrue(sibling.is_file()) + self.assertTrue( + all(not Path(entry["target"]).exists() for entry in applied.manifest["entries"]) + ) + self.assertTrue(all(not anchor.exists() for anchor in anchors)) + repeated = rollback_migration(planned) + self.assertEqual(repeated.report["status"], "unchanged") + self.assertTrue(sibling.is_file()) + self.assertNotIn(str(self.base), json.dumps(rolled_back.report)) + + def test_rollback_persists_delete_intent_and_recovers_after_state_write_failure(self): + self.write_sources() + planned = self.plan() + applied = apply_migration(planned) + targets = [Path(entry["target"]) for entry in applied.manifest["entries"]] + real_persist = migration_module._persist_manifest + failed = False + + def fail_after_first_delete(manifest): + nonlocal failed + if ( + not failed + and manifest["run_state"] == "rolling-back" + and manifest["entries"][0]["state"] == "deleted" + ): + failed = True + raise MigrationError("state_write_failed") + return real_persist(manifest) + + with mock.patch( + "scriptorium.migration._persist_manifest", + side_effect=fail_after_first_delete, + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "state_write_failed") + self.assertFalse(targets[0].exists()) + stored = json.loads(self.manifest_path(planned).read_text(encoding="utf-8")) + self.assertEqual(stored["entries"][0]["state"], "delete-pending") + + recovered = rollback_migration(planned) + self.assertEqual(recovered.report["status"], "rolled-back") + self.assertTrue(all(not target.exists() for target in targets)) + + def test_modified_target_fails_closed_before_any_rollback_deletion(self): + self.write_sources() + applied = apply_migration(self.plan()) + targets = [Path(entry["target"]) for entry in applied.manifest["entries"]] + targets[-1].write_bytes(b"user modification") + + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertTrue(all(path.exists() for path in targets)) + + def test_same_byte_replacement_is_not_deleted_by_rollback(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic replacement sentinel") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + self.assertTrue(os.path.samefile(anchor, target)) + + target.unlink() + target.write_bytes(source.read_bytes()) + self.assertFalse(os.path.samefile(anchor, target)) + + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertTrue(anchor.is_file()) + + def test_target_replaced_after_preflight_is_not_deleted(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic post-preflight sentinel") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + real_preflight = migration_module._preflight_rollback + replaced = False + + def replace_after_preflight(manifest, **kwargs): + nonlocal replaced + real_preflight(manifest, **kwargs) + if not replaced: + replaced = True + target.unlink() + target.write_bytes(source.read_bytes()) + + with mock.patch( + "scriptorium.migration._preflight_rollback", + side_effect=replace_after_preflight, + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertTrue(anchor.is_file()) + self.assertFalse(os.path.samefile(anchor, target)) + self.assertFalse( + any(target.parent.glob(".scriptorium-*.rollback")) + ) + + def test_preoccupied_quarantine_never_overwrites_or_deletes_either_file(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic quarantine source") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + token = "2" * 32 + quarantine = target.parent / ( + f".scriptorium-{applied.manifest['batch_id']}-{entry['id']}-" + f"{token}.rollback" + ) + quarantine.write_bytes(b"user quarantine occupant") + + with mock.patch( + "scriptorium.migration.secrets.token_hex", return_value=token + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "quarantine_conflict") + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertEqual(quarantine.read_bytes(), b"user quarantine occupant") + self.assertTrue(anchor.is_file()) + + def test_restore_blocked_preserves_moved_replacement_and_new_target(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic restore source") + applied = apply_migration(self.plan([source])) + entry = applied.manifest["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(applied.manifest, entry) + real_preflight = migration_module._preflight_rollback + real_rename = migration_module._rename_noreplace + replaced = False + reoccupied = False + + def replace_after_preflight(manifest, **kwargs): + nonlocal replaced + real_preflight(manifest, **kwargs) + if not replaced: + replaced = True + target.unlink() + target.write_bytes(source.read_bytes()) + + def reoccupy_after_move(old, new): + nonlocal reoccupied + real_rename(old, new) + if old == target and not reoccupied: + reoccupied = True + target.write_bytes(b"new target occupant") + + with mock.patch( + "scriptorium.migration._preflight_rollback", + side_effect=replace_after_preflight, + ), mock.patch( + "scriptorium.migration._rename_noreplace", + side_effect=reoccupy_after_move, + ): + with self.assertRaises(MigrationError) as raised: + rollback_migration(applied) + + self.assertEqual(raised.exception.code, "rollback_restore_blocked") + self.assertEqual(target.read_bytes(), b"new target occupant") + stored = migration_module.load_migration( + workspace=self.workspace, batch_id="batch-001" + ).manifest + quarantine = migration_module._quarantine( + stored, stored["entries"][0] + ) + self.assertEqual(quarantine.read_bytes(), source.read_bytes()) + self.assertTrue(anchor.is_file()) + + def test_random_stage_collision_preserves_the_competing_file(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic collision source") + planned = self.plan([source]) + entry = planned.manifest["entries"][0] + target = Path(entry["target"]) + target.parent.mkdir(parents=True) + collision_name = ( + f".scriptorium-{planned.manifest['batch_id']}-{entry['id']}-" + f"{'0' * 32}.stage" + ) + collision = target.parent / collision_name + collision.write_bytes(b"user replacement") + + with mock.patch( + "scriptorium.migration.secrets.token_hex", + side_effect=["0" * 32, "1" * 32], + ): + applied = apply_migration(planned) + + self.assertEqual(collision.read_bytes(), b"user replacement") + anchor = migration_module._stage( + applied.manifest, applied.manifest["entries"][0] + ) + self.assertNotEqual(anchor, collision) + self.assertEqual(anchor.read_bytes(), source.read_bytes()) + + def test_complete_unclaimed_stage_after_state_crash_does_not_block_retry(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic unclaimed stage") + planned = self.plan([source]) + real_persist = migration_module._persist_manifest + failed = False + + def fail_before_stage_claim(manifest): + nonlocal failed + entry = manifest["entries"][0] + if ( + not failed + and entry["state"] == "creating" + and entry["stage_name"] is not None + ): + failed = True + raise MigrationError("state_write_failed") + return real_persist(manifest) + + with mock.patch( + "scriptorium.migration._persist_manifest", + side_effect=fail_before_stage_claim, + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "state_write_failed") + target_parent = self.target(planned).parent + orphaned = list(target_parent.glob(".scriptorium-*.stage")) + self.assertEqual(len(orphaned), 1) + + recovered = apply_migration(planned) + + anchor = migration_module._stage( + recovered.manifest, recovered.manifest["entries"][0] + ) + self.assertNotEqual(anchor, orphaned[0]) + self.assertEqual(orphaned[0].read_bytes(), source.read_bytes()) + self.assertEqual(anchor.read_bytes(), source.read_bytes()) + + def test_stage_replaced_after_identity_persist_is_rejected_and_preserved(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic claimed stage") + planned = self.plan([source]) + real_persist = migration_module._persist_manifest + replaced = False + replacement = None + + def replace_after_stage_claim(manifest): + nonlocal replaced, replacement + real_persist(manifest) + entry = manifest["entries"][0] + if ( + not replaced + and entry["state"] == "creating" + and entry["stage_name"] is not None + ): + replaced = True + replacement = migration_module._stage(manifest, entry) + replacement.unlink() + replacement.write_bytes(source.read_bytes()) + + with mock.patch( + "scriptorium.migration._persist_manifest", + side_effect=replace_after_stage_claim, + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertIsNotNone(replacement) + self.assertEqual(replacement.read_bytes(), source.read_bytes()) + self.assertFalse(self.target(planned).exists()) + + def test_hash_mismatch_preserves_unclaimed_stage_for_manual_review(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic original") + planned = self.plan([source]) + real_verify = migration_module._verify_source + calls = 0 + + def change_after_second_verification(entry): + nonlocal calls + real_verify(entry) + calls += 1 + if calls == 2: + source.write_bytes(b"synthetic changed") + + with mock.patch( + "scriptorium.migration._verify_source", + side_effect=change_after_second_verification, + ): + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "source_hash_changed") + target = self.target(planned) + self.assertFalse(target.exists()) + orphaned = list(target.parent.glob(".scriptorium-*.stage")) + self.assertEqual(len(orphaned), 1) + self.assertEqual(orphaned[0].read_bytes(), b"synthetic changed") + + def test_changed_stage_is_not_deleted_during_apply_recovery(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic stage sentinel") + planned = self.plan([source]) + with mock.patch( + "scriptorium.migration.os.link", + side_effect=OSError(errno.EPERM, "synthetic"), + ): + with self.assertRaises(MigrationError): + apply_migration(planned) + + stored = json.loads(self.manifest_path(planned).read_text(encoding="utf-8")) + entry = stored["entries"][0] + anchor = migration_module._stage(stored, entry) + anchor.unlink() + anchor.write_bytes(b"user replacement") + + with self.assertRaises(MigrationError) as raised: + apply_migration(planned) + + self.assertEqual(raised.exception.code, "staging_conflict") + self.assertEqual(anchor.read_bytes(), b"user replacement") + + def test_delete_pending_same_byte_anchor_replacement_is_not_deleted(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic anchor sentinel") + applied = apply_migration(self.plan([source])) + current = copy.deepcopy(applied.manifest) + current["run_state"] = "rolling-back" + pending = current["entries"][0] + pending["state"] = "delete-pending" + pending["rollback_phase"] = "anchor" + pending["quarantine_name"] = migration_module._new_internal_name( + current, pending, "rollback" + ) + migration_module._persist_manifest(current) + + entry = current["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(current, entry) + target.unlink() + anchor.unlink() + anchor.write_bytes(source.read_bytes()) + self.assertNotEqual( + entry["file_identity"], migration_module._file_identity(anchor) + ) + + with self.assertRaises(MigrationError) as raised: + rollback_migration(current) + + self.assertEqual(raised.exception.code, "owned_target_changed") + self.assertEqual(anchor.read_bytes(), source.read_bytes()) + + def test_delete_pending_crash_recovers_with_persisted_anchor_identity(self): + source = self.sources / "idea.md" + source.write_bytes(b"synthetic crash sentinel") + applied = apply_migration(self.plan([source])) + current = copy.deepcopy(applied.manifest) + current["run_state"] = "rolling-back" + pending = current["entries"][0] + pending["state"] = "delete-pending" + pending["rollback_phase"] = "anchor" + pending["quarantine_name"] = migration_module._new_internal_name( + current, pending, "rollback" + ) + migration_module._persist_manifest(current) + + entry = current["entries"][0] + target = Path(entry["target"]) + anchor = migration_module._stage(current, entry) + target.unlink() + self.assertTrue(anchor.is_file()) + self.assertEqual( + entry["file_identity"], migration_module._file_identity(anchor) + ) + + recovered = rollback_migration(current) + + self.assertEqual(recovered.report["status"], "rolled-back") + self.assertFalse(target.exists()) + self.assertFalse(anchor.exists()) + + def test_each_persisted_quarantine_phase_recovers_after_crash(self): + phases = ( + "target", + "target-quarantined", + "anchor", + "anchor-quarantined", + ) + for index, phase in enumerate(phases, start=1): + with self.subTest(phase=phase): + source = self.sources / f"phase-{index}.md" + source.write_bytes(f"synthetic {phase}".encode()) + applied = apply_migration( + self.plan([source], batch_id=f"phase-{index}") + ) + current = copy.deepcopy(applied.manifest) + current["run_state"] = "rolling-back" + entry = current["entries"][0] + entry["state"] = "delete-pending" + entry["rollback_phase"] = phase + entry["quarantine_name"] = migration_module._new_internal_name( + current, entry, "rollback" + ) + target = Path(entry["target"]) + anchor = migration_module._stage(current, entry) + quarantine = migration_module._quarantine(current, entry) + migration_module._persist_manifest(current) + + if phase.startswith("anchor"): + target.unlink() + migration_module._rename_noreplace(anchor, quarantine) + else: + migration_module._rename_noreplace(target, quarantine) + + recovered = rollback_migration(current) + + self.assertEqual(recovered.report["status"], "rolled-back") + self.assertFalse(target.exists()) + self.assertFalse(anchor.exists()) + self.assertFalse(quarantine.exists()) + + def test_directory_without_supported_files_is_an_error(self): + (self.sources / "data.txt").write_text("synthetic", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan() + self.assertEqual(raised.exception.code, "no_supported_files") + self.assertEqual(snapshot(self.workspace), {}) + + def test_unsupported_explicit_file_is_rejected(self): + unsupported = self.sources / "data.csv" + unsupported.write_text("a,b\n1,2\n", encoding="utf-8") + with self.assertRaises(MigrationError) as raised: + self.plan([unsupported]) + self.assertEqual(raised.exception.code, "unsupported_source_type") + + def test_file_and_directory_selection_deduplicates_same_source(self): + markdown, _, _ = self.write_sources() + planned = self.plan([self.sources, markdown]) + sources = [entry["source"] for entry in planned.manifest["entries"]] + self.assertEqual(len(sources), len(set(sources))) + self.assertEqual(len(sources), 2) + + def test_reparse_attribute_is_rejected(self): + metadata = SimpleNamespace( + st_mode=stat.S_IFDIR, st_file_attributes=0x400 + ) + self.assertTrue(migration_module._linklike(metadata)) + + def test_nested_directory_symlink_is_rejected_when_supported(self): + outside = self.base / "outside" + outside.mkdir() + (outside / "private.md").write_text("private", encoding="utf-8") + linked = self.sources / "linked" + try: + linked.symlink_to(outside, target_is_directory=True) + except OSError as exc: + self.skipTest(f"directory symlinks are unavailable: {exc}") + with self.assertRaises(MigrationError) as raised: + self.plan() + self.assertEqual(raised.exception.code, "link_or_reparse_rejected") + + def test_kernel_lock_is_released_when_process_exits_without_cleanup(self): + lock = self.state_root / "migrations" / ("a" * 20) / "exit.lock" + source_root = Path(__file__).resolve().parents[1] / "src" + code = ( + "import os,sys;" + "from pathlib import Path;" + "from scriptorium.migration import _kernel_lock;" + "lock=Path(sys.argv[1]);" + "ctx=_kernel_lock(lock);" + "ctx.__enter__();" + "os._exit(0)" + ) + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + [str(source_root), environment.get("PYTHONPATH", "")] + ) + completed = subprocess.run( + [sys.executable, "-c", code, str(lock)], + env=environment, + timeout=10, + check=False, + ) + self.assertEqual(completed.returncode, 0) + with migration_module._kernel_lock(lock): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pull.py b/tests/test_pull.py index 9b69fe0..1e7fc64 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -29,7 +29,7 @@ def component_report( summary["codex_found"] = 1 return { "format_version": 1, - "generated_by": {"name": "provenance", "version": "0.17.0"}, + "generated_by": {"name": "provenance", "version": "0.18.0"}, "operation": "pull", "mode": mode, "status": status, @@ -338,7 +338,7 @@ def test_invalid_provenance_environment_root_does_not_fall_back_to_path(self): with self.assertRaisesRegex( PullError, "configured Provenance root is unavailable" ): - _resolve_provenance(None, expected_version="0.17.0") + _resolve_provenance(None, expected_version="0.18.0") installed.assert_not_called() def test_environment_is_narrow_and_raw_stderr_is_suppressed(self): @@ -444,14 +444,14 @@ def test_explicit_source_version_is_fixed_by_compatibility_manifest(self): encoding="utf-8", ) with self.assertRaisesRegex(PullError, "incompatible Provenance"): - _resolve_provenance(root, expected_version="0.17.0") + _resolve_provenance(root, expected_version="0.18.0") def test_non_json_extra_stdout_and_timeout_fail_closed(self): with self.assertRaisesRegex(PullError, "non-JSON or extra stdout"): _parse_component_report( json.dumps(component_report()) + "\nlog line", expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -499,18 +499,18 @@ def test_report_exit_one_is_trusted_when_process_and_status_match(self): report, producer = _parse_component_report( json.dumps(component_report(status="blocked", exit_code=1)), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=1, ) self.assertEqual(report["exit_code"], 1) - self.assertEqual(producer["version"], "0.17.0") + self.assertEqual(producer["version"], "0.18.0") def test_report_mismatch_or_wrong_producer_is_rejected(self): with self.assertRaisesRegex(PullError, "exit code disagrees"): _parse_component_report( json.dumps(component_report()), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=1, ) wrong = component_report() @@ -519,7 +519,7 @@ def test_report_mismatch_or_wrong_producer_is_rejected(self): _parse_component_report( json.dumps(wrong), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -530,7 +530,7 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): _parse_component_report( json.dumps(boolean_version), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -540,7 +540,7 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): _parse_component_report( json.dumps(non_string_status), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -550,7 +550,7 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): _parse_component_report( json.dumps(non_integer_exit), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) @@ -560,11 +560,11 @@ def test_report_types_are_strict_and_unknown_fields_are_dropped(self): parsed, producer = _parse_component_report( json.dumps(additive), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=0, ) self.assertNotIn("private_future_field", parsed) - self.assertEqual(producer, {"name": "provenance", "version": "0.17.0"}) + self.assertEqual(producer, {"name": "provenance", "version": "0.18.0"}) def test_nested_component_data_is_rebuilt_as_aggregate_only(self): report = component_report(status="blocked", exit_code=1) @@ -612,7 +612,7 @@ def test_nested_component_data_is_rebuilt_as_aggregate_only(self): parsed, _ = _parse_component_report( json.dumps(report), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=1, ) @@ -701,7 +701,7 @@ def test_required_nested_semantics_fail_closed(self): _parse_component_report( json.dumps(report), expected_mode="preview", - expected_version="0.17.0", + expected_version="0.18.0", process_exit_code=report["exit_code"], ) diff --git a/tests/test_resume.py b/tests/test_resume.py new file mode 100644 index 0000000..74bfdb2 --- /dev/null +++ b/tests/test_resume.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import contextlib +import io +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from scriptorium import cli +from scriptorium.config import SuiteConfig, render_config +from scriptorium.resume import ResumeError, _parse_capsule, format_resume_report, run_resume + + +def capsule() -> dict[str, object]: + return { + "capsule_version": "context-capsule/0.1", + "project": { + "project_id": "synthetic-catalyst", + "title": "Synthetic Catalyst Study", + "status": "active", + "stage": "evidence-review", + "priority": "high", + "updated": "2026-07-22", + "goal": "Test one falsifiable catalyst hypothesis.", + "conclusion": "", + }, + "next_actions": ["Compare the two synthetic papers."], + "blocked_by": "", + "recent_progress": [ + {"date": "2026-07-22", "items": ["Approved the research question."]} + ], + "literature": [ + { + "id": "paper-1", + "citekey": "synthetic2026", + "title": "A Synthetic Catalyst Paper", + "year": 2026, + "read_status": "deep_read", + "tldr": "A synthetic result for contract testing.", + } + ], + "research_artifacts": [ + { + "kind": "reading-note", + "schema_version": "reading-note/1.0", + "id": "reading-note-1", + "title": "Synthetic reading note", + "status": "reviewed", + "source_ids": ["paper-1"], + "summary": "Reference-only synthetic evidence.", + "trust": "reference_only", + } + ], + "reference_leads": { + "gaps": ["No synthetic replication yet."], + "priority_reads": ["paper-1"], + }, + "trust": { + "project_state": "human_or_approved", + "recent_progress": "auto_applied_low_risk_not_approved_claims", + "research_artifacts": "reference_only_not_approved_claims", + }, + "limits": {"max_markdown_chars": 8000, "truncated": False}, + } + + +def completed(value: dict[str, object], *, stderr: str = "") -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess( + args=["prov-context"], + returncode=0, + stdout=json.dumps(value), + stderr=stderr, + ) + + +class ResumeBoundaryTests(unittest.TestCase): + def test_run_resume_uses_public_command_and_suppresses_local_paths(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + home = root / "private data" + source = root / "Provenance" + home.mkdir() + source.mkdir() + with ( + mock.patch("scriptorium.resume._compatibility_version", return_value="0.17.0"), + mock.patch( + "scriptorium.resume._resolve_provenance", + return_value=(source, Path("prov-context")), + ), + mock.patch( + "scriptorium.resume.subprocess.run", + side_effect=[ + subprocess.CompletedProcess( + args=["prov-context", "--version"], + returncode=0, + stdout="0.17.0\n", + stderr="", + ), + completed(capsule(), stderr="private diagnostic"), + ], + ) as invoke, + ): + report = run_resume( + provenance_home=home, + project="synthetic-catalyst", + ) + + command = invoke.call_args_list[1].args[0] + self.assertEqual( + command, ["prov-context", "--project", "synthetic-catalyst", "--json"] + ) + self.assertEqual( + invoke.call_args_list[1].kwargs["env"]["PROVENANCE_HOME"], + str(home.resolve()), + ) + self.assertEqual(report["operation"], "resume") + self.assertEqual(report["capsule"]["project"]["project_id"], "synthetic-catalyst") + self.assertEqual(report["entry"]["stderr"], "suppressed") + rendered = json.dumps(report) + self.assertNotIn(str(home), rendered) + self.assertNotIn("private diagnostic", rendered) + + def test_unknown_component_fields_fail_closed(self): + value = capsule() + value["private_path"] = "hidden" + with self.assertRaisesRegex(ResumeError, "root.*incompatible"): + _parse_capsule(json.dumps(value)) + + def test_local_paths_fail_closed(self): + value = capsule() + value["project"]["goal"] = r"Read C:\Users\Researcher\private.md" + with self.assertRaisesRegex(ResumeError, "suppressed local path"): + _parse_capsule(json.dumps(value)) + + def test_reference_artifacts_cannot_claim_authoritative_trust(self): + value = capsule() + value["research_artifacts"][0]["trust"] = "authoritative" + with self.assertRaisesRegex(ResumeError, "artifact trust"): + _parse_capsule(json.dumps(value)) + + def test_component_failure_does_not_forward_diagnostics(self): + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + failed = subprocess.CompletedProcess( + args=["prov-context"], + returncode=2, + stdout="", + stderr=r"unknown project under C:\Users\Researcher\private", + ) + with ( + mock.patch("scriptorium.resume._compatibility_version", return_value="0.17.0"), + mock.patch( + "scriptorium.resume._resolve_provenance", + return_value=(None, Path("prov-context")), + ), + mock.patch( + "scriptorium.resume.subprocess.run", + side_effect=[ + subprocess.CompletedProcess( + args=["prov-context", "--version"], + returncode=0, + stdout="0.17.0\n", + stderr="", + ), + failed, + ], + ), + ): + with self.assertRaisesRegex(ResumeError, "did not return a capsule") as raised: + run_resume(provenance_home=home, project="missing") + self.assertNotIn("Researcher", str(raised.exception)) + + def test_runtime_version_mismatch_fails_before_capsule_request(self): + with tempfile.TemporaryDirectory() as temporary: + home = Path(temporary) + with ( + mock.patch( + "scriptorium.resume._compatibility_version", + return_value="0.17.0", + ), + mock.patch( + "scriptorium.resume._resolve_provenance", + return_value=(None, Path("prov-context")), + ), + mock.patch( + "scriptorium.resume.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["prov-context", "--version"], + returncode=0, + stdout="0.16.0\n", + stderr=r"private C:\Users\Researcher", + ), + ) as invoke, + ): + with self.assertRaisesRegex( + ResumeError, "incompatible runtime version" + ) as raised: + run_resume(provenance_home=home, project="synthetic-catalyst") + + self.assertEqual(invoke.call_count, 1) + self.assertNotIn("Researcher", str(raised.exception)) + + def test_human_report_marks_research_artifacts_reference_only(self): + report = { + "generated_by": {"version": "0.1.0"}, + "capsule": capsule(), + "path_selection": {}, + "warnings": [], + } + rendered = format_resume_report(report) + self.assertIn("Synthetic Catalyst Study", rendered) + self.assertIn("Reference context", rendered) + self.assertIn("reference-only", rendered) + + +class ResumeCliTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.workspace = self.root / "workspace" + self.home = self.root / "provenance" + self.workspace.mkdir() + self.home.mkdir() + self.config_root = self.root / "config" + config_path = self.config_root / "scriptorium" / "config.toml" + config_path.parent.mkdir(parents=True) + config_path.write_bytes( + render_config( + SuiteConfig( + workspace=self.workspace.resolve(), + provenance_home=self.home.resolve(), + hosts=("codex",), + default_project="configured-project", + ) + ) + ) + + def invoke_json(self, arguments: list[str]) -> tuple[int, dict[str, object], str]: + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = cli.main(arguments) + return exit_code, json.loads(stdout.getvalue()), stderr.getvalue() + + def test_resume_uses_configured_home_and_project(self): + component_report = {"operation": "resume", "exit_code": 0} + with ( + mock.patch.dict(os.environ, {}, clear=True), + mock.patch("scriptorium.cli.run_resume", return_value=component_report) as run, + ): + exit_code, report, stderr = self.invoke_json( + ["resume", "--config-dir", str(self.config_root), "--json"] + ) + + self.assertEqual(exit_code, 0) + self.assertEqual(stderr, "") + self.assertEqual(run.call_args.kwargs["provenance_home"], self.home.resolve()) + self.assertEqual(run.call_args.kwargs["project"], "configured-project") + self.assertEqual(report["path_selection"]["data_root"]["source"], "suite-config") + + def test_resume_json_usage_error_never_echoes_private_arguments(self): + secret = r"C:\Users\Researcher\private-project" + exit_code, report, stderr = self.invoke_json( + ["resume", "--project", secret, "--json", "--unknown", secret] + ) + self.assertEqual(exit_code, 2) + self.assertEqual(stderr, "") + self.assertEqual(report["operation"], "resume") + self.assertNotIn(secret, json.dumps(report)) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 5ff22cb..c3a1af6 100644 --- a/uv.lock +++ b/uv.lock @@ -4,5 +4,5 @@ requires-python = ">=3.11" [[package]] name = "scriptorium-suite" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } From e231b4473ba093ebc4456fbcc16fbf3912521341 Mon Sep 17 00:00:00 2001 From: foxsplendid <62349701+foxsplendid@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:08:20 +0800 Subject: [PATCH 2/2] Fix cross-platform acceptance CI --- .github/workflows/ci.yml | 7 +++++++ README.md | 11 +++++++---- README.zh.md | 8 +++++--- docs/architecture-and-acceptance.zh-CN.md | 12 +++++++----- tests/e2e_install_lifecycle.py | 5 +++-- tests/test_migration.py | 13 +++++++++---- 6 files changed, 38 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b93da0a..8ae2a93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,13 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" + - name: Install pinned build backend + run: >- + python -m pip install + --disable-pip-version-check + --no-deps + --only-binary=:all: + "setuptools==83.0.0" - name: Verify clean Windows install lifecycle shell: pwsh run: | diff --git a/README.md b/README.md index 1978c0c..715b165 100644 --- a/README.md +++ b/README.md @@ -130,10 +130,13 @@ automatically. Rollback records a random same-directory quarantine name before each target and anchor transition, atomically moves without replacement, then re-verifies content -and stable file identity before deletion. A foreign replacement is restored to its -original path when possible, otherwise preserved in quarantine while rollback -fails closed. Automatic rollback requires Windows no-replace rename or Linux -`renameat2(RENAME_NOREPLACE)`; other platforms fail closed. Private +and recorded file identity before deletion. A replacement detectable through +those recorded properties is restored to its original path when possible, +otherwise preserved in quarantine while rollback fails closed. This coordinates +cooperative local processes; it does not claim protection against a malicious +replacement after every link to a file identity has disappeared, because a +filesystem may reuse that identity. Automatic rollback requires Windows no-replace +rename or Linux `renameat2(RENAME_NOREPLACE)`; other platforms fail closed. Private path manifests use the canonical per-user local state root outside the workspace; sources may not overlap that private state root, and no arbitrary state-root flag is exposed. Terminal diff --git a/README.zh.md b/README.zh.md index faeb311..6fc5607 100644 --- a/README.zh.md +++ b/README.zh.md @@ -102,9 +102,11 @@ stage 与身份封入私有 manifest，并发布为目标的同文件所有权 可能留下未认领的随机 stage；系统不会猜测归属，也不会自动删除它。 回滚会先持久化随机的同目录 quarantine 名，再用不覆盖的原子移动分别隔离目标和锚点， -移动后重新检查内容与稳定文件身份，确认归属后才删除。错误替代文件会尽量原子恢复到 -原路径；恢复受阻时保留在 quarantine 并 fail closed。自动回滚要求 Windows -no-replace rename 或 Linux `renameat2(RENAME_NOREPLACE)`；其他平台不降级执行。 +移动后重新检查内容与已记录文件身份，确认归属后才删除。能由这些记录属性识别的外部 +替代文件会尽量原子恢复到原路径；恢复受阻时保留在 quarantine 并 fail closed。 +该机制只协调单用户下的协作进程；最后一个链接消失后文件系统可能复用 inode/file ID， +因此不宣称抵御恶意本地进程主动替换。自动回滚要求 Windows no-replace rename 或 +Linux `renameat2(RENAME_NOREPLACE)`；其他平台不降级执行。 含绝对路径的私有 manifest 固定存放在 workspace 外的用户级 canonical 本地状态根， 迁移来源不得与该私有状态根重叠，CLI 不提供任意 `state-root` 参数。终端和 JSON 报告只含聚合计数与状态。迁移只复制 diff --git a/docs/architecture-and-acceptance.zh-CN.md b/docs/architecture-and-acceptance.zh-CN.md index 43fe7e4..d11ac21 100644 --- a/docs/architecture-and-acceptance.zh-CN.md +++ b/docs/architecture-and-acceptance.zh-CN.md @@ -156,10 +156,12 @@ stage；后续重试不会扫描、认领或自动删除它，因此不会把竞 回滚采用 `target -> target-quarantined -> anchor -> anchor-quarantined -> deleted` 持久化状态机。每一步先记录 128-bit 随机同目录 quarantine 名，再用 Windows no-replace rename 或 Linux `renameat2(RENAME_NOREPLACE)` 原子移动，移动后重新验证 -哈希、大小和持久文件身份；错误替代文件只恢复或保留，不删除。平台不支持原子 -no-replace move、恢复路径被占用或身份不一致时均 fail closed。active batch 期间不得 -删除已登记的 `.scriptorium-*.stage` / `.scriptorium-*.rollback`；成功回滚最后清理 -已登记条目，未认领 orphan 只供人工核查。终端与 +哈希、大小和已记录文件身份；能由这些属性识别的外部替代文件只恢复或保留，不删除。 +该状态机协调单用户下的协作进程；最后一个链接消失后文件系统可能复用 inode/file ID， +因此不宣称抵御恶意本地进程主动替换。平台不支持原子 no-replace move、恢复路径被占用 +或身份不一致时均 fail closed。active batch 期间不得删除已登记的 +`.scriptorium-*.stage` / `.scriptorium-*.rollback`；成功回滚最后清理已登记条目， +未认领 orphan 只供人工核查。终端与 JSON 报告、参数错误和运行时错误均不回显路径、文件名或正文。 当前已由合成测试覆盖显式输入、无覆盖、拒绝链接/重解析点、进程退出释放锁、字节哈希 @@ -357,7 +359,7 @@ V0.4 是面向外部 Alpha 的产品验收，而不只是开发者测试： - 报告没有正文、邮箱、密钥和绝对路径； - 待审批内容没有自动变成正式状态。 9. 重复运行相同步骤，验证没有重复记录、重复复制或字节漂移。 -10. 用 `migrate verify` 检查批次后，在隔离副本上执行 `migrate rollback`；回滚只能删除本批次确认创建且身份、字节均未变的副本，不会删除或覆盖替代文件；恢复受阻时必须保留 quarantine 并 fail closed。 +10. 用 `migrate verify` 检查批次后，在隔离副本上执行 `migrate rollback`；回滚只能删除本批次确认创建且身份、字节均未变的副本；能由记录属性识别的替代文件不会被删除或覆盖，恢复受阻时必须保留 quarantine 并 fail closed。该验收只覆盖单用户下的协作进程，不覆盖恶意本地替换。 11. 用新会话运行 `resume`，确认只恢复已接受的状态，拒绝项、原始聊天和未审批草稿没有进入胶囊。 12. 最终 UAT 报告只记录版本、操作系统、通过/失败、计数、匿名 ID、耗时和已知限制。公开前再做一次秘密、路径和私人关键词扫描。 diff --git a/tests/e2e_install_lifecycle.py b/tests/e2e_install_lifecycle.py index 78e138d..bffda9f 100644 --- a/tests/e2e_install_lifecycle.py +++ b/tests/e2e_install_lifecycle.py @@ -513,9 +513,10 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: ) try: import setuptools # noqa: F401 - import wheel # noqa: F401 except ImportError as exc: - raise LifecycleFailure("build-tooling", "Local wheel build tooling is unavailable.") from exc + raise LifecycleFailure( + "build-tooling", "The local setuptools build backend is unavailable." + ) from exc stages.append({"name": "build-tooling", "status": "passed"}) with tempfile.TemporaryDirectory(prefix="scriptorium-install-lifecycle-") as raw: diff --git a/tests/test_migration.py b/tests/test_migration.py index 8a017a9..3062e03 100644 --- a/tests/test_migration.py +++ b/tests/test_migration.py @@ -704,8 +704,12 @@ def replace_after_stage_claim(manifest): ): replaced = True replacement = migration_module._stage(manifest, entry) - replacement.unlink() - replacement.write_bytes(source.read_bytes()) + foreign = replacement.with_name(f"{replacement.name}.foreign") + foreign.write_bytes(source.read_bytes()) + self.assertNotEqual( + entry["file_identity"], migration_module._file_identity(foreign) + ) + os.replace(foreign, replacement) with mock.patch( "scriptorium.migration._persist_manifest", @@ -787,9 +791,10 @@ def test_delete_pending_same_byte_anchor_replacement_is_not_deleted(self): entry = current["entries"][0] target = Path(entry["target"]) anchor = migration_module._stage(current, entry) + foreign = anchor.with_name(f"{anchor.name}.foreign") + foreign.write_bytes(source.read_bytes()) target.unlink() - anchor.unlink() - anchor.write_bytes(source.read_bytes()) + os.replace(foreign, anchor) self.assertNotEqual( entry["file_identity"], migration_module._file_identity(anchor) )