From d0963a700a1303bc8d3d331d6b6a22e2d6ab5052 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 19:16:59 -0700 Subject: [PATCH 01/50] Docs: Specify the version two retention store --- CHANGELOG.md | 5 + docs/formats/README.md | 3 +- docs/formats/segment-store-v2/README.md | 67 ++++ docs/formats/segment-store-v2/gc.md | 193 +++++++++++ .../segment-store-v2/migration-crash.md | 81 +++++ docs/formats/segment-store-v2/rationale.md | 88 ++++++ docs/formats/segment-store-v2/recovery.md | 285 +++++++++++++++++ docs/formats/segment-store-v2/requirements.md | 63 ++++ docs/formats/segment-store-v2/retention.md | 299 ++++++++++++++++++ .../retention_store_v2_protocol_contract.rs | 228 +++++++++++++ 10 files changed, 1311 insertions(+), 1 deletion(-) create mode 100644 docs/formats/segment-store-v2/README.md create mode 100644 docs/formats/segment-store-v2/gc.md create mode 100644 docs/formats/segment-store-v2/migration-crash.md create mode 100644 docs/formats/segment-store-v2/rationale.md create mode 100644 docs/formats/segment-store-v2/recovery.md create mode 100644 docs/formats/segment-store-v2/requirements.md create mode 100644 docs/formats/segment-store-v2/retention.md create mode 100644 xtask/tests/retention_store_v2_protocol_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c1454..0da89d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -303,6 +303,11 @@ after its public API and format compatibility policies are established. ### Added +- Specified `keep.segment-store/v2` retention values, root generations, + liveness manifests, reader snapshots, one-way staged migration, exact crash + boundaries, and reserved GC/disposition records. Version-1 immutable bytes + remain authoritative; production version-2 writing remains unavailable until + issue #19's executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/docs/formats/README.md b/docs/formats/README.md index 64db302..6b8ce93 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -8,7 +8,8 @@ admitted merely because one Rust type can serialize and deserialize it. | Format | Coordinate | Status | Evidence | | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | -| [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Specified in issue #14; segment I/O implemented in issue #15; publication and recovery remain in issues #16–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | +| [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | +| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation and executable evidence planned in issue #19 | Golden corpus planned in issue #19 | The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md new file mode 100644 index 0000000..c908aab --- /dev/null +++ b/docs/formats/segment-store-v2/README.md @@ -0,0 +1,67 @@ +# Durable Segment Store Version 2 + +`keep.segment-store/v2` is the accepted successor to +`keep.segment-store/v1`. It preserves every admitted version-1 segment, +catalog, and publication-head byte while adding explicit retention state, +reader fences, migration evidence, and reserved GC and recovery-disposition +namespaces. + +ADR-0009 owns the cross-cutting retention and liveness decision. These pages +own its durable representation. Issue #19 must supply the production retention +implementation and executable evidence before any version-2 writer is +available. Until that implementation lands, version 1 remains the only +admitted production store. + +## Core laws + +Version 2 retains every version-1 physical law and adds these: + +- the format version is explicit and cannot be inferred from path existence; +- a complete version-2 store is entered only by the specified version-1 + migration; direct version-2 initialization is undefined; +- migration is one-way, writer-authorized, durable, and recoverable from every + documented prefix; +- retention authority exists only through one verified retention head and its + complete immutable manifest; +- each manifest binds every admitted namespace to one exact root generation + and canonical digest; +- root closure is derived from a verified catalog, never from paths, caller + claims, recent access, or application identity; +- catalog publication preserves every current retained closure before + replacing the catalog head; +- readers acquire the version-2 reader fence before opening the catalog head; + and +- ambiguous, corrupt, missing, excessive, or unsupported evidence refuses + before mutation. + +## Normative pages + +The following pages form one protocol: + +- [Retention records and publication](retention.md) owns canonical namespace, + root-generation, manifest, retention-head, closure, and transition rules. +- [GC and disposition records](gc.md) owns the canonical planned intent, + completion, and recovery-disposition byte grammars. +- [Migration and recovery](recovery.md) owns the exact root namespace, + version marker, reader fence, one-way migration, crash states, GC reservation, + recovery-disposition reservation, and restart behavior. +- [Migration crash points](migration-crash.md) owns fixed-stage publication and + the exact process-death boundaries for migration. +- [Requirements and evidence](requirements.md) owns stable requirement and + crash identifiers, evidence status, compatibility, and nonclaims. +- [Format rationale](rationale.md) records format-local choices and rejected + alternatives. + +The version-1 [segment](../segment-store-v1/segment.md), +[catalog](../segment-store-v1/catalog.md), and +[publication-head](../segment-store-v1/catalog.md#publication-head) +grammars remain byte-for-byte authoritative. Version 2 does not reinterpret or +re-encode them. + +## Status + +The format contract is frozen by ADR-0009 and this specification. Requirements +marked **Planned in #19** or **Planned in #21** are not implementation evidence. +A store must refuse version-2 state until the relevant parser, corruption, +golden-format, model-based, crash-injection, recovery, and fuzz evidence is +implemented. diff --git a/docs/formats/segment-store-v2/gc.md b/docs/formats/segment-store-v2/gc.md new file mode 100644 index 0000000..708afab --- /dev/null +++ b/docs/formats/segment-store-v2/gc.md @@ -0,0 +1,193 @@ +# GC and Disposition Records + +This page owns the canonical planned `GcRetirementIntent`, +`GcRetirementReceipt`, and `RecoveryDispositionReceipt` byte grammars for +`keep.segment-store/v2`. + +Issue #21 owns their implementation. They are specified now so version 2 has +one exact root grammar, but their presence remains unsupported mandatory state +until every **Planned in #21** requirement becomes executable evidence. + +## Common rules + +All integers are unsigned and big-endian. Flags and reserved bytes are zero. +Every length and count is checked before allocation. Decoders reject truncation, +trailing bytes, unsupported versions, unknown mandatory flags, nonzero reserved +bytes, overflow, noncanonical ordering, duplicates, digest or checksum +mismatch, and values above fixed ceilings. + +Every digest and checksum uses domain-separated BLAKE3-256. Fixed names are +never replaced to obtain idempotence. + +## GC retirement intent + +`GcRetirementIntent` consists of: + +```text +320-byte fixed-width header +candidate-count × 72-byte candidate entries +32-byte intent digest +32-byte checksum +``` + +The maximum candidate count is 65,536. Its maximum encoded length is +4,718,976 bytes. + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:GC:INTENT2\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | header length | `320` | +| 20 | 4 | flags | `0` | +| 24 | 8 | total record length | derived exact length | +| 32 | 8 | GC generation | positive checked successor | +| 40 | 2 | candidate width | `72` | +| 42 | 2 | reserved | zero | +| 44 | 4 | candidate count | `1..=65,536` | +| 48 | 8 | liveness generation | exact current value | +| 56 | 32 | retention-manifest digest | exact current digest | +| 88 | 8 | catalog generation | exact successor value | +| 96 | 32 | catalog digest | names no candidate segment | +| 128 | 4 | realization-profile identity | exact retained profile | +| 132 | 4 | realization-profile version | exact retained profile | +| 136 | 32 | realization-profile digest | exact retained profile | +| 168 | 32 | catalog-successor proof digest | complete verified proof | +| 200 | 32 | segment-pool identity digest | exact admitted pool | +| 232 | 32 | disposition-set digest | exact admitted receipts | +| 264 | 8 | reader-lock device identity | exact locked file | +| 272 | 8 | reader-lock mount identity | exact locked file | +| 280 | 8 | reader-lock file identity | exact locked file | +| 288 | 32 | candidate-entry-set digest | exact canonical entries | + + + +Each 72-byte candidate entry is: + +| Offset | Width | Field | +| ---: | ---: | --- | +| 0 | 32 | segment digest | +| 32 | 8 | segment length | +| 40 | 32 | complete verification-evidence digest | + +Candidate entries use canonical segment-digest order and are duplicate-free. +The entry-set, intent, and checksum domains are: + +```text +keep.gc-candidate-set/v2\0 +keep.gc-retirement-intent/v2\0 +keep.gc-retirement-intent-checksum/v2\0 +``` + +The checksum covers header, entries, and intent digest. The intent digest +covers the header and entries. + +## GC retirement receipt + +`GcRetirementReceipt` is exactly 320 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:GC:RECEIPT2` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `320` | +| 20 | 4 | flags | `0` | +| 24 | 8 | GC generation | exact intent generation | +| 32 | 32 | intent digest | exact durable intent | +| 64 | 32 | retired candidate-set digest | exact intent set | +| 96 | 32 | post-retirement pool-state digest | verified synchronized pool | +| 128 | 8 | liveness generation | revalidated exact value | +| 136 | 32 | retention-manifest digest | revalidated exact value | +| 168 | 8 | catalog generation | revalidated exact value | +| 176 | 32 | catalog digest | revalidated exact value | +| 208 | 8 | reader-lock device identity | exact exclusive lock | +| 216 | 8 | reader-lock mount identity | exact exclusive lock | +| 224 | 8 | reader-lock file identity | exact exclusive lock | +| 232 | 8 | completed synchronization count | exact intent-derived count | +| 240 | 48 | reserved | zero | +| 288 | 32 | checksum | BLAKE3-256 over bytes `0..288` | + + + +The checksum domain is `keep.gc-retirement-receipt-checksum/v2\0`. + +## Recovery disposition receipt + +`RecoveryDispositionReceipt` is exactly 320 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:REC:DISP2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `320` | +| 20 | 4 | flags | `0` | +| 24 | 2 | artifact kind | registered enum | +| 26 | 2 | decision | finalize or retire | +| 28 | 2 | admitted recovery classification | registered enum | +| 30 | 2 | reserved | zero | +| 32 | 8 | artifact length | exact observed length | +| 40 | 32 | artifact identity digest | physical evidence identity | +| 72 | 32 | artifact content digest | exact verified bytes | +| 104 | 8 | publication-head generation | exact observed value | +| 112 | 32 | publication-head checksum | exact observed value | +| 144 | 8 | catalog generation | exact observed value | +| 152 | 32 | catalog digest | exact observed value | +| 184 | 8 | liveness generation | exact observed value | +| 192 | 32 | retention-manifest digest | exact observed value | +| 224 | 8 | reader-lock device identity | exact safety coordinate | +| 232 | 8 | reader-lock mount identity | exact safety coordinate | +| 240 | 8 | reader-lock file identity | exact safety coordinate | +| 248 | 32 | decision-evidence digest | complete canonical proof | +| 280 | 8 | reserved | zero | +| 288 | 32 | checksum | BLAKE3-256 over bytes `0..288` | + + + +The checksum domain is `keep.recovery-disposition-receipt-checksum/v2\0`. +Unknown artifact kinds, decisions, or classifications refuse. + +The pool coordinate is: + +```text +recovery/dispositions/.receipt +``` + +The version-2 maximum is 65,536 disposition receipts. A future successor must +migrate the namespace before raising the ceiling. + +## State and recovery + +GC admits these states: + + + +| State | Evidence | Recovery | +| --- | --- | --- | +| idle | no `gc/intent` or `gc/receipt` | no retirement authority | +| active | exact intent, every candidate present | begin execution | +| partial | exact intent, one canonical absent candidate prefix | continue at first present candidate | +| completion pending | exact intent, every candidate absent | publish receipt | +| receipt transition | exact intent and exact receipt | synchronize receipt, remove intent, synchronize `gc` | +| complete | exact receipt only | return exact completion | + + + +An absent candidate outside the canonical absent candidate prefix, substituted +candidate, changed pool, stale coordinate, conflicting receipt, malformed +record, or unexplained absence is unrecoverable ambiguity. Recovery never +guesses which deletion occurred. + +A disposition transition writes and synchronizes +`recovery/disposition.next`, verifies and links the immutable receipt without +replacement, synchronizes `recovery/dispositions`, removes the stage, and +synchronizes `recovery`. Until that completes, the artifact remains +recovery-protected. + +These grammars, their golden fixtures, parsers, corruption matrices, crash +points, model, benchmarks, and fuzz targets are **Planned in #21**. Issue #19 +must refuse their physical presence without mutating it. diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md new file mode 100644 index 0000000..c5d7041 --- /dev/null +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -0,0 +1,81 @@ +# Migration Crash Points + +This page owns fixed-record publication and process-death boundaries for the +one-way `keep.segment-store/v1` to `keep.segment-store/v2` migration. + +## Fixed-stage law + +Migration never writes canonical fixed names in place: + +| Stage | Canonical target | +| --- | --- | +| `migration.intent.next` | `migration.intent` | +| `FORMAT.next` | `FORMAT` | +| `migration.receipt.next` | `migration.receipt` | + +For each pair, migration: + +1. creates the stage exclusively as a pinned regular file; +2. writes bounded complete bytes, synchronizes, reopens, and verifies them; +3. links the stage to the canonical target without replacement; +4. synchronizes the store root; +5. removes the retained stage; and +6. synchronizes the store root again. + +The verified stage is linked without replacement. The canonical target is +immutable. Recovery never truncates, replaces, or repairs it. An exact stage +with an absent target resumes at the link. Exact stage and target bytes resume +at the required synchronization or cleanup. Different bytes, a substituted +inode, a link, or a wrong file kind refuse. + +A pre-effect incomplete stage may be removed only when its canonical target and +every later-ordered migration effect are absent and every earlier effect admits +exactly. Recovery pins the stage, removes it, synchronizes the store root, and +returns a typed discard report. Any later effect makes incomplete or corrupt +stage bytes unrecoverable ambiguity. + +The fixed stage is not authority. `migration.intent` becomes migration +authority only after its canonical link and store-root synchronization. +`migration.receipt` becomes completion evidence at the equivalent boundary. + +## Namespace prefix + +After durable intent publication, migration creates persistent `reader.lock` +and the exact nested directory prefix in the order specified by +[migration recovery](recovery.md). Each existing name must be the exact pinned +file or directory expected at that position. Each new nested name is followed +by synchronization of its parent. The final store-root synchronization admits +the complete prefix. A wrong kind, link, out-of-order name, or unknown entry +refuses. + +## Process-death matrix + +| Identifier | Boundary | +| --- | --- | +| `KEEP-CRASH-053` | migration-intent stage write | +| `KEEP-CRASH-054` | migration-intent stage synchronization | +| `KEEP-CRASH-055` | migration-intent canonical link | +| `KEEP-CRASH-056` | store-root synchronization after intent link | +| `KEEP-CRASH-057` | migration-intent stage removal | +| `KEEP-CRASH-058` | store-root synchronization after intent cleanup | +| `KEEP-CRASH-059` | persistent reader-fence creation | +| `KEEP-CRASH-060` | canonical nested directory-prefix creation | +| `KEEP-CRASH-061` | store-root synchronization after namespace creation | +| `KEEP-CRASH-062` | format-marker stage write | +| `KEEP-CRASH-063` | format-marker stage synchronization | +| `KEEP-CRASH-064` | format-marker canonical link | +| `KEEP-CRASH-065` | store-root synchronization after marker link | +| `KEEP-CRASH-066` | format-marker stage removal | +| `KEEP-CRASH-067` | store-root synchronization after marker cleanup | +| `KEEP-CRASH-068` | migration-receipt stage write | +| `KEEP-CRASH-069` | migration-receipt stage synchronization | +| `KEEP-CRASH-070` | migration-receipt canonical link | +| `KEEP-CRASH-071` | store-root synchronization after receipt link | +| `KEEP-CRASH-072` | migration-receipt stage removal | +| `KEEP-CRASH-073` | final store-root synchronization | + +Every identifier requires before, during, and after process-death evidence. +`KEEP-CRASH-060` additionally requires one case for every admitted directory +prefix length. Restart must classify exact stages, canonical targets, namespace +prefix, marker, receipt, and cleanup state without depending on a clock, +filesystem iteration order, or file existence alone. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md new file mode 100644 index 0000000..315bee8 --- /dev/null +++ b/docs/formats/segment-store-v2/rationale.md @@ -0,0 +1,88 @@ +# Format Rationale + +This note records choices local to `keep.segment-store/v2`. ADR-0009 remains +authoritative for the cross-cutting retention and GC decision. + +## Use a successor store version + +Extending the version-1 root shape was rejected. Version 1 deliberately refuses +unknown entries, so treating new retention state as optional would weaken its +admission law and make old readers misclassify a mutated store. A durable +migration intent creates an explicit authority boundary. + +Direct version-2 initialization was rejected for this version. Requiring one +admitted version-1 predecessor gives migration, compatibility, and recovery one +starting law instead of defining a second initialization protocol without a +consumer requirement. + +## Stage fixed migration records + +Writing `migration.intent`, `FORMAT`, or `migration.receipt` in place was +rejected because process death can expose partial canonical bytes. Exact +`.next` stages make incomplete bytes non-authoritative and publish each +canonical fixed record through an immutable no-replacement link. + +## Preserve version-1 immutable bytes + +Re-encoding segments, catalogs, or publication heads during migration was +rejected. Their bytes are already canonical and independently evidenced. +Preservation narrows migration to new authority and namespace state and permits +byte-for-byte rollback analysis without promising an automatic downgrade. + +## Keep namespace bytes out of paths + +Using caller namespace text as a directory name was rejected. Namespace bytes +may contain separators, zero bytes, or non-Unicode data and have no filesystem +semantics. A domain-separated digest supplies the physical coordinate while +the root record retains the exact bytes to detect collision or substitution. + +## Retain empty namespace generations + +Deleting empty namespaces was rejected because an old absent-state compare and +swap could become valid again. Persistent empty generations prevent that ABA +hazard. The fixed 4,096-namespace ceiling bounds the resulting manifest and +makes capacity refusal explicit. + +## Use one global manifest + +Enumerating mutable namespace directories during GC was rejected. One immutable +manifest binds the complete namespace map under a `LivenessGeneration`, so a +reader or GC planner cannot miss a concurrently created namespace. + +## Store semantic records, not serializer output + +Serde-defined persistence was rejected. Fixed headers, explicit widths, +big-endian integers, zero reserved bytes, canonical ordering, named digest +domains, and golden fixtures keep the protocol independent of Rust layout and +dependency defaults. + +## Start with one realization profile + +Version-2 catalogs expose one canonical location per logical record identity. +Pretending to support multiple representation policies would add an unproved +abstraction. The registered single-witness profile states the current law +exactly; another profile requires a successor specification and evidence. + +## Use a kernel reader fence + +A durable reader registry, lease, clock, and process liveness inference were +rejected. A persistent file with shared reader locks and an exclusive GC lock +has an observable process-death lifecycle. The fixed writer-then-reader lock +order avoids lock inversion. Publication does not take the reader lock because +it deletes no published immutable segment. Readers therefore double-collect +both mutable heads around transitive admission and reject a mixed view. + +## Derive logical store identity + +Random or physical-location store identifiers were rejected. A deterministic +digest of the admitted version-1 catalog, immutable pools, and target format +definition gives byte-identical stores one logical identity while the migration +intent separately binds physical coordinates for in-place recovery. + +## Reserve GC names but refuse their state + +Leaving the future GC namespace undefined was rejected because adding it later +would mutate the exact version-2 root grammar. Accepting placeholder bytes was +also rejected. Version 2 reserves the names, while their presence remains an +unsupported mandatory state until issue #21 supplies complete byte, parser, +crash, recovery, corruption, and fuzz evidence. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md new file mode 100644 index 0000000..871e675 --- /dev/null +++ b/docs/formats/segment-store-v2/recovery.md @@ -0,0 +1,285 @@ +# Migration and Recovery + +This page owns the version-2 filesystem namespace, format marker, reader fence, +one-way migration, fixed-stage recovery, GC reservation, and +recovery-disposition reservation. + +## Exact filesystem namespace + +Version 2 preserves the version-1 files and directories and admits these new +coordinates: + +```text +reader.lock +FORMAT +migration.intent +migration.intent.next +migration.receipt +migration.receipt.next +FORMAT.next +retention/HEAD +retention/head.next +retention/root.next +retention/manifest.next +retention/roots//-.root +retention/manifests/-.manifest +gc/intent +gc/receipt +recovery/disposition.next +recovery/dispositions/.receipt +``` + +`retention/HEAD`, every fixed `.next` stage, `gc/intent`, and `gc/receipt` are +optional according to the exact state tables below. Immutable-pool coordinates +are data-dependent but canonically named. Every other root or +protocol-directory entry is an unknown entry and unrecoverable ambiguity. +Operations are capability-relative and never follow links. + +## Format marker + +`FORMAT` is exactly 96 bytes: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:STORE:V2\0\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `96` | +| 20 | 4 | flags | `0` | +| 24 | 32 | format-definition digest | registered v2 digest | +| 56 | 4 | maximum namespace count | `4,096` | +| 60 | 4 | reserved | zero | +| 64 | 32 | checksum | BLAKE3-256 over bytes `0..64` | + +The definition and checksum domains are +`keep.segment-store-definition/v2\0` and +`keep.segment-store-marker-checksum/v2\0`. A missing marker is version 1 only +when the exact version-1 namespace admits. An unsupported, corrupt, +substituted, or same-name/different-digest marker refuses. + +## Reader fence + +`reader.lock` is a persistent regular zero-length file. Its contents and +existence alone prove nothing. + +A version-2 reader acquires a kernel-managed shared lock on `reader.lock` +before opening catalog `HEAD` or `retention/HEAD`. The returned `ReaderFence` +owns that lock for the complete snapshot lifetime. Close, drop, or process +death releases only the kernel lock and never deletes the persistent file. + +GC acquires the store writer authority and then an exclusive `reader.lock`, in +that fixed order. New readers wait and existing readers drain before GC +revalidation or physical deletion. Catalog and retention publication may +proceed beside readers because they publish immutable successors and delete no +published segment. + +## Migration records + +Migration is a one-way explicit migration under exclusive writer authority. +Version 1 is never extended in place without durable migration evidence. + +`migration.intent` is exactly 256 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:MIG:INT2\0\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `256` | +| 20 | 4 | flags | `0` | +| 24 | 8 | catalog generation named by version-1 `HEAD` | positive | +| 32 | 8 | catalog length named by version-1 `HEAD` | exact admitted length | +| 40 | 32 | catalog digest named by version-1 `HEAD` | exact admitted digest | +| 72 | 32 | predecessor catalog digest | zero for generation 1 | +| 104 | 32 | immutable-pool inventory digest | canonical complete inventory | +| 136 | 8 | root device identity | admitted platform value | +| 144 | 8 | root mount identity | admitted platform value | +| 152 | 8 | root file identity | admitted platform value | +| 160 | 32 | target format-definition digest | exact registered v2 digest | +| 192 | 32 | new store identifier | deterministic derivation below | +| 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | + + + +The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The pool +inventory digest uses `keep.store-v1-pool-inventory/v2\0` over the sorted, +duplicate-free canonical names, lengths, and verified content digests from +both immutable pools. The intent therefore binds the exact catalog generation, +length, and digest named by the admitted version-1 `HEAD`. + +The deterministically derived store identifier is: + +```text +BLAKE3-256("keep.store-identifier/v2\0" || + catalog-generation-u64 || + catalog-length-u64 || + catalog-digest || + predecessor-catalog-digest || + immutable-pool-inventory-digest || + target-format-definition-digest) +``` + +Integer fields use their fixed-width big-endian bytes. Root device, mount, file +identity, caller identity, path, and time do not enter the identifier. The +migration intent separately binds the physical root coordinates so in-place +recovery refuses a substituted store. + +`migration.receipt` is exactly 256 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:MIG:REC2\0\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `256` | +| 20 | 4 | flags | `0` | +| 24 | 32 | migration-intent digest | exact durable intent | +| 56 | 32 | store identifier | exact intent value | +| 88 | 32 | format-marker digest | exact verified marker | +| 120 | 32 | initial retention-state digest | exact no-payload digest below | +| 152 | 32 | initial GC-state digest | exact no-payload digest below | +| 184 | 32 | disposition namespace digest | exact no-payload digest below | +| 216 | 8 | completed synchronization mask | every mandatory bit set | +| 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | + + + +Its checksum domain is `keep.store-migration-receipt-checksum/v2\0`. Unknown +synchronization bits, a missing mandatory bit, or any mismatch with the intent +refuses. + +The three initial-state fields are the no-payload digests +`BLAKE3-256("keep.initial-retention-state/v2\0")`, +`BLAKE3-256("keep.initial-gc-state/v2\0")`, and +`BLAKE3-256("keep.empty-disposition-set/v2\0")`. At completed migration, +absence of `retention/HEAD` is the canonical empty retention state only while +all retention stages and pools are empty. Any retention artifact routes through +recovery instead. Direct version-2 initialization is undefined. + +The byte-exact offset tables and golden fixtures are requirements +`KEEP-MIGRATION-002` and `KEEP-MIGRATION-007`; no production writer exists +until those planned items become implemented evidence. + +## One-way migration protocol + +Migration performs these ordered steps: + +1. Admit and completely recover the exact version-1 store. +2. Revalidate its head, catalog, pools, root identity, and writer authority. +3. Publish `migration.intent` from `migration.intent.next` through the + no-replacement fixed-stage protocol. +4. Create and verify persistent `reader.lock`. +5. Create the exact `retention`, `retention/roots`, + `retention/manifests`, `gc`, `recovery`, and + `recovery/dispositions` directories. +6. Synchronize every created parent and the store root. +7. Publish `FORMAT` from `FORMAT.next` through the fixed-stage protocol. +8. Reopen and verify the complete version-2 view. +9. Publish `migration.receipt` from `migration.receipt.next` through the + fixed-stage protocol. + +The [migration crash-point specification](migration-crash.md) owns that +protocol and spans `KEEP-CRASH-053` through `KEEP-CRASH-073`. + +Migration never rewrites or deletes admitted version-1 immutable bytes and +provides no automatic downgrade. + +Version-1 admission refuses once any migration stage, `migration.intent`, +`reader.lock`, `FORMAT`, or version-2 directory is present. Once the canonical +intent is durable, only version-2 migration recovery may continue. + +## Partial migration recovery + +The migration recovery boundary admits only these ordered prefixes: + + + +| State | Required response | +| --- | --- | +| no migration artifact | admit exact version 1 | +| intent stage only | finalize an exact stage or explicitly discard an incomplete pre-effect stage | +| durable intent only | verify intent and continue | +| intent plus a canonical prefix of v2 names | verify each name and continue | +| complete v2 shape without marker | verify directories and write marker | +| marker without receipt | reopen full v2 view and publish receipt | +| exact receipt with optional exact receipt stage | clean the stage and admit complete migration | + + + +A partial migration retry revalidates the intent and every existing byte, +continues idempotently at the first absent canonical step, and never replaces +an existing entry. A missing predecessor, changed version-1 coordinate, +out-of-order name, wrong file kind, substituted byte, conflicting receipt, +unknown entry, or changed root identity is unrecoverable ambiguity. + +Process death before durable canonical intent leaves version 1 plus at most its +non-authoritative stage. Process death after durable intent leaves +recovery-required version-2 migration state. + +## Retention publication recovery + +At restart, a fixed retention stage is classified from its exact framing and +transitive evidence: + +The forward protocol guarantees that `root.next` is durable before a new +namespace directory is created. A new digest-named directory is created +exclusively, verified as the exact regular directory rather than a link, and +followed by synchronization of `retention/roots` before the immutable root is +linked. An existing exact directory is idempotent; any wrong kind, substituted +namespace, or unexpected entry refuses. Directory existence alone never proves +a retained root. + + + +| Fixed stage | Complete evidence | Recovery | +| --- | --- | --- | +| `root.next` | canonical successor root, matching namespace and closure proof | finalize its immutable pool link and retain the stage | +| `manifest.next` | canonical successor manifest naming only admitted roots | finalize its immutable pool link and retain both stages | +| `head.next` | canonical successor head naming the staged manifest | finalize the head, synchronize it, then remove retained stages | + + + +A pre-effect incomplete stage may be removed only when every later-ordered +effect is absent and all earlier evidence admits exactly. Recovery pins that +regular file, removes it, synchronizes `retention`, and returns a typed discard +report. Any later effect, stale generation, mismatched digest, missing +transitive member, reappeared stage, conflicting pool entry, or other +corruption is a typed refusal. A complete valid orphan remains +recovery-protected until explicit disposition. + +The retention crash points are: + +| Identifier | Boundary | +| --- | --- | +| `KEEP-CRASH-036` | root stage write | +| `KEEP-CRASH-037` | root stage synchronization | +| `KEEP-CRASH-038` | new namespace-directory creation or exact admission | +| `KEEP-CRASH-039` | namespace-pool synchronization after creation | +| `KEEP-CRASH-040` | immutable root link | +| `KEEP-CRASH-041` | root namespace-directory synchronization | +| `KEEP-CRASH-042` | manifest stage write | +| `KEEP-CRASH-043` | manifest stage synchronization | +| `KEEP-CRASH-044` | immutable manifest link | +| `KEEP-CRASH-045` | manifest pool synchronization | +| `KEEP-CRASH-046` | retention-head stage write | +| `KEEP-CRASH-047` | retention-head stage synchronization | +| `KEEP-CRASH-048` | retention-head atomic replacement | +| `KEEP-CRASH-049` | committed retention namespace synchronization | +| `KEEP-CRASH-050` | retained root-stage removal | +| `KEEP-CRASH-051` | retained manifest-stage removal | +| `KEEP-CRASH-052` | retention cleanup synchronization | + +Each point requires before, during, and after process-death evidence. Restart +must establish exact catalog visibility, retention head, namespace generation, +orphan classification, stage disposition, and recovery report. + +## GC and recovery-disposition recovery + +The [GC and disposition record specification](gc.md) owns the exact +`GcRetirementIntent`, `GcRetirementReceipt`, and +`RecoveryDispositionReceipt` grammars and state transitions. Issue #21 owns +their executable parser, corruption, crash, recovery, and fuzz evidence. +Issue #19 admits only the absent `gc/intent`, `gc/receipt`, +`recovery/disposition.next`, and disposition-receipt pool. Any presence is +unsupported mandatory state and refuses without mutation. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md new file mode 100644 index 0000000..b011772 --- /dev/null +++ b/docs/formats/segment-store-v2/requirements.md @@ -0,0 +1,63 @@ +# Requirements and Evidence + +This ledger owns stable requirements for `keep.segment-store/v2`. A planned +case is not evidence. + +## Retention transitions + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | unit and public API tests | Planned in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | golden-format fixtures plus independent oracle | Planned in #19 | +| `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | +| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | +| `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | retry and stale-successor tests | Planned in #19 | +| `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | + + + +## Migration + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | +| `KEEP-MIGRATION-002` | Intent and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | golden-format fixtures | Planned in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | +| `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | +| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | +| `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | `KEEP-CRASH-053..=073` crash-injection matrix | Planned in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | + + + +## Garbage collection reservation + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-GC-001` | Version 2 specifies exact bounded GC intent, receipt, and recovery-disposition grammars but refuses their presence until their parser and recovery protocol are implemented | namespace admission tests | Planned in #21 | +| `KEEP-GC-002` | GC intent, receipt, disposition, reader-fence, retirement, compaction, and recovery laws implement ADR-0009 without changing logical identity | golden-format, model-based, corruption, crash-injection, benchmark, and fuzz evidence | Planned in #21 | + + + +## Compatibility and nonclaims + +- Version 2 preserves exact version-1 segment, catalog, and publication-head + bytes. +- Migration is one-way and provides no downgrade. +- Retention evidence proves a bounded physical reconstruction claim, not + application meaning, causal ownership, future policy, or secure erasure. +- A version-2 format specification is not proof that a version-2 production + writer exists. +- Benchmarks are required before performance-sensitive retention or migration + optimization. diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md new file mode 100644 index 0000000..ca4811b --- /dev/null +++ b/docs/formats/segment-store-v2/retention.md @@ -0,0 +1,299 @@ +# Retention Records and Publication + +This page owns retention values, root-generation records, manifests, heads, +closure admission, and publication for `keep.segment-store/v2`. + +## Scalar and identity rules + +All integers are unsigned and big-endian. Reserved bytes and unassigned flags +are zero. Decoders reject unknown mandatory flags, nonzero reserved bytes, +truncation, trailing bytes, unsupported versions, noncanonical ordering, +duplicates, overflow, and values above a fixed limit. + +Checksums and durable digests use each record's named domain-separated +BLAKE3-256 profile, including the domain string's terminating zero byte. No +digest covers a serializer-owned value. + +### Retention namespace + +`RetentionNamespace` is an opaque, nonempty byte string of 1 through 255 bytes. +Every byte is admitted and canonical as-is; the value is not Unicode, a path, +an account, a process, or an application identity. No normalization, case +folding, alias, implicit namespace, or alternate encoding exists. + +The namespace digest is: + +```text +BLAKE3-256("keep.retention-namespace/v1\0" || + namespace-length-u16 || + namespace-bytes) +``` + +The 32-byte digest supplies the physical namespace-directory coordinate. The +root-generation record also stores the exact namespace bytes, so a digest +collision or substituted spelling refuses instead of aliasing two authorities. + +### Generations + +`RootGeneration` and `LivenessGeneration` are positive `u64` values. Generation +1 is initial. A successor is exactly the observed value plus one under checked +arithmetic. Zero and overflow refuse. An empty root set remains a new +`RootGeneration`; namespace identity and generation history are never deleted +or reused in version 2. + +The maximum admitted namespace count is 4,096, including current manifest +namespaces, empty generations, and recovery-protected orphan namespace +directories; directory existence alone is not authority. Admission computes +the attempted total with checked arithmetic and refuses above that maximum +before any namespace-generation or manifest bytes are staged. Existing +namespaces may transition while the store is at capacity. + +### Reconstruction anchor + +One anchor is exactly 119 bytes: + +| Offset | Width | Field | +| ---: | ---: | --- | +| 0 | 59 | canonical `BlobId` binary bytes | +| 59 | 60 | canonical `LayoutId` binary bytes | + +Anchors are ordered by the lexicographic order of their complete canonical +bytes. The set is sorted, duplicate-free before admission. The maximum +anchor count in one namespace generation is 65,536. + +### Realization profile and limits + +Version 2 admits one realization profile: + +- identity `1`; +- version `1`; +- canonical name `keep.retention-single-canonical-witness/v1`; +- exact witness count `1` for each layout and chunk identity; and +- selection by canonical physical catalog coordinate. + +The stored profile coordinate is the `u32` identity, `u32` version, and +BLAKE3-256 digest of its canonical definition bytes. Any unknown or mismatched +coordinate refuses. A future profile requires a successor specification. + +Each root generation stores caller-selected limits no greater than these +implementation ceilings: + +| Limit | Ceiling | +| --- | ---: | +| anchors | 65,536 | +| closure nodes | 1,048,576 | +| closure depth | 8 | +| encoded bytes inspected | 16,777,216 | +| physical bytes inspected | 1,073,741,824 | + +All limits are positive. Cross-field validation and the ceiling check complete +before traversal or materialization. + +## Root-generation record + +One root-generation file is: + +```text +192-byte fixed-width header +namespace bytes +anchor-count × 119-byte anchors +32-byte root digest +32-byte checksum +``` + +Its total maximum length is 7,799,295 bytes. + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:RET:ROOT2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | header length | `192` | +| 20 | 4 | flags | `0` | +| 24 | 8 | total record length | derived exact length | +| 32 | 8 | root generation | positive | +| 40 | 2 | namespace length | `1..=255` | +| 42 | 2 | anchor width | `119` | +| 44 | 4 | anchor count | `0..=65,536` | +| 48 | 4 | profile identity | `1` | +| 52 | 4 | profile version | `1` | +| 56 | 32 | profile-definition digest | registered exact digest | +| 88 | 8 | closure-node limit | positive and at most ceiling | +| 96 | 2 | closure-depth limit | positive and at most ceiling | +| 98 | 2 | reserved | zero | +| 100 | 8 | encoded-byte limit | positive and at most ceiling | +| 108 | 8 | physical-byte limit | positive and at most ceiling | +| 116 | 32 | predecessor root digest | zero for generation 1 | +| 148 | 32 | anchor-set digest | exact body-anchor digest | +| 180 | 12 | reserved | zero | + + + +The anchor-set digest is: + +```text +BLAKE3-256("keep.retention-anchor-set/v2\0" || + anchor-count-u32 || + canonical-anchor-bytes) +``` + +The root digest covers the header and body: + +```text +BLAKE3-256("keep.retention-root/v2\0" || header || body) +``` + +The checksum covers the header, body, and root digest: + +```text +BLAKE3-256("keep.retention-root-checksum/v2\0" || + header || body || root-digest) +``` + +The pool coordinate is: + +```text +retention/roots// + -.root +``` + +Names with alternate width, case, suffix, generation, or digest refuse. + +## Global retention manifest + +One manifest binds every admitted namespace to its exact root generation and +canonical digest: + +```text +160-byte fixed-width header +entry-count × 72-byte entries +32-byte manifest digest +32-byte checksum +``` + +Each entry is: + +| Offset | Width | Field | +| ---: | ---: | --- | +| 0 | 32 | namespace digest | +| 32 | 8 | root generation | +| 40 | 32 | root digest | + +Entries are sorted by namespace digest and duplicate-free. The maximum entry +count is 4,096 and the maximum manifest length is 295,136 bytes. + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:RET:LIVE2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | header length | `160` | +| 20 | 4 | flags | `0` | +| 24 | 8 | total record length | derived exact length | +| 32 | 8 | liveness generation | positive | +| 40 | 2 | entry width | `72` | +| 42 | 2 | reserved | zero | +| 44 | 4 | entry count | `0..=4,096` | +| 48 | 32 | predecessor manifest digest | zero for generation 1 | +| 80 | 32 | entry-set digest | exact canonical entries | +| 112 | 48 | reserved | zero | + + + +The entry-set, manifest, and checksum domains are respectively: + +```text +keep.retention-manifest-entries/v2\0 +keep.retention-manifest/v2\0 +keep.retention-manifest-checksum/v2\0 +``` + +The manifest pool coordinate is: + +```text +retention/manifests/ + -.manifest +``` + +## Retention head + +`retention/HEAD` and `retention/head.next` use one exact 144-byte record: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:RET:HEAD2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `144` | +| 20 | 4 | flags | `0` | +| 24 | 8 | liveness generation | positive | +| 32 | 8 | manifest length | exact admitted length | +| 40 | 32 | manifest digest | exact pool digest | +| 72 | 32 | predecessor manifest digest | zero for generation 1 | +| 104 | 8 | reserved | zero | +| 112 | 32 | checksum | BLAKE3-256 over bytes `0..112` | + +The checksum domain is `keep.retention-head-checksum/v2\0`. + +## Closure admission + +Before publication, Keep pins one completely verified catalog generation and +derives the complete closure for every anchor: + +1. Resolve and admit the exact layout record named by `LayoutId`. +2. Require its embedded `BlobId` to equal the anchor `BlobId`. +3. Resolve and admit every ordered chunk identity required by that layout. +4. Verify each physical record, identity, checksum, digest, and catalog + coordinate under the stored realization profile. +5. Enforce the stored limits with checked counters and a visited set. +6. Reconstruct and authenticate the complete blob identity. + +A missing or corrupt closure member, ambiguous catalog claim, unsupported +profile, limit breach, cycle, unknown mandatory edge, identity mismatch, or +ordering error refuses the entire transition. Keep never omits one failed +member and continues with a smaller live set. + +Version-2 catalog publication holds the same writer authority and proves every +current retained closure against its candidate catalog before replacing the +catalog `HEAD`. + +## Generation transition + +A transition supplies a namespace, an expected state of absent or one exact +`RootGeneration`, a complete canonical anchor set, the exact realization +profile coordinate, and admitted limits. + +Under exclusive writer authority, publication: + +1. completes recovery of every fixed retention stage; +2. admits the current retention head, manifest, and selected namespace root; +3. compares expected and observed generations; +4. verifies the candidate closure against one pinned catalog; +5. writes and synchronizes `retention/root.next`; +6. for a new namespace, exclusively creates and verifies its exact digest-named + directory, then synchronizes `retention/roots`; +7. links and verifies the root pool entry, then synchronizes its directory; +8. writes and synchronizes `retention/manifest.next`; +9. links and verifies the manifest pool entry and synchronizes its directory; +10. writes and synchronizes `retention/head.next`; +11. atomically replaces `retention/HEAD` and synchronizes `retention`; + `root.next` and `manifest.next` remain durable until the retention head + commits, then are removed and `retention` is synchronized again; and +12. returns a consequential `#[must_use]` receipt. + +The receipt binds the namespace, expected and observed generations, committed +root generation and digest, global manifest generation and digest, profile +coordinate, anchor-set and closure digests, catalog generation and digest, and +every durable publication outcome. + +A stale transition preserves expected and observed generations. A +byte-identical retry returns **already committed** only while that exact root +successor remains current; otherwise it returns the precise stale state. + +A reader holds one shared `ReaderFence` and double-collects the catalog and +retention heads around complete transitive admission. It accepts only the same +coordinates before and after for both heads. Any generation, length, digest, or +checksum change discards the view and retries within a bounded attempt limit; +exhaustion refuses. The accepted view observes one complete root generation for +its snapshot lifetime. diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs new file mode 100644 index 0000000..7c80151 --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -0,0 +1,228 @@ +//! Written-contract evidence for the version-2 retention store. + +#![cfg(feature = "repository-tasks")] + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +const FORMAT_ROOT: &str = "docs/formats/segment-store-v2"; +const DOCUMENT_REVIEW_LIMIT_LINES: usize = 300; + +fn repository_root() -> Result { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| io::Error::other("xtask manifest directory has no parent")) +} + +fn read(relative: &str) -> Result { + fs::read_to_string(repository_root()?.join(relative)) +} + +fn normalized(document: &str) -> String { + document.split_whitespace().collect::>().join(" ") +} + +#[test] +fn version_two_is_one_routed_protocol() -> Result<(), Box> { + let format_index = read("docs/formats/README.md")?; + let changelog = read("CHANGELOG.md")?; + let overview = normalized(&read(&format!("{FORMAT_ROOT}/README.md"))?); + + assert!( + format_index.contains("segment-store-v2/README.md"), + "format index does not route to segment-store v2" + ); + assert!( + changelog.contains("`keep.segment-store/v2`"), + "changelog does not record the segment-store v2 contract" + ); + for required in [ + "`keep.segment-store/v2`", + "successor to `keep.segment-store/v1`", + "[Retention records and publication](retention.md)", + "[GC and disposition records](gc.md)", + "[Migration and recovery](recovery.md)", + "[Migration crash points](migration-crash.md)", + "[Requirements and evidence](requirements.md)", + "[Format rationale](rationale.md)", + ] { + assert!( + overview.contains(required), + "segment-store v2 overview omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn retention_records_have_exact_canonical_grammars() -> Result<(), Box> { + let retention = normalized(&read(&format!("{FORMAT_ROOT}/retention.md"))?); + + for required in [ + "`RetentionNamespace`", + "1 through 255 bytes", + "`RootGeneration`", + "`LivenessGeneration`", + "big-endian", + "fixed-width header", + "sorted, duplicate-free", + "BLAKE3-256", + "domain-separated", + "trailing bytes", + "unknown mandatory flags", + "maximum admitted namespace count", + "before any namespace-generation or manifest bytes are staged", + "expected and observed generations", + "already committed", + "one complete root generation", + "remain durable until the retention head", + "double-collects the catalog and retention heads", + "same coordinates before and after", + ] { + assert!( + retention.contains(required), + "segment-store v2 retention grammar omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> +{ + let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); + + for required in [ + "one-way explicit migration", + "`migration.intent`", + "`migration.intent.next`", + "`migration.receipt`", + "`migration.receipt.next`", + "`FORMAT.next`", + "`migration.intent` is exactly 256 bytes", + "`migration.receipt` is exactly 256 bytes", + "catalog generation, length, and digest", + "deterministically derived store identifier", + "absence of `retention/HEAD` is the canonical empty retention state", + "pre-effect incomplete stage", + "keep.initial-retention-state/v2\\0", + "keep.initial-gc-state/v2\\0", + "keep.empty-disposition-set/v2\\0", + "root.next` is durable before a new namespace directory", + "`KEEP-CRASH-036`", + "`KEEP-CRASH-073`", + "partial migration", + "Version-1 admission refuses", + "`reader.lock`", + "`GcRetirementIntent`", + "`GcRetirementReceipt`", + "`RecoveryDispositionReceipt`", + "unknown entry", + "unrecoverable ambiguity", + "idempotent", + "process death", + ] { + assert!( + recovery.contains(required), + "segment-store v2 recovery contract omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> +{ + let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); + + for required in [ + "never writes canonical fixed names in place", + "`migration.intent.next`", + "`FORMAT.next`", + "`migration.receipt.next`", + "linked without replacement", + "pre-effect incomplete stage", + "`KEEP-CRASH-053`", + "`KEEP-CRASH-073`", + "before, during, and after process-death evidence", + ] { + assert!( + migration.contains(required), + "segment-store v2 migration crash protocol omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn gc_records_are_bounded_before_their_implementation() -> Result<(), Box> { + let gc = normalized(&read(&format!("{FORMAT_ROOT}/gc.md"))?); + + for required in [ + "`GcRetirementIntent`", + "320-byte fixed-width header", + "72-byte candidate entries", + "65,536", + "`GcRetirementReceipt`", + "exactly 320 bytes", + "`RecoveryDispositionReceipt`", + "canonical absent candidate prefix", + "unrecoverable ambiguity", + "Planned in #21", + ] { + assert!( + gc.contains(required), + "segment-store v2 GC grammar omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn requirement_ledger_names_planned_and_executable_evidence() +-> Result<(), Box> { + let requirements = normalized(&read(&format!("{FORMAT_ROOT}/requirements.md"))?); + + for required in [ + "`KEEP-RETENTION-001`", + "`KEEP-RETENTION-010`", + "`KEEP-MIGRATION-001`", + "`KEEP-MIGRATION-008`", + "`KEEP-GC-001`", + "Planned in #19", + "Planned in #21", + "golden-format", + "model-based", + "corruption", + "crash-injection", + "fuzz", + ] { + assert!( + requirements.contains(required), + "segment-store v2 requirement ledger omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box> { + for name in [ + "README.md", + "gc.md", + "migration-crash.md", + "rationale.md", + "recovery.md", + "requirements.md", + "retention.md", + ] { + let line_count = read(&format!("{FORMAT_ROOT}/{name}"))?.lines().count(); + assert!( + line_count <= DOCUMENT_REVIEW_LIMIT_LINES, + "{name} has {line_count} lines; review threshold is {DOCUMENT_REVIEW_LIMIT_LINES}" + ); + } + Ok(()) +} From b1b4f23467c89542eb5d55c9936c7d0575e922db Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 19:45:55 -0700 Subject: [PATCH 02/50] Test: Freeze version two retention bytes --- conformance/segment-store/v2/ORIGIN.md | 67 ++++++ conformance/segment-store/v2/README.md | 74 +++++++ conformance/segment-store/v2/artifacts.tsv | 8 + conformance/segment-store/v2/definition.tsv | 90 ++++++++ .../segment-store/v2/format-marker.hex | 1 + conformance/segment-store/v2/inventory.tsv | 4 + .../segment-store/v2/migration-intent.hex | 1 + .../segment-store/v2/migration-receipt.hex | 1 + .../segment-store/v2/migration-source.tsv | 3 + .../segment-store/v2/one-anchor-root.hex | 1 + .../segment-store/v2/one-root-head.hex | 1 + .../segment-store/v2/one-root-manifest.hex | 1 + .../segment-store/v2/retention-profile.tsv | 3 + docs/formats/README.md | 2 +- docs/formats/segment-store-v2/README.md | 6 + .../segment-store-v2/migration-crash.md | 23 ++ .../segment-store-v2/migration-inventory.md | 49 +++++ docs/formats/segment-store-v2/recovery.md | 15 +- docs/formats/segment-store-v2/retention.md | 18 +- ...retention_store_v2_conformance_contract.rs | 61 ++++++ .../tests/retention_store_v2_format_oracle.rs | 75 +++++++ .../artifacts.rs | 145 +++++++++++++ .../artifacts/migration.rs | 96 +++++++++ .../artifacts/retention.rs | 155 ++++++++++++++ .../encoding.rs | 126 +++++++++++ .../fixture_assertion.rs | 198 ++++++++++++++++++ .../retention_store_v2_protocol_contract.rs | 11 + 27 files changed, 1220 insertions(+), 15 deletions(-) create mode 100644 conformance/segment-store/v2/ORIGIN.md create mode 100644 conformance/segment-store/v2/README.md create mode 100644 conformance/segment-store/v2/artifacts.tsv create mode 100644 conformance/segment-store/v2/definition.tsv create mode 100644 conformance/segment-store/v2/format-marker.hex create mode 100644 conformance/segment-store/v2/inventory.tsv create mode 100644 conformance/segment-store/v2/migration-intent.hex create mode 100644 conformance/segment-store/v2/migration-receipt.hex create mode 100644 conformance/segment-store/v2/migration-source.tsv create mode 100644 conformance/segment-store/v2/one-anchor-root.hex create mode 100644 conformance/segment-store/v2/one-root-head.hex create mode 100644 conformance/segment-store/v2/one-root-manifest.hex create mode 100644 conformance/segment-store/v2/retention-profile.tsv create mode 100644 docs/formats/segment-store-v2/migration-inventory.md create mode 100644 xtask/tests/retention_store_v2_conformance_contract.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/encoding.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs diff --git a/conformance/segment-store/v2/ORIGIN.md b/conformance/segment-store/v2/ORIGIN.md new file mode 100644 index 0000000..305c4d2 --- /dev/null +++ b/conformance/segment-store/v2/ORIGIN.md @@ -0,0 +1,67 @@ +# Version 2 Corpus Origin + +The corpus was constructed on 2026-07-29 with: + +- `rustc 1.96.0 (ac68faa20 2026-05-25)`; +- `cargo 1.96.0 (30a34c682 2026-05-25)`; and +- `b3sum 1.8.5`. + +## Independent inputs + +The oracle imports exact bytes only from these previously accepted fixtures: + +- `conformance/segment-store/v1/one-zero-segment.hex`; +- `conformance/segment-store/v1/one-zero-catalog.hex`; +- `conformance/segment-store/v1/one-zero-head.hex`; +- the one-zero `BlobId` canonical text and `LayoutId` binary identity from + `conformance/layout/v1/layouts.tsv`. + +It parses the version-1 head coordinate, catalog predecessor, and segment and +catalog semantic digests directly from fixed offsets. The oracle constructs the +59-byte `BlobId` from the accepted binary grammar and verifies its length and +digest against the layout table; the table directly supplies the 60-byte +`LayoutId`. It does not call a production encoder, decoder, retention type, +migration adapter, serializer, or filesystem implementation. + +## Definition verification + +The profile digest was checked independently with: + +```bash +{ + printf 'keep.retention-realization-profile/v1\0' + cat conformance/segment-store/v2/retention-profile.tsv +} | b3sum --no-names +``` + +Exact output: + +```text +db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59 +``` + +The format-definition digest was checked independently with: + +```bash +{ + printf 'keep.segment-store-definition/v2\0' + cat conformance/segment-store/v2/definition.tsv +} | b3sum --no-names +``` + +Exact output: + +```text +32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427 +``` + +## Materialization boundary + +A temporary ignored Rust test wrote the initially reviewed TSV and hexadecimal +artifacts from the handwritten oracle. That write path was removed immediately +after materialization. The committed oracle is read-only and rejects drift. + +Changing any fixture requires a deliberate specification change, an updated +definition or profile digest when affected, fresh independent construction, +and review of every dependent migration and retention coordinate. A fixture is +never regenerated to make a production implementation pass. diff --git a/conformance/segment-store/v2/README.md b/conformance/segment-store/v2/README.md new file mode 100644 index 0000000..7417eb2 --- /dev/null +++ b/conformance/segment-store/v2/README.md @@ -0,0 +1,74 @@ +# Durable Segment Store Version 2 Corpus + +This corpus freezes independent canonical inputs and golden bytes for +`keep.segment-store/v2`. It proves the written format has one executable byte +interpretation. It does not prove that a production encoder, decoder, +migration, retention transition, or garbage collector exists. + +## Corpus files + +| File | Contract | +| --- | --- | +| `definition.tsv` | Sorted format-definition key/value bytes | +| `retention-profile.tsv` | Registered realization-profile definition | +| `inventory.tsv` | Canonical one-segment, one-catalog migration inventory | +| `migration-source.tsv` | Exact version-1 and derived migration coordinates | +| `artifacts.tsv` | Golden artifact lengths, digests, checksums, and filenames | +| `format-marker.hex` | Canonical 96-byte `FORMAT` record | +| `migration-intent.hex` | Canonical 256-byte migration intent | +| `migration-receipt.hex` | Canonical 256-byte migration receipt | +| `one-anchor-root.hex` | Generation-1 root with one nontext namespace | +| `one-root-manifest.hex` | Generation-1 one-namespace manifest | +| `one-root-head.hex` | Generation-1 retention head | +| `ORIGIN.md` | Construction provenance and verification boundary | + +Every text file uses UTF-8 or ASCII, LF line endings, and one final newline. +Every hex fixture is one lowercase hexadecimal line with one final newline. +In `artifacts.tsv`, `bound_digest_hex` is the marker content digest for +`format-marker`, the intent digest for `migration-intent`, the referenced +intent digest for `migration-receipt`, the canonical record digest for +`retention-root` and `retention-manifest`, and the referenced manifest digest +for `retention-head`. + +## Frozen identities + +The realization-profile digest is +`db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59`. +It hashes the exact `retention-profile.tsv` bytes under the registered profile +domain. + +The format-definition digest is +`32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427`. +It hashes the exact `definition.tsv` bytes under the registered format domain. +The definition binds the profile digest, every named domain, magic, version, +field order, record width, format limit, and migration synchronization mask. + +The migration fixture preserves the version-1 one-zero segment and generation-1 +catalog. Its canonical two-entry inventory digest is +`40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9`. +The derived logical store identifier is +`0cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd79`. +Fixture-only root device, mount, and file coordinates are `1`, `2`, and `3`; +they bind in-place recovery but do not enter the logical store identifier. + +The retention fixture uses namespace bytes `00 2f ff`, proving the namespace is +opaque and not a path or Unicode string. Its one anchor combines the canonical +one-zero `BlobId` and `LayoutId` values from the existing layout corpus. + +## Verification + +Run: + +```bash +cargo test --manifest-path xtask/Cargo.toml \ + --test retention_store_v2_format_oracle +``` + +The test-only oracle constructs every record from handwritten offsets and +domain preimages, compares exact fixture bytes and tables, and imports no +production version-2 codec. The repository protocol and documentation gates +route this corpus separately. + +Passing this corpus is necessary but insufficient for issue #19. Production +code still needs parser, corruption, property, model, crash, recovery, +concurrency, fuzz, and public API evidence. diff --git a/conformance/segment-store/v2/artifacts.tsv b/conformance/segment-store/v2/artifacts.tsv new file mode 100644 index 0000000..dace87c --- /dev/null +++ b/conformance/segment-store/v2/artifacts.tsv @@ -0,0 +1,8 @@ +keep.segment-store-v2.artifacts/v1 +case kind byte_length generation entry_count bound_digest_hex final_checksum_hex fixture +format-marker format-marker 96 - - 4b063c329085abdebe86b256d531b112c7ea33cb2f545caa40a7a869ff3337ce 06384cbaf2b69e0a12eeb2bf62df4c49e193d56f2bde940b3c5637320458abc1 format-marker.hex +migration-intent migration-intent 256 1 2 a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b44 7bec10cc8c1eef5ab0e8e8b6a33240bba291252d4263147df134062eb70d3f1f migration-intent.hex +migration-receipt migration-receipt 256 1 2 a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b44 3a6a5f29bfafeffb9401de5ba814c09c345adbad69e8ba0531e3eb1ebb0b681d migration-receipt.hex +one-anchor-root retention-root 378 1 1 ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b60085 28c52ff0f8d6533234be083f425e921d699639e204e2c66dec0cae2ff0a2dc34 one-anchor-root.hex +one-root-manifest retention-manifest 296 1 1 f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb 10597643c3fc9485c7ecd3bb511d6726e726fd92f0f769a204b899c5fdc77d2c one-root-manifest.hex +one-root-head retention-head 144 1 1 f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb ac049edb33af7e957c6ff11ead7e1bcf9c40fa9793cc84979215ffbba5f630b7 one-root-head.hex diff --git a/conformance/segment-store/v2/definition.tsv b/conformance/segment-store/v2/definition.tsv new file mode 100644 index 0000000..cea0627 --- /dev/null +++ b/conformance/segment-store/v2/definition.tsv @@ -0,0 +1,90 @@ +keep.segment-store.definition/v2 +key value +domain.empty-disposition-set keep.empty-disposition-set/v2\0 +domain.format-definition keep.segment-store-definition/v2\0 +domain.format-marker keep.store-format-marker/v2\0 +domain.format-marker-checksum keep.segment-store-marker-checksum/v2\0 +domain.gc-candidate-set keep.gc-candidate-set/v2\0 +domain.gc-intent keep.gc-retirement-intent/v2\0 +domain.gc-intent-checksum keep.gc-retirement-intent-checksum/v2\0 +domain.gc-receipt-checksum keep.gc-retirement-receipt-checksum/v2\0 +domain.initial-gc-state keep.initial-gc-state/v2\0 +domain.initial-retention-state keep.initial-retention-state/v2\0 +domain.migration-intent keep.store-migration-intent/v2\0 +domain.migration-intent-checksum keep.store-migration-intent-checksum/v2\0 +domain.migration-inventory keep.store-v1-pool-inventory/v2\0 +domain.migration-receipt-checksum keep.store-migration-receipt-checksum/v2\0 +domain.recovery-disposition-checksum keep.recovery-disposition-receipt-checksum/v2\0 +domain.retention-anchor-set keep.retention-anchor-set/v2\0 +domain.retention-head-checksum keep.retention-head-checksum/v2\0 +domain.retention-manifest keep.retention-manifest/v2\0 +domain.retention-manifest-checksum keep.retention-manifest-checksum/v2\0 +domain.retention-manifest-entries keep.retention-manifest-entries/v2\0 +domain.retention-namespace keep.retention-namespace/v1\0 +domain.retention-profile keep.retention-realization-profile/v1\0 +domain.retention-root keep.retention-root/v2\0 +domain.retention-root-checksum keep.retention-root-checksum/v2\0 +domain.store-identifier keep.store-identifier/v2\0 +format.coordinate keep.segment-store/v2 +format.marker.fields magic:16,version:u16,record_length:u16,flags:u32,definition_digest:32,maximum_namespace_count:u32,reserved:u32,checksum:32 +format.marker.length 96 +format.marker.magic KEEP:STORE:V2\0\0\0 +format.marker.version 2 +gc.intent.candidate-width 72 +gc.intent.fields magic:16,version:u16,header_length:u16,flags:u32,total_length:u64,generation:u64,candidate_width:u16,reserved:u16,candidate_count:u32,liveness_generation:u64,manifest_digest:32,catalog_generation:u64,catalog_digest:32,profile_identity:u32,profile_version:u32,profile_digest:32,catalog_proof_digest:32,pool_digest:32,disposition_set_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,candidate_set_digest:32,candidates:count*72,intent_digest:32,checksum:32 +gc.intent.header-length 320 +gc.intent.magic KEEP:GC:INTENT2\0 +gc.intent.maximum-candidates 65536 +gc.intent.maximum-length 4718976 +gc.intent.version 2 +gc.receipt.fields magic:16,version:u16,record_length:u16,flags:u32,generation:u64,intent_digest:32,retired_set_digest:32,pool_state_digest:32,liveness_generation:u64,manifest_digest:32,catalog_generation:u64,catalog_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,synchronization_count:u64,reserved:48,checksum:32 +gc.receipt.length 320 +gc.receipt.magic KEEP:GC:RECEIPT2 +gc.receipt.version 2 +migration.intent.fields magic:16,version:u16,record_length:u16,flags:u32,catalog_generation:u64,catalog_length:u64,catalog_digest:32,predecessor_digest:32,inventory_digest:32,root_device:u64,root_mount:u64,root_file:u64,definition_digest:32,store_id:32,checksum:32 +migration.intent.length 256 +migration.intent.magic KEEP:MIG:INT2\0\0\0 +migration.intent.version 2 +migration.inventory.entry-fields kind:u8,reserved:7,catalog_generation:u64,artifact_length:u64,artifact_digest:32 +migration.inventory.entry-width 56 +migration.inventory.maximum-entries 2097152 +migration.receipt.fields magic:16,version:u16,record_length:u16,flags:u32,intent_digest:32,store_id:32,format_marker_digest:32,initial_retention_digest:32,initial_gc_digest:32,disposition_set_digest:32,synchronization_mask:u64,checksum:32 +migration.receipt.length 256 +migration.receipt.magic KEEP:MIG:REC2\0\0\0 +migration.receipt.synchronization-mask 0x00000000000003ff +migration.receipt.version 2 +recovery.disposition.fields magic:16,version:u16,record_length:u16,flags:u32,artifact_kind:u16,decision:u16,classification:u16,reserved:u16,artifact_length:u64,artifact_identity_digest:32,artifact_content_digest:32,publication_generation:u64,publication_checksum:32,catalog_generation:u64,catalog_digest:32,liveness_generation:u64,manifest_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,decision_evidence_digest:32,reserved:8,checksum:32 +recovery.disposition.length 320 +recovery.disposition.magic KEEP:REC:DISP2\0\0 +recovery.disposition.maximum-receipts 65536 +recovery.disposition.version 2 +retention.anchor.fields blob_id:59,layout_id:60 +retention.anchor.width 119 +retention.closure.maximum-depth 8 +retention.closure.maximum-encoded-bytes 16777216 +retention.closure.maximum-nodes 1048576 +retention.closure.maximum-physical-bytes 1073741824 +retention.head.fields magic:16,version:u16,record_length:u16,flags:u32,liveness_generation:u64,manifest_length:u64,manifest_digest:32,predecessor_manifest_digest:32,reserved:u64,checksum:32 +retention.head.length 144 +retention.head.magic KEEP:RET:HEAD2\0\0 +retention.head.version 2 +retention.manifest.entry-fields namespace_digest:32,root_generation:u64,root_digest:32 +retention.manifest.entry-width 72 +retention.manifest.fields magic:16,version:u16,header_length:u16,flags:u32,total_length:u64,liveness_generation:u64,entry_width:u16,reserved:u16,entry_count:u32,predecessor_digest:32,entry_set_digest:32,reserved:48,entries:count*72,manifest_digest:32,checksum:32 +retention.manifest.header-length 160 +retention.manifest.magic KEEP:RET:LIVE2\0\0 +retention.manifest.maximum-entries 4096 +retention.manifest.maximum-length 295136 +retention.manifest.version 2 +retention.maximum-namespaces 4096 +retention.namespace.maximum-length 255 +retention.namespace.minimum-length 1 +retention.profile.digest db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59 +retention.profile.identity 1 +retention.profile.version 1 +retention.root.fields magic:16,version:u16,header_length:u16,flags:u32,total_length:u64,root_generation:u64,namespace_length:u16,anchor_width:u16,anchor_count:u32,profile_identity:u32,profile_version:u32,profile_digest:32,closure_node_limit:u64,closure_depth_limit:u16,reserved:u16,encoded_byte_limit:u64,physical_byte_limit:u64,predecessor_digest:32,anchor_set_digest:32,reserved:12,namespace:namespace_length,anchors:count*119,root_digest:32,checksum:32 +retention.root.header-length 192 +retention.root.magic KEEP:RET:ROOT2\0\0 +retention.root.maximum-anchors 65536 +retention.root.maximum-length 7799295 +retention.root.version 2 diff --git a/conformance/segment-store/v2/format-marker.hex b/conformance/segment-store/v2/format-marker.hex new file mode 100644 index 0000000..30640a9 --- /dev/null +++ b/conformance/segment-store/v2/format-marker.hex @@ -0,0 +1 @@ +4b4545503a53544f52453a5632000000000200600000000032381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427000010000000000006384cbaf2b69e0a12eeb2bf62df4c49e193d56f2bde940b3c5637320458abc1 diff --git a/conformance/segment-store/v2/inventory.tsv b/conformance/segment-store/v2/inventory.tsv new file mode 100644 index 0000000..cd0443a --- /dev/null +++ b/conformance/segment-store/v2/inventory.tsv @@ -0,0 +1,4 @@ +keep.segment-store-v2.inventory/v1 +kind generation byte_length artifact_digest_hex source_fixture +segment 0 337 b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc one-zero-segment.hex +catalog 1 352 04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320 one-zero-catalog.hex diff --git a/conformance/segment-store/v2/migration-intent.hex b/conformance/segment-store/v2/migration-intent.hex new file mode 100644 index 0000000..5ce426b --- /dev/null +++ b/conformance/segment-store/v2/migration-intent.hex @@ -0,0 +1 @@ +4b4545503a4d49473a494e543200000000020100000000000000000000000001000000000000016004b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320000000000000000000000000000000000000000000000000000000000000000040bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f900000000000000010000000000000002000000000000000332381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b8734270cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd797bec10cc8c1eef5ab0e8e8b6a33240bba291252d4263147df134062eb70d3f1f diff --git a/conformance/segment-store/v2/migration-receipt.hex b/conformance/segment-store/v2/migration-receipt.hex new file mode 100644 index 0000000..66b524e --- /dev/null +++ b/conformance/segment-store/v2/migration-receipt.hex @@ -0,0 +1 @@ +4b4545503a4d49473a524543320000000002010000000000a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b440cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd794b063c329085abdebe86b256d531b112c7ea33cb2f545caa40a7a869ff3337ced52f1f022edb1de7b840c5bf8fb55de7932ca69370ae85e2bee4179143792bc3ba0ea200a5b06741564c43a79a91945bef0b0fac51c960ea4f8207094f3e1e31a80259fcd1237203ea6c6cc5065514abdeb01da603c3194b096a045cf694c95a00000000000003ff3a6a5f29bfafeffb9401de5ba814c09c345adbad69e8ba0531e3eb1ebb0b681d diff --git a/conformance/segment-store/v2/migration-source.tsv b/conformance/segment-store/v2/migration-source.tsv new file mode 100644 index 0000000..5d0204b --- /dev/null +++ b/conformance/segment-store/v2/migration-source.tsv @@ -0,0 +1,3 @@ +keep.segment-store-v2.migration-source/v1 +case catalog_generation catalog_length catalog_digest_hex predecessor_digest_hex inventory_digest_hex definition_digest_hex store_id_hex root_device root_mount root_file +one-zero 1 352 04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320 0000000000000000000000000000000000000000000000000000000000000000 40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9 32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427 0cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd79 1 2 3 diff --git a/conformance/segment-store/v2/one-anchor-root.hex b/conformance/segment-store/v2/one-anchor-root.hex new file mode 100644 index 0000000..caeb194 --- /dev/null +++ b/conformance/segment-store/v2/one-anchor-root.hex @@ -0,0 +1 @@ +4b4545503a5245543a524f4f54320000000200c000000000000000000000017a000000000000000100030077000000010000000100000001db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59000000000000000400020000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000000000227f6333d1bcc380899ba25903b5d7d2b8804cc828e8f580b5876b0677d024f5000000000000000000000000002fff4b4545503a424c4f423a49440000000000010100000000000000011cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b6064b4545503a4c41594f55543a494400000001000100000000000000dc887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b6008528c52ff0f8d6533234be083f425e921d699639e204e2c66dec0cae2ff0a2dc34 diff --git a/conformance/segment-store/v2/one-root-head.hex b/conformance/segment-store/v2/one-root-head.hex new file mode 100644 index 0000000..47df02a --- /dev/null +++ b/conformance/segment-store/v2/one-root-head.hex @@ -0,0 +1 @@ +4b4545503a5245543a48454144320000000200900000000000000000000000010000000000000128f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb00000000000000000000000000000000000000000000000000000000000000000000000000000000ac049edb33af7e957c6ff11ead7e1bcf9c40fa9793cc84979215ffbba5f630b7 diff --git a/conformance/segment-store/v2/one-root-manifest.hex b/conformance/segment-store/v2/one-root-manifest.hex new file mode 100644 index 0000000..c2fed0b --- /dev/null +++ b/conformance/segment-store/v2/one-root-manifest.hex @@ -0,0 +1 @@ +4b4545503a5245543a4c495645320000000200a0000000000000000000000128000000000000000100480000000000010000000000000000000000000000000000000000000000000000000000000000e763e4d12e1ed333daeb84cc6336d9c3262f639b9d410c0a69c8d23680e9049a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ddde2ac65c5ba3829bf0fbd6f36e90272d69a0459fade92272b728a80d7ae6e20000000000000001ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b60085f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb10597643c3fc9485c7ecd3bb511d6726e726fd92f0f769a204b899c5fdc77d2c diff --git a/conformance/segment-store/v2/retention-profile.tsv b/conformance/segment-store/v2/retention-profile.tsv new file mode 100644 index 0000000..69c31b9 --- /dev/null +++ b/conformance/segment-store/v2/retention-profile.tsv @@ -0,0 +1,3 @@ +keep.retention-realization-profiles/v1 +identity version canonical_name witness_count selection +1 1 keep.retention-single-canonical-witness/v1 1 canonical-physical-catalog-coordinate diff --git a/docs/formats/README.md b/docs/formats/README.md index 6b8ce93..6952726 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -9,7 +9,7 @@ admitted merely because one Rust type can serialize and deserialize it. | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | | [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | -| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation and executable evidence planned in issue #19 | Golden corpus planned in issue #19 | +| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation planned in issue #19 | [Golden corpus](../../conformance/segment-store/v2/README.md) | The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index c908aab..4d7ea1b 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -47,11 +47,17 @@ The following pages form one protocol: recovery-disposition reservation, and restart behavior. - [Migration crash points](migration-crash.md) owns fixed-stage publication and the exact process-death boundaries for migration. +- [Migration inventory](migration-inventory.md) owns the bounded canonical + digest over preserved version-1 immutable pools. - [Requirements and evidence](requirements.md) owns stable requirement and crash identifiers, evidence status, compatibility, and nonclaims. - [Format rationale](rationale.md) records format-local choices and rejected alternatives. +The [version-2 golden corpus](../../../conformance/segment-store/v2/README.md) +freezes independent definition, profile, inventory, and record bytes. It is +format evidence, not production-writer evidence. + The version-1 [segment](../segment-store-v1/segment.md), [catalog](../segment-store-v1/catalog.md), and [publication-head](../segment-store-v1/catalog.md#publication-head) diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md index c5d7041..16127b0 100644 --- a/docs/formats/segment-store-v2/migration-crash.md +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -38,6 +38,29 @@ The fixed stage is not authority. `migration.intent` becomes migration authority only after its canonical link and store-root synchronization. `migration.receipt` becomes completion evidence at the equivalent boundary. +## Receipt synchronization mask + +`migration.receipt` records the exact pre-receipt mask +`0x00000000000003ff`. Bits are: + +| Bit | Completed evidence | +| ---: | --- | +| 0 | canonical migration intent and store root synchronized | +| 1 | `reader.lock` verified, synchronized, and root-synchronized | +| 2 | `retention` created and parent synchronized | +| 3 | `retention/roots` created and parent synchronized | +| 4 | `retention/manifests` created and parent synchronized | +| 5 | `gc` created and parent synchronized | +| 6 | `recovery` created and parent synchronized | +| 7 | `recovery/dispositions` created and parent synchronized | +| 8 | canonical format marker and store root synchronized | +| 9 | complete version-2 view reopened and verified | + +Bits 10 through 63 are zero and refuse when set. The mask records only +evidence completed before receipt construction; receipt-stage publication and +its final root synchronizations are established by admission of the canonical +receipt, not claimed by its own bytes. + ## Namespace prefix After durable intent publication, migration creates persistent `reader.lock` diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md new file mode 100644 index 0000000..b50501e --- /dev/null +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -0,0 +1,49 @@ +# Migration Inventory + +This page owns the bounded canonical digest over version-1 immutable segment +and catalog pools used by `migration.intent`. + +## Entry grammar + +One migration inventory entry is exactly 56 bytes: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 1 | artifact kind | `1` segment or `2` catalog | +| 1 | 7 | reserved | zero | +| 8 | 8 | catalog generation | zero for segment; positive for catalog | +| 16 | 8 | artifact length | exact positive length | +| 24 | 32 | artifact digest | exact admitted segment or catalog digest | + +Entries are sorted by their complete 56-byte canonical encoding and are +duplicate-free. The maximum is 2,097,152 entries across both pools. Count and +length arithmetic is checked before bytes are retained or allocated. + +The inventory digest is: + +```text +BLAKE3-256("keep.store-v1-pool-inventory/v2\0" || + entry-count-u32 || + canonical-entry-bytes) +``` + +The digest is streamed; the complete encoded inventory is never required in +memory. + +## Admission + +Migration inventories the exact pinned version-1 `segments` and `catalogs` +directories under writer authority. Every regular entry must have the one +canonical physical name derived from its verified semantic digest and, for a +catalog, generation. Each artifact is reopened without following links and +completely admitted before its semantic coordinate enters the digest. + +An unknown name, alternate case or width, alias, duplicate semantic coordinate, +wrong kind, link, changed directory, changed artifact, corrupt bytes, count +overflow, or entry above the fixed maximum refuses migration. File existence, +iteration order, path spelling, modification time, and physical file identity +do not enter the digest. + +The exact one-segment, one-catalog input and its canonical entries are frozen in +the version-2 corpus +[`inventory.tsv`](../../../conformance/segment-store/v2/inventory.tsv). diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 871e675..dfd29dc 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -56,6 +56,10 @@ The definition and checksum domains are when the exact version-1 namespace admits. An unsupported, corrupt, substituted, or same-name/different-digest marker refuses. +The format-definition digest is BLAKE3-256 of its domain followed by the exact +corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of +`keep.store-format-marker/v2\0` followed by all 96 marker bytes. + ## Reader fence `reader.lock` is a persistent regular zero-length file. Its contents and @@ -101,10 +105,13 @@ Version 1 is never extended in place without durable migration evidence. -The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The pool -inventory digest uses `keep.store-v1-pool-inventory/v2\0` over the sorted, -duplicate-free canonical names, lengths, and verified content digests from -both immutable pools. The intent therefore binds the exact catalog generation, +The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The +receipt's intent digest is BLAKE3-256 of +`keep.store-migration-intent/v2\0` followed by all 256 intent bytes. + +The [migration inventory](migration-inventory.md) defines its domain and law: +each migration inventory entry is exactly 56 bytes, and the fixed maximum is +2,097,152 entries. The intent therefore binds the exact catalog generation, length, and digest named by the admitted version-1 `HEAD`. The deterministically derived store identifier is: diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index ca4811b..6462ce4 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -71,9 +71,10 @@ Version 2 admits one realization profile: - exact witness count `1` for each layout and chunk identity; and - selection by canonical physical catalog coordinate. -The stored profile coordinate is the `u32` identity, `u32` version, and -BLAKE3-256 digest of its canonical definition bytes. Any unknown or mismatched -coordinate refuses. A future profile requires a successor specification. +The stored coordinate is its `u32` identity, `u32` version, and BLAKE3-256 of +`keep.retention-realization-profile/v1\0` followed by the exact corpus +`retention-profile.tsv` bytes. Any mismatch refuses. A future profile requires +a successor specification. Each root generation stores caller-selected limits no greater than these implementation ceilings: @@ -202,13 +203,10 @@ count is 4,096 and the maximum manifest length is 295,136 bytes. -The entry-set, manifest, and checksum domains are respectively: - -```text -keep.retention-manifest-entries/v2\0 -keep.retention-manifest/v2\0 -keep.retention-manifest-checksum/v2\0 -``` +The `keep.retention-manifest-entries/v2\0` preimage is +`entry-count-u32 || entries`. The `keep.retention-manifest/v2\0` preimage is +`header || entries`. The `keep.retention-manifest-checksum/v2\0` preimage is +`header || entries || manifest-digest`. Each operation is BLAKE3-256. The manifest pool coordinate is: diff --git a/xtask/tests/retention_store_v2_conformance_contract.rs b/xtask/tests/retention_store_v2_conformance_contract.rs new file mode 100644 index 0000000..38f7a60 --- /dev/null +++ b/xtask/tests/retention_store_v2_conformance_contract.rs @@ -0,0 +1,61 @@ +//! Repository-shape evidence for the version-2 segment-store corpus. + +#![cfg(feature = "repository-tasks")] + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +const CORPUS_ROOT: &str = "conformance/segment-store/v2"; +const REQUIRED_PATHS: &[&str] = &[ + "README.md", + "ORIGIN.md", + "definition.tsv", + "retention-profile.tsv", + "inventory.tsv", + "migration-source.tsv", + "artifacts.tsv", + "format-marker.hex", + "migration-intent.hex", + "migration-receipt.hex", + "one-anchor-root.hex", + "one-root-manifest.hex", + "one-root-head.hex", +]; + +fn repository_root() -> Result { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| io::Error::other("xtask manifest directory has no parent")) +} + +#[test] +fn version_two_corpus_has_one_complete_regular_file_shape() -> Result<(), io::Error> { + let root = repository_root()?.join(CORPUS_ROOT); + let expected: BTreeSet = REQUIRED_PATHS.iter().map(OsString::from).collect(); + let mut observed = BTreeSet::new(); + for entry in fs::read_dir(&root)? { + let entry = entry?; + assert!( + entry.file_type()?.is_file(), + "{} is not a regular file", + entry.path().display() + ); + observed.insert(entry.file_name()); + } + assert_eq!(observed, expected, "version-2 corpus shape drifted"); + Ok(()) +} + +#[test] +fn format_registry_routes_to_executable_version_two_evidence() -> Result<(), io::Error> { + let format_index = fs::read_to_string(repository_root()?.join("docs/formats/README.md"))?; + assert!( + format_index.contains("../../conformance/segment-store/v2/README.md"), + "format registry does not route to the version-2 corpus" + ); + Ok(()) +} diff --git a/xtask/tests/retention_store_v2_format_oracle.rs b/xtask/tests/retention_store_v2_format_oracle.rs new file mode 100644 index 0000000..86081ec --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle.rs @@ -0,0 +1,75 @@ +//! Independent construction oracle for version-2 segment-store golden bytes. + +#![cfg(feature = "repository-tasks")] + +const CORPUS_ROOT: &str = "conformance/segment-store/v2"; +const PROFILE_DEFINITION: &str = + include_str!("../../conformance/segment-store/v2/retention-profile.tsv"); +const FORMAT_DEFINITION: &str = include_str!("../../conformance/segment-store/v2/definition.tsv"); +const LAYOUTS: &str = include_str!("../../conformance/layout/v1/layouts.tsv"); +const V1_SEGMENT: &str = include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); +const V1_CATALOG: &str = include_str!("../../conformance/segment-store/v1/one-zero-catalog.hex"); +const V1_HEAD: &str = include_str!("../../conformance/segment-store/v1/one-zero-head.hex"); + +struct Artifact { + case_name: &'static str, + kind: &'static str, + generation: &'static str, + entry_count: &'static str, + bound_digest: [u8; 32], + final_checksum: [u8; 32], + fixture: &'static str, + bytes: Vec, +} + +struct Corpus { + profile_digest: [u8; 32], + definition_digest: [u8; 32], + inventory: Inventory, + migration: MigrationSource, + artifacts: Vec, +} + +struct Inventory { + rows: Vec, + digest: [u8; 32], +} + +struct InventoryRow { + kind: &'static str, + generation: u64, + byte_length: u64, + artifact_digest: [u8; 32], + source_fixture: &'static str, + bytes: [u8; 56], +} + +struct MigrationSource { + catalog_generation: u64, + catalog_length: u64, + catalog_digest: [u8; 32], + predecessor_digest: [u8; 32], + inventory_digest: [u8; 32], + definition_digest: [u8; 32], + store_id: [u8; 32], + root_device: u64, + root_mount: u64, + root_file: u64, +} + +struct RootArtifact { + bytes: Vec, + digest: [u8; 32], + checksum: [u8; 32], + namespace_digest: [u8; 32], +} + +struct ManifestArtifact { + bytes: Vec, + digest: [u8; 32], + checksum: [u8; 32], +} + +include!("retention_store_v2_format_oracle/encoding.rs"); +include!("retention_store_v2_format_oracle/artifacts.rs"); +include!("retention_store_v2_format_oracle/fixture_assertion.rs"); diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts.rs new file mode 100644 index 0000000..06b2f50 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts.rs @@ -0,0 +1,145 @@ +// This included source owns handwritten construction of every golden artifact. + +const PROFILE_DOMAIN: &[u8] = b"keep.retention-realization-profile/v1\0"; +const DEFINITION_DOMAIN: &[u8] = b"keep.segment-store-definition/v2\0"; +const INVENTORY_DOMAIN: &[u8] = b"keep.store-v1-pool-inventory/v2\0"; +const STORE_ID_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; + +fn build_corpus() -> Result { + let profile_digest = hash(PROFILE_DOMAIN, &[PROFILE_DEFINITION.as_bytes()]); + let definition_digest = hash(DEFINITION_DOMAIN, &[FORMAT_DEFINITION.as_bytes()]); + let inventory = build_inventory()?; + let migration = migration_source(definition_digest, inventory.digest)?; + let format = build_format_marker(definition_digest)?; + let intent = build_migration_intent(&migration)?; + let receipt = build_migration_receipt(&migration, &format, &intent)?; + let root = build_retention_root(profile_digest)?; + let manifest = build_retention_manifest(&root)?; + let head = build_retention_head(&manifest)?; + let artifacts = vec![ + format, + intent, + receipt, + Artifact { + case_name: "one-anchor-root", + kind: "retention-root", + generation: "1", + entry_count: "1", + bound_digest: root.digest, + final_checksum: root.checksum, + fixture: "one-anchor-root.hex", + bytes: root.bytes, + }, + Artifact { + case_name: "one-root-manifest", + kind: "retention-manifest", + generation: "1", + entry_count: "1", + bound_digest: manifest.digest, + final_checksum: manifest.checksum, + fixture: "one-root-manifest.hex", + bytes: manifest.bytes, + }, + head, + ]; + Ok(Corpus { + profile_digest, + definition_digest, + inventory, + migration, + artifacts, + }) +} + +fn build_inventory() -> Result { + let segment = decode_hex(V1_SEGMENT)?; + let catalog = decode_hex(V1_CATALOG)?; + require_length(&segment, 337, "version-1 source segment")?; + require_length(&catalog, 352, "version-1 source catalog")?; + let segment_digest = array_32(&segment, 273)?; + let catalog_digest = array_32(&catalog, 320)?; + let mut rows = vec![ + inventory_row(1, 0, &segment, segment_digest, "one-zero-segment.hex")?, + inventory_row(2, 1, &catalog, catalog_digest, "one-zero-catalog.hex")?, + ]; + rows.sort_by_key(|row| row.bytes); + let entry_count = + u32::try_from(rows.len()).map_err(|_| "inventory entry count overflow".to_owned())?; + let mut count = entry_count.to_be_bytes().to_vec(); + for row in &rows { + count.extend_from_slice(&row.bytes); + } + let digest = hash(INVENTORY_DOMAIN, &[&count]); + Ok(Inventory { rows, digest }) +} + +fn inventory_row( + kind: u8, + generation: u64, + artifact: &[u8], + digest: [u8; 32], + source_fixture: &'static str, +) -> Result { + let byte_length = + u64::try_from(artifact.len()).map_err(|_| "inventory length overflow".to_owned())?; + let mut bytes = Vec::with_capacity(56); + bytes.push(kind); + bytes.extend_from_slice(&[0; 7]); + push_u64(&mut bytes, generation); + push_u64(&mut bytes, byte_length); + bytes.extend_from_slice(&digest); + require_length(&bytes, 56, "migration inventory entry")?; + let kind_name = match kind { + 1 => "segment", + 2 => "catalog", + _ => return Err("unregistered inventory artifact kind".to_owned()), + }; + Ok(InventoryRow { + kind: kind_name, + generation, + byte_length, + artifact_digest: digest, + source_fixture, + bytes: <[u8; 56]>::try_from(bytes) + .map_err(|_| "inventory entry conversion failed".to_owned())?, + }) +} + +fn migration_source( + definition_digest: [u8; 32], + inventory_digest: [u8; 32], +) -> Result { + let head = decode_hex(V1_HEAD)?; + let catalog = decode_hex(V1_CATALOG)?; + require_length(&head, 128, "version-1 source head")?; + let catalog_generation = u64_at(&head, 24)?; + let catalog_length = u64_at(&head, 32)?; + let catalog_digest = array_32(&head, 40)?; + let predecessor_digest = array_32(&catalog, 32)?; + let store_id = hash( + STORE_ID_DOMAIN, + &[ + &catalog_generation.to_be_bytes(), + &catalog_length.to_be_bytes(), + &catalog_digest, + &predecessor_digest, + &inventory_digest, + &definition_digest, + ], + ); + Ok(MigrationSource { + catalog_generation, + catalog_length, + catalog_digest, + predecessor_digest, + inventory_digest, + definition_digest, + store_id, + root_device: 1, + root_mount: 2, + root_file: 3, + }) +} + +include!("artifacts/migration.rs"); +include!("artifacts/retention.rs"); diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs new file mode 100644 index 0000000..4120258 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs @@ -0,0 +1,96 @@ +// This included source owns construction of the format marker and migration records. + +fn build_format_marker(definition_digest: [u8; 32]) -> Result { + let mut bytes = Vec::with_capacity(96); + bytes.extend_from_slice(b"KEEP:STORE:V2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 96); + push_u32(&mut bytes, 0); + bytes.extend_from_slice(&definition_digest); + push_u32(&mut bytes, 4_096); + push_u32(&mut bytes, 0); + require_length(&bytes, 64, "format-marker checksum preimage")?; + let checksum = hash(b"keep.segment-store-marker-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 96, "format marker")?; + let marker_digest = hash(b"keep.store-format-marker/v2\0", &[&bytes]); + Ok(Artifact { + case_name: "format-marker", + kind: "format-marker", + generation: "-", + entry_count: "-", + bound_digest: marker_digest, + final_checksum: checksum, + fixture: "format-marker.hex", + bytes, + }) +} + +fn build_migration_intent(source: &MigrationSource) -> Result { + let mut bytes = Vec::with_capacity(256); + bytes.extend_from_slice(b"KEEP:MIG:INT2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 256); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, source.catalog_generation); + push_u64(&mut bytes, source.catalog_length); + bytes.extend_from_slice(&source.catalog_digest); + bytes.extend_from_slice(&source.predecessor_digest); + bytes.extend_from_slice(&source.inventory_digest); + push_u64(&mut bytes, source.root_device); + push_u64(&mut bytes, source.root_mount); + push_u64(&mut bytes, source.root_file); + bytes.extend_from_slice(&source.definition_digest); + bytes.extend_from_slice(&source.store_id); + require_length(&bytes, 224, "migration-intent checksum preimage")?; + let checksum = hash(b"keep.store-migration-intent-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 256, "migration intent")?; + let intent_digest = hash(b"keep.store-migration-intent/v2\0", &[&bytes]); + Ok(Artifact { + case_name: "migration-intent", + kind: "migration-intent", + generation: "1", + entry_count: "2", + bound_digest: intent_digest, + final_checksum: checksum, + fixture: "migration-intent.hex", + bytes, + }) +} + +fn build_migration_receipt( + source: &MigrationSource, + format: &Artifact, + intent: &Artifact, +) -> Result { + let initial_retention = hash(b"keep.initial-retention-state/v2\0", &[]); + let initial_gc = hash(b"keep.initial-gc-state/v2\0", &[]); + let empty_dispositions = hash(b"keep.empty-disposition-set/v2\0", &[]); + let mut bytes = Vec::with_capacity(256); + bytes.extend_from_slice(b"KEEP:MIG:REC2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 256); + push_u32(&mut bytes, 0); + bytes.extend_from_slice(&intent.bound_digest); + bytes.extend_from_slice(&source.store_id); + bytes.extend_from_slice(&format.bound_digest); + bytes.extend_from_slice(&initial_retention); + bytes.extend_from_slice(&initial_gc); + bytes.extend_from_slice(&empty_dispositions); + push_u64(&mut bytes, 0x03ff); + require_length(&bytes, 224, "migration-receipt checksum preimage")?; + let checksum = hash(b"keep.store-migration-receipt-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 256, "migration receipt")?; + Ok(Artifact { + case_name: "migration-receipt", + kind: "migration-receipt", + generation: "1", + entry_count: "2", + bound_digest: intent.bound_digest, + final_checksum: checksum, + fixture: "migration-receipt.hex", + bytes, + }) +} diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs new file mode 100644 index 0000000..44ad067 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs @@ -0,0 +1,155 @@ +// This included source owns construction of retention root, manifest, and head records. + +const NAMESPACE: &[u8] = &[0x00, 0x2f, 0xff]; +const BLOB_ID: [u8; 59] = [ + 0x4b, 0x45, 0x45, 0x50, 0x3a, 0x42, 0x4c, 0x4f, 0x42, 0x3a, 0x49, 0x44, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x1c, + 0xfb, 0x8f, 0xa9, 0xe9, 0x17, 0xab, 0xa1, 0x5a, 0x1f, 0x59, 0x20, 0x95, 0xf3, 0x77, + 0xff, 0x18, 0x07, 0x55, 0xfe, 0x12, 0x12, 0xb0, 0xd7, 0xd2, 0xec, 0x75, 0x0b, 0xd1, + 0x28, 0xb6, 0x06, +]; +const LAYOUT_ID: [u8; 60] = [ + 0x4b, 0x45, 0x45, 0x50, 0x3a, 0x4c, 0x41, 0x59, 0x4f, 0x55, 0x54, 0x3a, 0x49, 0x44, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, + 0x88, 0x7d, 0xa2, 0x3f, 0x1a, 0x74, 0x83, 0x35, 0x9a, 0x78, 0xfc, 0x9a, 0x7f, 0xde, + 0x80, 0x03, 0x0e, 0xc2, 0xc4, 0x69, 0x06, 0x03, 0x80, 0x3f, 0x0a, 0xb7, 0xd0, 0xed, + 0xb5, 0x65, 0x75, 0xb8, +]; + +fn build_retention_root(profile_digest: [u8; 32]) -> Result { + let mut anchor = Vec::with_capacity(119); + anchor.extend_from_slice(&BLOB_ID); + anchor.extend_from_slice(&LAYOUT_ID); + require_length(&anchor, 119, "retention anchor")?; + let anchor_count = 1u32.to_be_bytes(); + let anchor_set_digest = hash( + b"keep.retention-anchor-set/v2\0", + &[&anchor_count, &anchor], + ); + let mut header = root_header(profile_digest, anchor_set_digest)?; + let mut body = NAMESPACE.to_vec(); + body.extend_from_slice(&anchor); + let digest = hash(b"keep.retention-root/v2\0", &[&header, &body]); + let checksum = hash( + b"keep.retention-root-checksum/v2\0", + &[&header, &body, &digest], + ); + header.extend_from_slice(&body); + header.extend_from_slice(&digest); + header.extend_from_slice(&checksum); + require_length(&header, 378, "retention root")?; + let namespace_length = + u16::try_from(NAMESPACE.len()).map_err(|_| "namespace length overflow".to_owned())?; + let namespace_digest = hash( + b"keep.retention-namespace/v1\0", + &[&namespace_length.to_be_bytes(), NAMESPACE], + ); + Ok(RootArtifact { + bytes: header, + digest, + checksum, + namespace_digest, + }) +} + +fn root_header( + profile_digest: [u8; 32], + anchor_set_digest: [u8; 32], +) -> Result, String> { + let mut bytes = Vec::with_capacity(192); + bytes.extend_from_slice(b"KEEP:RET:ROOT2\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 192); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, 378); + push_u64(&mut bytes, 1); + push_u16(&mut bytes, 3); + push_u16(&mut bytes, 119); + push_u32(&mut bytes, 1); + push_u32(&mut bytes, 1); + push_u32(&mut bytes, 1); + bytes.extend_from_slice(&profile_digest); + push_u64(&mut bytes, 4); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 0); + push_u64(&mut bytes, 4_096); + push_u64(&mut bytes, 4_096); + bytes.extend_from_slice(&[0; 32]); + bytes.extend_from_slice(&anchor_set_digest); + bytes.extend_from_slice(&[0; 12]); + require_length(&bytes, 192, "retention-root header")?; + Ok(bytes) +} + +fn build_retention_manifest(root: &RootArtifact) -> Result { + let mut entry = Vec::with_capacity(72); + entry.extend_from_slice(&root.namespace_digest); + push_u64(&mut entry, 1); + entry.extend_from_slice(&root.digest); + require_length(&entry, 72, "retention manifest entry")?; + let entry_count = 1u32.to_be_bytes(); + let entry_set_digest = hash( + b"keep.retention-manifest-entries/v2\0", + &[&entry_count, &entry], + ); + let mut header = manifest_header(entry_set_digest)?; + let digest = hash(b"keep.retention-manifest/v2\0", &[&header, &entry]); + let checksum = hash( + b"keep.retention-manifest-checksum/v2\0", + &[&header, &entry, &digest], + ); + header.extend_from_slice(&entry); + header.extend_from_slice(&digest); + header.extend_from_slice(&checksum); + require_length(&header, 296, "retention manifest")?; + Ok(ManifestArtifact { + bytes: header, + digest, + checksum, + }) +} + +fn manifest_header(entry_set_digest: [u8; 32]) -> Result, String> { + let mut bytes = Vec::with_capacity(160); + bytes.extend_from_slice(b"KEEP:RET:LIVE2\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 160); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, 296); + push_u64(&mut bytes, 1); + push_u16(&mut bytes, 72); + push_u16(&mut bytes, 0); + push_u32(&mut bytes, 1); + bytes.extend_from_slice(&[0; 32]); + bytes.extend_from_slice(&entry_set_digest); + bytes.extend_from_slice(&[0; 48]); + require_length(&bytes, 160, "retention-manifest header")?; + Ok(bytes) +} + +fn build_retention_head(manifest: &ManifestArtifact) -> Result { + let mut bytes = Vec::with_capacity(144); + bytes.extend_from_slice(b"KEEP:RET:HEAD2\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 144); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, 1); + push_u64(&mut bytes, 296); + bytes.extend_from_slice(&manifest.digest); + bytes.extend_from_slice(&[0; 32]); + push_u64(&mut bytes, 0); + require_length(&bytes, 112, "retention-head checksum preimage")?; + let checksum = hash(b"keep.retention-head-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 144, "retention head")?; + Ok(Artifact { + case_name: "one-root-head", + kind: "retention-head", + generation: "1", + entry_count: "1", + bound_digest: manifest.digest, + final_checksum: checksum, + fixture: "one-root-head.hex", + bytes, + }) +} diff --git a/xtask/tests/retention_store_v2_format_oracle/encoding.rs b/xtask/tests/retention_store_v2_format_oracle/encoding.rs new file mode 100644 index 0000000..2275b17 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/encoding.rs @@ -0,0 +1,126 @@ +// This included source owns primitive binary construction and fixture transport. + +use std::fmt::Write as _; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +fn hash(domain: &[u8], parts: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + for part in parts { + hasher.update(part); + } + *hasher.finalize().as_bytes() +} + +fn push_u16(bytes: &mut Vec, value: u16) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u64(bytes: &mut Vec, value: u64) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn array_32(bytes: &[u8], offset: usize) -> Result<[u8; 32], String> { + let end = offset + .checked_add(32) + .ok_or_else(|| "32-byte field offset overflow".to_owned())?; + let field = bytes + .get(offset..end) + .ok_or_else(|| format!("missing 32-byte field at offset {offset}"))?; + <[u8; 32]>::try_from(field).map_err(|_| "32-byte field conversion failed".to_owned()) +} + +fn u64_at(bytes: &[u8], offset: usize) -> Result { + let end = offset + .checked_add(8) + .ok_or_else(|| "u64 field offset overflow".to_owned())?; + let field = bytes + .get(offset..end) + .ok_or_else(|| format!("missing u64 field at offset {offset}"))?; + let encoded = + <[u8; 8]>::try_from(field).map_err(|_| "u64 field conversion failed".to_owned())?; + Ok(u64::from_be_bytes(encoded)) +} + +fn decode_hex(source: &str) -> Result, String> { + let encoded = source + .strip_suffix('\n') + .ok_or_else(|| "hex fixture lacks one final newline".to_owned())?; + if encoded.contains('\n') || encoded.contains('\r') { + return Err("hex fixture contains embedded line ending".to_owned()); + } + if encoded.len() % 2 != 0 { + return Err("hex fixture has odd encoded length".to_owned()); + } + encoded + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let [high_byte, low_byte] = <[u8; 2]>::try_from(pair) + .map_err(|_| "hex pair conversion failed".to_owned())?; + let high = hex_nibble(high_byte)?; + let low = hex_nibble(low_byte)?; + Ok((high << 4) | low) + }) + .collect() +} + +fn hex_nibble(byte: u8) -> Result { + match byte { + b'0'..=b'9' => byte + .checked_sub(b'0') + .ok_or_else(|| "decimal hex nibble underflow".to_owned()), + b'a'..=b'f' => byte + .checked_sub(b'a') + .and_then(|value| value.checked_add(10)) + .ok_or_else(|| "alphabetic hex nibble overflow".to_owned()), + _ => Err("hex fixture contains a non-lowercase hexadecimal byte".to_owned()), + } +} + +fn encode_hex(bytes: &[u8]) -> Result { + let capacity = bytes + .len() + .checked_mul(2) + .and_then(|length| length.checked_add(1)) + .ok_or_else(|| "hex output length overflow".to_owned())?; + let mut encoded = String::with_capacity(capacity); + for byte in bytes { + write!(&mut encoded, "{byte:02x}") + .map_err(|_| "hex output formatting failed".to_owned())?; + } + encoded.push('\n'); + Ok(encoded) +} + +fn repository_root() -> Result { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| io::Error::other("xtask manifest directory has no parent")) +} + +fn corpus_path(relative: &str) -> Result { + Ok(repository_root()?.join(CORPUS_ROOT).join(relative)) +} + +fn read_corpus_file(relative: &str) -> Result { + fs::read_to_string(corpus_path(relative)?) +} + +fn require_length(bytes: &[u8], expected: usize, name: &str) -> Result<(), String> { + if bytes.len() == expected { + Ok(()) + } else { + Err(format!( + "{name} has {} bytes; expected {expected}", + bytes.len() + )) + } +} diff --git a/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs b/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs new file mode 100644 index 0000000..88b3570 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs @@ -0,0 +1,198 @@ +// This included source owns assertions over constructed bytes and canonical tables. + +#[test] +fn golden_artifacts_match_the_independent_oracle() -> Result<(), String> { + let corpus = build_corpus()?; + let expected_manifest = artifacts_table(&corpus.artifacts)?; + assert_eq!( + read_corpus_file("artifacts.tsv").map_err(|error| error.to_string())?, + expected_manifest + ); + for artifact in &corpus.artifacts { + let fixture = + read_corpus_file(artifact.fixture).map_err(|error| error.to_string())?; + assert_eq!( + fixture, + encode_hex(&artifact.bytes)?, + "golden fixture drifted: {}", + artifact.fixture + ); + } + Ok(()) +} + +#[test] +fn definition_profile_and_migration_sources_match_the_oracle() -> Result<(), String> { + let corpus = build_corpus()?; + let profile_hex = encode_digest(&corpus.profile_digest)?; + let definition_hex = encode_digest(&corpus.definition_digest)?; + assert!( + FORMAT_DEFINITION.contains(&format!("retention.profile.digest\t{profile_hex}")), + "format definition does not bind the exact profile digest" + ); + assert_eq!( + corpus.definition_digest, corpus.migration.definition_digest, + "migration source does not bind the exact format definition" + ); + assert_eq!( + read_corpus_file("inventory.tsv").map_err(|error| error.to_string())?, + inventory_table(&corpus.inventory)? + ); + assert_eq!( + read_corpus_file("migration-source.tsv").map_err(|error| error.to_string())?, + migration_source_table(&corpus.migration)? + ); + for documentation in ["README.md", "ORIGIN.md"] { + assert!( + read_corpus_file(documentation) + .map_err(|error| error.to_string())? + .contains(&definition_hex), + "{documentation} does not name the exact format-definition digest" + ); + } + Ok(()) +} + +#[test] +fn definition_and_profile_tables_are_exact_and_canonical() -> Result<(), String> { + assert_eq!( + PROFILE_DEFINITION, + "keep.retention-realization-profiles/v1\n\ + identity\tversion\tcanonical_name\twitness_count\tselection\n\ + 1\t1\tkeep.retention-single-canonical-witness/v1\t1\t\ + canonical-physical-catalog-coordinate\n" + ); + assert!(FORMAT_DEFINITION.ends_with('\n')); + assert!(!FORMAT_DEFINITION.contains('\r')); + let mut rows = FORMAT_DEFINITION.lines(); + assert_eq!(rows.next(), Some("keep.segment-store.definition/v2")); + assert_eq!(rows.next(), Some("key\tvalue")); + let mut previous: Option<&str> = None; + for row in rows { + let (key, value) = row + .split_once('\t') + .ok_or_else(|| format!("definition row lacks one key/value boundary: {row}"))?; + assert!(!key.is_empty(), "definition row has an empty key"); + assert!(!value.is_empty(), "definition row has an empty value"); + assert!( + !value.contains('\t'), + "definition row has more than one key/value boundary: {row}" + ); + if let Some(prior) = previous { + assert!(prior < key, "definition keys are not strictly sorted"); + } + previous = Some(key); + } + Ok(()) +} + +#[test] +fn retention_anchor_ids_are_derived_from_the_accepted_layout_corpus() -> Result<(), String> { + let row = LAYOUTS + .lines() + .find(|line| line.starts_with("one-zero\t")) + .ok_or_else(|| "layout corpus lacks the one-zero case".to_owned())?; + let fields: Vec<&str> = row.split('\t').collect(); + let blob_id = fields + .get(5) + .ok_or_else(|| "one-zero layout row lacks BlobId text".to_owned())?; + assert_eq!( + *blob_id, + "keep:blob:v1:blake3-256:1:\ + 1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" + ); + let blob_digest = blob_id + .rsplit(':') + .next() + .ok_or_else(|| "one-zero BlobId lacks a digest".to_owned())?; + assert_eq!( + BLOB_ID.get(..16), + Some(b"KEEP:BLOB:ID\0\0\0\0".as_slice()) + ); + assert_eq!(BLOB_ID.get(16..18), Some([0_u8, 1].as_slice())); + assert_eq!(BLOB_ID.get(18), Some(&1)); + assert_eq!(u64_at(&BLOB_ID, 19)?, 1); + assert_eq!( + BLOB_ID.get(27..), + Some(decode_hex(&format!("{blob_digest}\n"))?.as_slice()) + ); + let layout_id_binary = encode_hex(&LAYOUT_ID)?; + assert_eq!( + fields.get(11).copied(), + Some(layout_id_binary.trim_end()) + ); + Ok(()) +} + +fn artifacts_table(artifacts: &[Artifact]) -> Result { + let mut table = String::from( + "keep.segment-store-v2.artifacts/v1\n\ + case\tkind\tbyte_length\tgeneration\tentry_count\tbound_digest_hex\t\ + final_checksum_hex\tfixture\n", + ); + for artifact in artifacts { + writeln!( + table, + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + artifact.case_name, + artifact.kind, + artifact.bytes.len(), + artifact.generation, + artifact.entry_count, + encode_digest(&artifact.bound_digest)?, + encode_digest(&artifact.final_checksum)?, + artifact.fixture + ) + .map_err(|_| "artifact table formatting failed".to_owned())?; + } + Ok(table) +} + +fn inventory_table(inventory: &Inventory) -> Result { + let mut table = String::from( + "keep.segment-store-v2.inventory/v1\n\ + kind\tgeneration\tbyte_length\tartifact_digest_hex\tsource_fixture\n", + ); + for row in &inventory.rows { + writeln!( + table, + "{}\t{}\t{}\t{}\t{}", + row.kind, + row.generation, + row.byte_length, + encode_digest(&row.artifact_digest)?, + row.source_fixture + ) + .map_err(|_| "inventory table formatting failed".to_owned())?; + } + Ok(table) +} + +fn migration_source_table(source: &MigrationSource) -> Result { + let mut table = String::from( + "keep.segment-store-v2.migration-source/v1\n\ + case\tcatalog_generation\tcatalog_length\tcatalog_digest_hex\t\ + predecessor_digest_hex\tinventory_digest_hex\tdefinition_digest_hex\t\ + store_id_hex\troot_device\troot_mount\troot_file\n", + ); + writeln!( + table, + "one-zero\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + source.catalog_generation, + source.catalog_length, + encode_digest(&source.catalog_digest)?, + encode_digest(&source.predecessor_digest)?, + encode_digest(&source.inventory_digest)?, + encode_digest(&source.definition_digest)?, + encode_digest(&source.store_id)?, + source.root_device, + source.root_mount, + source.root_file + ) + .map_err(|_| "migration-source table formatting failed".to_owned())?; + Ok(table) +} + +fn encode_digest(digest: &[u8; 32]) -> Result { + encode_hex(digest).map(|encoded| encoded.trim_end().to_owned()) +} diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index 7c80151..34ebdcb 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -45,6 +45,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", "[Migration crash points](migration-crash.md)", + "[Migration inventory](migration-inventory.md)", "[Requirements and evidence](requirements.md)", "[Format rationale](rationale.md)", ] { @@ -68,6 +69,9 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Date: Wed, 29 Jul 2026 20:00:18 -0700 Subject: [PATCH 03/50] Add: Validate core retention values --- CHANGELOG.md | 9 ++- docs/formats/segment-store-v2/README.md | 9 ++- docs/formats/segment-store-v2/requirements.md | 2 +- src/lib.rs | 11 ++- src/retention/anchor.rs | 35 ++++++++ src/retention/liveness_generation.rs | 46 +++++++++++ src/retention/liveness_generation_error.rs | 30 +++++++ src/retention/mod.rs | 24 ++++++ src/retention/namespace.rs | 78 ++++++++++++++++++ src/retention/namespace_digest.rs | 21 +++++ src/retention/namespace_error.rs | 32 ++++++++ src/retention/root_generation.rs | 46 +++++++++++ src/retention/root_generation_error.rs | 30 +++++++ tests/retention_values.rs | 79 +++++++++++++++++++ 14 files changed, 442 insertions(+), 10 deletions(-) create mode 100644 src/retention/anchor.rs create mode 100644 src/retention/liveness_generation.rs create mode 100644 src/retention/liveness_generation_error.rs create mode 100644 src/retention/mod.rs create mode 100644 src/retention/namespace.rs create mode 100644 src/retention/namespace_digest.rs create mode 100644 src/retention/namespace_error.rs create mode 100644 src/retention/root_generation.rs create mode 100644 src/retention/root_generation_error.rs create mode 100644 tests/retention_values.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da89d9..dd30ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -305,9 +305,12 @@ after its public API and format compatibility policies are established. - Specified `keep.segment-store/v2` retention values, root generations, liveness manifests, reader snapshots, one-way staged migration, exact crash - boundaries, and reserved GC/disposition records. Version-1 immutable bytes - remain authoritative; production version-2 writing remains unavailable until - issue #19's executable evidence is complete. + boundaries, and reserved GC/disposition records. Validated public + `RetentionNamespace`, namespace-digest, `RootGeneration`, + `LivenessGeneration`, and `RetentionAnchor` values now establish the core + boundary. Version-1 immutable bytes remain authoritative; production + version-2 writing remains unavailable until issue #19's executable evidence + is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 4d7ea1b..3fe3d8b 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -66,8 +66,11 @@ re-encode them. ## Status -The format contract is frozen by ADR-0009 and this specification. Requirements -marked **Planned in #19** or **Planned in #21** are not implementation evidence. -A store must refuse version-2 state until the relevant parser, corruption, +The format contract is frozen by ADR-0009 and this specification. Public core +types now admit exact namespace bytes, namespace digests, root and liveness +generations, and reconstruction anchors. No production version-2 parser, +transition, migration, or writer exists yet. Requirements that remain marked +as planned in issue #19 or issue #21 are not implementation evidence. A store +must refuse version-2 state until the relevant parser, corruption, golden-format, model-based, crash-injection, recovery, and fuzz evidence is implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index b011772..e57e60b 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -9,7 +9,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | unit and public API tests | Planned in #19 | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` covers namespace, digest, generations, and anchors; profile and limit evidence remains | In progress in #19 | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | golden-format fixtures plus independent oracle | Planned in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | diff --git a/src/lib.rs b/src/lib.rs index 84b5b78..88026b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,9 +20,9 @@ //! explicit. Exact next-head finalization now has a storage-independent //! contract and a pinned writer-authorized filesystem adapter. Reusable-stage //! continuation has a storage-independent planning and execution boundary plus -//! a pinned writer-authorized filesystem adapter. Retention and garbage -//! collection remain intentionally absent until their contracts have -//! executable specifications. +//! a pinned writer-authorized filesystem adapter. Core retention namespaces, +//! generations, and reconstruction anchors are validated. Retention +//! publication, recovery, and garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -34,6 +34,7 @@ mod chunk; mod layout; mod profile; mod reference; +mod retention; #[cfg(feature = "repository-tasks")] #[doc(hidden)] @@ -119,3 +120,7 @@ pub use reference::{ RangeReadError, RangeReadReceipt, ReconstructionError, ReconstructionReceipt, ReferenceStore, ReferenceStoreCapacity, StagedBlob, }; +pub use retention::{ + LivenessGeneration, LivenessGenerationError, RetentionAnchor, RetentionNamespace, + RetentionNamespaceDigest, RetentionNamespaceError, RootGeneration, RootGenerationError, +}; diff --git a/src/retention/anchor.rs b/src/retention/anchor.rs new file mode 100644 index 0000000..f30e799 --- /dev/null +++ b/src/retention/anchor.rs @@ -0,0 +1,35 @@ +//! This module owns one typed logical reconstruction anchor. + +use crate::blob::BlobId; +use crate::layout::LayoutId; + +/// Exact logical blob and canonical layout coordinates retained together. +/// +/// Construction cannot fail because both component identities are already +/// validated. An anchor proves only the requested coordinates; closure +/// admission must separately prove the named layout, chunks, and blob bytes. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionAnchor { + blob_id: BlobId, + layout_id: LayoutId, +} + +impl RetentionAnchor { + /// Combines one validated logical blob and layout coordinate. + pub const fn new(blob_id: BlobId, layout_id: LayoutId) -> Self { + Self { blob_id, layout_id } + } + + /// Returns the exact retained logical blob coordinate. + #[must_use] + pub const fn blob_id(self) -> BlobId { + self.blob_id + } + + /// Returns the exact retained layout coordinate. + #[must_use] + pub const fn layout_id(self) -> LayoutId { + self.layout_id + } +} diff --git a/src/retention/liveness_generation.rs b/src/retention/liveness_generation.rs new file mode 100644 index 0000000..3936c5d --- /dev/null +++ b/src/retention/liveness_generation.rs @@ -0,0 +1,46 @@ +//! This module owns checked global retention liveness generations. + +use std::num::NonZeroU64; + +use super::LivenessGenerationError; + +/// Positive generation of the global retention manifest. +/// +/// This coordinate is deliberately distinct from every per-namespace +/// [`RootGeneration`](super::RootGeneration). +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct LivenessGeneration(NonZeroU64); + +impl LivenessGeneration { + /// Admits one positive liveness generation. + /// + /// # Errors + /// + /// Returns [`LivenessGenerationError::Zero`] when `value` is zero. + pub const fn new(value: u64) -> Result { + match NonZeroU64::new(value) { + Some(value) => Ok(Self(value)), + None => Err(LivenessGenerationError::Zero), + } + } + + /// Returns the exact positive generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } + + /// Derives the exact successor through checked addition. + /// + /// # Errors + /// + /// Returns [`LivenessGenerationError::Exhausted`] at `u64::MAX`. + pub const fn successor(self) -> Result { + let current = self.get(); + let Some(next) = current.checked_add(1) else { + return Err(LivenessGenerationError::Exhausted { current }); + }; + Self::new(next) + } +} diff --git a/src/retention/liveness_generation_error.rs b/src/retention/liveness_generation_error.rs new file mode 100644 index 0000000..e48d594 --- /dev/null +++ b/src/retention/liveness_generation_error.rs @@ -0,0 +1,30 @@ +//! This module owns typed retention liveness-generation failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit or advance a retention liveness generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LivenessGenerationError { + /// Generation zero is outside the version-2 protocol. + Zero, + /// The current generation has no representable successor. + Exhausted { + /// Exact generation that could not advance. + current: u64, + }, +} + +impl fmt::Display for LivenessGenerationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero => formatter.write_str("retention liveness generation must be positive"), + Self::Exhausted { current } => write!( + formatter, + "retention liveness generation {current} has no successor" + ), + } + } +} + +impl Error for LivenessGenerationError {} diff --git a/src/retention/mod.rs b/src/retention/mod.rs new file mode 100644 index 0000000..43494e9 --- /dev/null +++ b/src/retention/mod.rs @@ -0,0 +1,24 @@ +//! Semantic retention coordinates and reconstruction anchors. +//! +//! This module owns validated namespace bytes, namespace identity, +//! generation coordinates, and logical reconstruction anchors. It does not own +//! record encoding, filesystem layout, publication, recovery, or garbage +//! collection. + +mod anchor; +mod liveness_generation; +mod liveness_generation_error; +mod namespace; +mod namespace_digest; +mod namespace_error; +mod root_generation; +mod root_generation_error; + +pub use anchor::RetentionAnchor; +pub use liveness_generation::LivenessGeneration; +pub use liveness_generation_error::LivenessGenerationError; +pub use namespace::RetentionNamespace; +pub use namespace_digest::RetentionNamespaceDigest; +pub use namespace_error::RetentionNamespaceError; +pub use root_generation::RootGeneration; +pub use root_generation_error::RootGenerationError; diff --git a/src/retention/namespace.rs b/src/retention/namespace.rs new file mode 100644 index 0000000..01063b6 --- /dev/null +++ b/src/retention/namespace.rs @@ -0,0 +1,78 @@ +//! This module owns admission and identity of opaque retention namespace bytes. + +use std::num::NonZeroU8; + +use super::{RetentionNamespaceDigest, RetentionNamespaceError}; + +const DIGEST_DOMAIN: &[u8] = b"keep.retention-namespace/v1\0"; + +/// One validated opaque retention authority namespace. +/// +/// Every nonempty byte string through 255 bytes is canonical as-is. Admission +/// performs no Unicode, path, case, or application-level interpretation. +/// +/// Constructing from a borrowed slice allocates one owned copy. Constructing +/// from a `Vec` consumes it; boxed-slice conversion may discard excess +/// capacity. +#[must_use] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionNamespace { + bytes: Box<[u8]>, + length: NonZeroU8, +} + +impl RetentionNamespace { + /// Maximum admitted namespace length in bytes. + pub const MAXIMUM_BYTE_LENGTH: u8 = u8::MAX; + + /// Returns the exact opaque namespace bytes. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// Derives the canonical physical namespace-directory identity. + /// + /// The digest binds the domain, the fixed-width big-endian byte length, + /// and the exact namespace bytes. This operation does not allocate. + pub fn digest(&self) -> RetentionNamespaceDigest { + let length = u16::from(self.length.get()).to_be_bytes(); + let mut hasher = blake3::Hasher::new(); + hasher.update(DIGEST_DOMAIN); + hasher.update(&length); + hasher.update(&self.bytes); + RetentionNamespaceDigest::from_hash(*hasher.finalize().as_bytes()) + } + + fn admit_length(observed: usize) -> Result { + let length = u8::try_from(observed).map_err(|_| RetentionNamespaceError::TooLong { + maximum: Self::MAXIMUM_BYTE_LENGTH, + observed, + })?; + NonZeroU8::new(length).ok_or(RetentionNamespaceError::Empty) + } +} + +impl TryFrom> for RetentionNamespace { + type Error = RetentionNamespaceError; + + fn try_from(bytes: Vec) -> Result { + let length = Self::admit_length(bytes.len())?; + Ok(Self { + bytes: bytes.into_boxed_slice(), + length, + }) + } +} + +impl TryFrom<&[u8]> for RetentionNamespace { + type Error = RetentionNamespaceError; + + fn try_from(bytes: &[u8]) -> Result { + let length = Self::admit_length(bytes.len())?; + Ok(Self { + bytes: Box::from(bytes), + length, + }) + } +} diff --git a/src/retention/namespace_digest.rs b/src/retention/namespace_digest.rs new file mode 100644 index 0000000..d68dda4 --- /dev/null +++ b/src/retention/namespace_digest.rs @@ -0,0 +1,21 @@ +//! This module owns the canonical retention namespace digest coordinate. + +/// Canonical BLAKE3-256 identity of one exact retention namespace. +/// +/// This coordinate selects a physical namespace directory. Authority still +/// requires the matching root record to contain the exact namespace bytes. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionNamespaceDigest([u8; 32]); + +impl RetentionNamespaceDigest { + pub(super) const fn from_hash(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the exact 32 digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/retention/namespace_error.rs b/src/retention/namespace_error.rs new file mode 100644 index 0000000..f266ec2 --- /dev/null +++ b/src/retention/namespace_error.rs @@ -0,0 +1,32 @@ +//! This module owns typed retention namespace admission failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit opaque retention namespace bytes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionNamespaceError { + /// The namespace was empty. + Empty, + /// The namespace exceeded the version-2 byte ceiling. + TooLong { + /// Maximum admitted length. + maximum: u8, + /// Observed byte length. + observed: usize, + }, +} + +impl fmt::Display for RetentionNamespaceError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => formatter.write_str("retention namespace must not be empty"), + Self::TooLong { maximum, observed } => write!( + formatter, + "retention namespace has {observed} bytes; maximum is {maximum}" + ), + } + } +} + +impl Error for RetentionNamespaceError {} diff --git a/src/retention/root_generation.rs b/src/retention/root_generation.rs new file mode 100644 index 0000000..488565c --- /dev/null +++ b/src/retention/root_generation.rs @@ -0,0 +1,46 @@ +//! This module owns checked retention root-generation coordinates. + +use std::num::NonZeroU64; + +use super::RootGenerationError; + +/// Positive generation of one retention namespace root. +/// +/// Generation `1` is initial. Empty retained sets still publish a successor; +/// generations are never reused. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RootGeneration(NonZeroU64); + +impl RootGeneration { + /// Admits one positive root generation. + /// + /// # Errors + /// + /// Returns [`RootGenerationError::Zero`] when `value` is zero. + pub const fn new(value: u64) -> Result { + match NonZeroU64::new(value) { + Some(value) => Ok(Self(value)), + None => Err(RootGenerationError::Zero), + } + } + + /// Returns the exact positive generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } + + /// Derives the exact successor through checked addition. + /// + /// # Errors + /// + /// Returns [`RootGenerationError::Exhausted`] at `u64::MAX`. + pub const fn successor(self) -> Result { + let current = self.get(); + let Some(next) = current.checked_add(1) else { + return Err(RootGenerationError::Exhausted { current }); + }; + Self::new(next) + } +} diff --git a/src/retention/root_generation_error.rs b/src/retention/root_generation_error.rs new file mode 100644 index 0000000..2c92989 --- /dev/null +++ b/src/retention/root_generation_error.rs @@ -0,0 +1,30 @@ +//! This module owns typed retention root-generation failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit or advance a retention root generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RootGenerationError { + /// Generation zero is outside the version-2 protocol. + Zero, + /// The current generation has no representable successor. + Exhausted { + /// Exact generation that could not advance. + current: u64, + }, +} + +impl fmt::Display for RootGenerationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero => formatter.write_str("retention root generation must be positive"), + Self::Exhausted { current } => write!( + formatter, + "retention root generation {current} has no successor" + ), + } + } +} + +impl Error for RootGenerationError {} diff --git a/tests/retention_values.rs b/tests/retention_values.rs new file mode 100644 index 0000000..8eea6a0 --- /dev/null +++ b/tests/retention_values.rs @@ -0,0 +1,79 @@ +//! Public laws for version-2 retention values. + +use keep::{ + BlobId, LayoutId, LivenessGeneration, LivenessGenerationError, RetentionAnchor, + RetentionNamespace, RetentionNamespaceError, RootGeneration, RootGenerationError, +}; + +const ONE_ZERO_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:1:", + "1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" +); +const ONE_ZERO_LAYOUT: &str = concat!( + "keep:layout:v1:flat-chunks-v1:blake3-256:220:", + "887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8" +); + +#[test] +fn retention_namespaces_preserve_every_admitted_byte_and_bind_length() +-> Result<(), Box> { + assert_eq!( + RetentionNamespace::try_from(Vec::new()), + Err(RetentionNamespaceError::Empty) + ); + assert_eq!( + RetentionNamespace::try_from(vec![0_u8; 256]), + Err(RetentionNamespaceError::TooLong { + maximum: 255, + observed: 256, + }) + ); + + let bytes = [0x00, 0x2f, 0xff]; + let namespace = RetentionNamespace::try_from(bytes.as_slice())?; + assert_eq!(namespace.as_bytes(), bytes.as_slice()); + assert_eq!( + namespace.digest().as_bytes(), + &[ + 0xdd, 0xde, 0x2a, 0xc6, 0x5c, 0x5b, 0xa3, 0x82, 0x9b, 0xf0, 0xfb, 0xd6, 0xf3, 0x6e, + 0x90, 0x27, 0x2d, 0x69, 0xa0, 0x45, 0x9f, 0xad, 0xe9, 0x22, 0x72, 0xb7, 0x28, 0xa8, + 0x0d, 0x7a, 0xe6, 0xe2, + ] + ); + Ok(()) +} + +#[test] +fn retention_generations_are_positive_checked_and_semantically_distinct() +-> Result<(), Box> { + assert_eq!(RootGeneration::new(0), Err(RootGenerationError::Zero)); + assert_eq!( + LivenessGeneration::new(0), + Err(LivenessGenerationError::Zero) + ); + + let root = RootGeneration::new(1)?; + let liveness = LivenessGeneration::new(1)?; + assert_eq!(root.successor().map(RootGeneration::get), Ok(2)); + assert_eq!(liveness.successor().map(LivenessGeneration::get), Ok(2)); + assert_eq!( + RootGeneration::new(u64::MAX).and_then(RootGeneration::successor), + Err(RootGenerationError::Exhausted { current: u64::MAX }) + ); + assert_eq!( + LivenessGeneration::new(u64::MAX).and_then(LivenessGeneration::successor), + Err(LivenessGenerationError::Exhausted { current: u64::MAX }) + ); + Ok(()) +} + +#[test] +fn retention_anchors_preserve_exact_logical_and_layout_coordinates() +-> Result<(), Box> { + let blob: BlobId = ONE_ZERO_BLOB.parse()?; + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + let anchor = RetentionAnchor::new(blob, layout); + assert_eq!(anchor.blob_id(), blob); + assert_eq!(anchor.layout_id(), layout); + Ok(()) +} From 8200c5393ae48b604c7c18ef688a156ccd381d0a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 20:21:06 -0700 Subject: [PATCH 04/50] Add: Encode canonical retention roots --- CHANGELOG.md | 9 +- docs/formats/segment-store-v2/README.md | 10 +- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 8 + src/adapters/retention/canonical_root.rs | 42 ++++ src/adapters/retention/root_encode_error.rs | 48 +++++ src/adapters/retention/root_encoder.rs | 150 +++++++++++++++ src/lib.rs | 55 +++--- src/retention/closure_limit.rs | 27 +++ src/retention/closure_limit_error.rs | 43 +++++ src/retention/closure_limits.rs | 110 +++++++++++ src/retention/mod.rs | 18 ++ src/retention/policy.rs | 28 +++ src/retention/profile.rs | 75 ++++++++ src/retention/profile_admission_error.rs | 51 +++++ src/retention/root.rs | 128 +++++++++++++ src/retention/root_digest.rs | 18 ++ src/retention/root_error.rs | 59 ++++++ tests/retention_root_encoding.rs | 180 ++++++++++++++++++ 20 files changed, 1032 insertions(+), 36 deletions(-) create mode 100644 src/adapters/retention.rs create mode 100644 src/adapters/retention/canonical_root.rs create mode 100644 src/adapters/retention/root_encode_error.rs create mode 100644 src/adapters/retention/root_encoder.rs create mode 100644 src/retention/closure_limit.rs create mode 100644 src/retention/closure_limit_error.rs create mode 100644 src/retention/closure_limits.rs create mode 100644 src/retention/policy.rs create mode 100644 src/retention/profile.rs create mode 100644 src/retention/profile_admission_error.rs create mode 100644 src/retention/root.rs create mode 100644 src/retention/root_digest.rs create mode 100644 src/retention/root_error.rs create mode 100644 tests/retention_root_encoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dd30ed6..7ea1ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -307,10 +307,11 @@ after its public API and format compatibility policies are established. liveness manifests, reader snapshots, one-way staged migration, exact crash boundaries, and reserved GC/disposition records. Validated public `RetentionNamespace`, namespace-digest, `RootGeneration`, - `LivenessGeneration`, and `RetentionAnchor` values now establish the core - boundary. Version-1 immutable bytes remain authoritative; production - version-2 writing remains unavailable until issue #19's executable evidence - is complete. + `LivenessGeneration`, `RetentionAnchor`, realization profile, closure limits, + and semantic root values now establish the core boundary. The canonical root + encoder reproduces the independent version-2 golden bytes. Version-1 + immutable bytes remain authoritative; production version-2 writing remains + unavailable until issue #19's executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 3fe3d8b..63e3731 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -68,9 +68,11 @@ re-encode them. The format contract is frozen by ADR-0009 and this specification. Public core types now admit exact namespace bytes, namespace digests, root and liveness -generations, and reconstruction anchors. No production version-2 parser, -transition, migration, or writer exists yet. Requirements that remain marked -as planned in issue #19 or issue #21 are not implementation evidence. A store -must refuse version-2 state until the relevant parser, corruption, +generations, registered realization profiles, bounded closure policies, +reconstruction anchors, and semantic roots. The canonical root encoder matches +the independent golden record. No production version-2 decoder, transition, +migration, or writer exists yet. Requirements that remain marked as planned or +in progress in issue #19 or issue #21 are not complete implementation evidence. +A store must refuse version-2 state until the relevant parser, corruption, golden-format, model-based, crash-injection, recovery, and fuzz evidence is implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e57e60b..c0ffe91 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -9,8 +9,8 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` covers namespace, digest, generations, and anchors; profile and limit evidence remains | In progress in #19 | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | golden-format fixtures plus independent oracle | Planned in #19 | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder evidence in `tests/retention_root_encoding.rs`; root decoder and manifest/head codecs remain | In progress in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index ead1ffd..77c463a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -3,7 +3,8 @@ //! This module owns decoding raw input into validated domain types, encoding //! validated domain types into canonical bytes, and exact immutable-segment //! ingress and egress. It does not own identity calculation, logical layout -//! policy, physical location, namespace publication, recovery, or retention. +//! policy, physical location, namespace publication, recovery, or retention +//! policy. mod admitted_catalog; mod admitted_recovery_stage_bytes; @@ -233,6 +234,7 @@ mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; +mod retention; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -447,6 +449,7 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; +pub use retention::{CanonicalRetentionRoot, RetentionRootEncodeError}; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs new file mode 100644 index 0000000..d2a8653 --- /dev/null +++ b/src/adapters/retention.rs @@ -0,0 +1,8 @@ +//! This module owns canonical retention record boundary adapters. + +mod canonical_root; +mod root_encode_error; +mod root_encoder; + +pub use canonical_root::CanonicalRetentionRoot; +pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/canonical_root.rs b/src/adapters/retention/canonical_root.rs new file mode 100644 index 0000000..dfdaf64 --- /dev/null +++ b/src/adapters/retention/canonical_root.rs @@ -0,0 +1,42 @@ +//! This boundary module owns materialized canonical retention root bytes. + +use super::{RetentionRootEncodeError, root_encoder}; +use crate::{RetentionRoot, RetentionRootDigest}; + +/// Owned canonical version-2 retention root record. +/// +/// The complete record is materialized in memory after semantic bounds are +/// admitted and exact checked length calculation succeeds. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct CanonicalRetentionRoot { + encoded: Vec, + digest: RetentionRootDigest, +} + +impl CanonicalRetentionRoot { + /// Encodes one validated semantic retention root. + /// + /// # Errors + /// + /// Returns [`RetentionRootEncodeError`] for checked length overflow, + /// allocation refusal, or an internal construction-length mismatch. + pub fn from_root(root: &RetentionRoot) -> Result { + root_encoder::encode(root) + } + + /// Returns the complete canonical root bytes. + #[must_use] + pub fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the canonical root digest embedded in the record. + pub const fn digest(&self) -> RetentionRootDigest { + self.digest + } + + pub(super) const fn admitted(encoded: Vec, digest: RetentionRootDigest) -> Self { + Self { encoded, digest } + } +} diff --git a/src/adapters/retention/root_encode_error.rs b/src/adapters/retention/root_encode_error.rs new file mode 100644 index 0000000..eea750c --- /dev/null +++ b/src/adapters/retention/root_encode_error.rs @@ -0,0 +1,48 @@ +//! This boundary module owns typed retention root encoding failures. + +use std::collections::TryReserveError; +use std::error::Error; +use std::fmt; + +/// Failure to materialize one canonical retention root record. +#[derive(Debug)] +pub enum RetentionRootEncodeError { + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// Exact record allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// Construction produced a length different from its admitted plan. + ConstructionLength { + /// Planned exact length. + expected: usize, + /// Materialized length. + observed: usize, + }, +} + +impl fmt::Display for RetentionRootEncodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LengthOverflow => formatter.write_str("retention root record length overflow"), + Self::Allocation { .. } => { + formatter.write_str("retention root record allocation failed") + } + Self::ConstructionLength { expected, observed } => write!( + formatter, + "retention root construction produced {observed} bytes; expected {expected}" + ), + } + } +} + +impl Error for RetentionRootEncodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Allocation { source } => Some(source), + Self::LengthOverflow | Self::ConstructionLength { .. } => None, + } + } +} diff --git a/src/adapters/retention/root_encoder.rs b/src/adapters/retention/root_encoder.rs new file mode 100644 index 0000000..b80c1d6 --- /dev/null +++ b/src/adapters/retention/root_encoder.rs @@ -0,0 +1,150 @@ +//! This boundary module owns canonical version-2 retention root encoding. + +use super::{CanonicalRetentionRoot, RetentionRootEncodeError}; +use crate::{RetentionRoot, RetentionRootDigest}; + +const HEADER_LENGTH: usize = 192; +const ANCHOR_WIDTH: usize = 119; +const TRAILER_LENGTH: usize = 64; + +struct EncodingPlan { + total_length: usize, + digest_preimage_length: usize, + anchor_set_digest: [u8; 32], +} + +pub(super) fn encode( + root: &RetentionRoot, +) -> Result { + let plan = plan(root)?; + let mut encoded = Vec::new(); + encoded + .try_reserve_exact(plan.total_length) + .map_err(|source| RetentionRootEncodeError::Allocation { source })?; + write_header(&mut encoded, root, &plan)?; + write_body(&mut encoded, root); + require_length(&encoded, plan.digest_preimage_length)?; + let digest = hash(b"keep.retention-root/v2\0", &encoded); + encoded.extend_from_slice(&digest); + let checksum = hash(b"keep.retention-root-checksum/v2\0", &encoded); + encoded.extend_from_slice(&checksum); + require_length(&encoded, plan.total_length)?; + Ok(CanonicalRetentionRoot::admitted( + encoded, + RetentionRootDigest::from_hash(digest), + )) +} + +fn plan(root: &RetentionRoot) -> Result { + let anchor_bytes = usize::try_from(root.anchor_count()) + .map_err(|_| RetentionRootEncodeError::LengthOverflow)? + .checked_mul(ANCHOR_WIDTH) + .ok_or(RetentionRootEncodeError::LengthOverflow)?; + let digest_preimage_length = HEADER_LENGTH + .checked_add(root.namespace().as_bytes().len()) + .and_then(|length| length.checked_add(anchor_bytes)) + .ok_or(RetentionRootEncodeError::LengthOverflow)?; + let total_length = digest_preimage_length + .checked_add(TRAILER_LENGTH) + .ok_or(RetentionRootEncodeError::LengthOverflow)?; + Ok(EncodingPlan { + total_length, + digest_preimage_length, + anchor_set_digest: anchor_set_digest(root), + }) +} + +fn anchor_set_digest(root: &RetentionRoot) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-anchor-set/v2\0"); + hasher.update(&root.anchor_count().to_be_bytes()); + for anchor in root.anchors() { + hasher.update(&anchor.blob_id().encode_binary()); + hasher.update(&anchor.layout_id().encode_binary()); + } + *hasher.finalize().as_bytes() +} + +fn write_header( + encoded: &mut Vec, + root: &RetentionRoot, + plan: &EncodingPlan, +) -> Result<(), RetentionRootEncodeError> { + encoded.extend_from_slice(b"KEEP:RET:ROOT2\0\0"); + push_u16(encoded, 2); + push_u16(encoded, 192); + push_u32(encoded, 0); + push_u64( + encoded, + u64::try_from(plan.total_length).map_err(|_| RetentionRootEncodeError::LengthOverflow)?, + ); + push_u64(encoded, root.generation().get()); + push_u16( + encoded, + u16::try_from(root.namespace().as_bytes().len()) + .map_err(|_| RetentionRootEncodeError::LengthOverflow)?, + ); + push_u16(encoded, 119); + push_u32(encoded, root.anchor_count()); + write_policy(encoded, root); + encoded.extend_from_slice(&predecessor_bytes(root)); + encoded.extend_from_slice(&plan.anchor_set_digest); + encoded.extend_from_slice(&[0_u8; 12]); + require_length(encoded, HEADER_LENGTH) +} + +fn write_policy(encoded: &mut Vec, root: &RetentionRoot) { + let profile = root.profile(); + let limits = root.limits(); + push_u32(encoded, profile.identity()); + push_u32(encoded, profile.version()); + encoded.extend_from_slice(profile.digest()); + push_u64(encoded, limits.nodes()); + push_u16(encoded, limits.depth()); + push_u16(encoded, 0); + push_u64(encoded, limits.encoded_bytes()); + push_u64(encoded, limits.physical_bytes()); +} + +fn predecessor_bytes(root: &RetentionRoot) -> [u8; 32] { + root.predecessor() + .map_or([0_u8; 32], |digest| *digest.as_bytes()) +} + +fn write_body(encoded: &mut Vec, root: &RetentionRoot) { + encoded.extend_from_slice(root.namespace().as_bytes()); + for anchor in root.anchors() { + encoded.extend_from_slice(&anchor.blob_id().encode_binary()); + encoded.extend_from_slice(&anchor.layout_id().encode_binary()); + } +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} + +fn push_u16(encoded: &mut Vec, value: u16) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(encoded: &mut Vec, value: u32) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u64(encoded: &mut Vec, value: u64) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +const fn require_length(encoded: &[u8], expected: usize) -> Result<(), RetentionRootEncodeError> { + if encoded.len() == expected { + Ok(()) + } else { + Err(RetentionRootEncodeError::ConstructionLength { + expected, + observed: encoded.len(), + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 88026b5..72dc778 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,8 @@ //! contract and a pinned writer-authorized filesystem adapter. Reusable-stage //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, -//! generations, and reconstruction anchors are validated. Retention +//! generations, realization policy, reconstruction anchors, and semantic roots +//! are validated; canonical root encoding is available. Retention decoding, //! publication, recovery, and garbage collection remain intentionally absent. #[cfg(test)] @@ -42,14 +43,14 @@ pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, + CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, + CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, @@ -83,20 +84,21 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, - SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, - SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, + SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, + SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, + SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, + SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, + StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, + StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, + WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, + classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, + classify_recovery_segment_stage, execute_recovery_next_head_finalization, + execute_recovery_segment_resume, execute_recovery_stage_completion, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; @@ -121,6 +123,9 @@ pub use reference::{ ReferenceStoreCapacity, StagedBlob, }; pub use retention::{ - LivenessGeneration, LivenessGenerationError, RetentionAnchor, RetentionNamespace, - RetentionNamespaceDigest, RetentionNamespaceError, RootGeneration, RootGenerationError, + LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionNamespace, + RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, + RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, + RootGeneration, RootGenerationError, }; diff --git a/src/retention/closure_limit.rs b/src/retention/closure_limit.rs new file mode 100644 index 0000000..378b096 --- /dev/null +++ b/src/retention/closure_limit.rs @@ -0,0 +1,27 @@ +//! This module owns semantic names for bounded closure resources. + +use std::fmt; + +/// One independently bounded retention closure resource. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RetentionClosureLimit { + /// Number of logical and physical closure nodes. + Nodes, + /// Maximum traversal depth. + Depth, + /// Total encoded bytes inspected. + EncodedBytes, + /// Total physical bytes inspected. + PhysicalBytes, +} + +impl fmt::Display for RetentionClosureLimit { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Nodes => "closure nodes", + Self::Depth => "closure depth", + Self::EncodedBytes => "encoded bytes", + Self::PhysicalBytes => "physical bytes", + }) + } +} diff --git a/src/retention/closure_limit_error.rs b/src/retention/closure_limit_error.rs new file mode 100644 index 0000000..156f333 --- /dev/null +++ b/src/retention/closure_limit_error.rs @@ -0,0 +1,43 @@ +//! This module owns typed retention closure-limit failures. + +use std::error::Error; +use std::fmt; + +use super::RetentionClosureLimit; + +/// Failure to admit one bounded closure resource. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionClosureLimitError { + /// A required positive limit was zero. + Zero { + /// Resource whose limit was zero. + limit: RetentionClosureLimit, + }, + /// A limit exceeded its fixed implementation ceiling. + AboveMaximum { + /// Resource whose limit was excessive. + limit: RetentionClosureLimit, + /// Fixed implementation ceiling. + maximum: u64, + /// Caller-observed limit. + observed: u64, + }, +} + +impl fmt::Display for RetentionClosureLimitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero { limit } => write!(formatter, "retention {limit} limit must be positive"), + Self::AboveMaximum { + limit, + maximum, + observed, + } => write!( + formatter, + "retention {limit} limit {observed} exceeds maximum {maximum}" + ), + } + } +} + +impl Error for RetentionClosureLimitError {} diff --git a/src/retention/closure_limits.rs b/src/retention/closure_limits.rs new file mode 100644 index 0000000..ad73135 --- /dev/null +++ b/src/retention/closure_limits.rs @@ -0,0 +1,110 @@ +//! This module owns one fully admitted retention closure resource policy. + +use std::num::{NonZeroU16, NonZeroU64}; + +use super::{RetentionClosureLimit, RetentionClosureLimitError}; + +/// Positive closure limits bounded by the version-2 implementation ceilings. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionClosureLimits { + nodes: NonZeroU64, + depth: NonZeroU16, + encoded_bytes: NonZeroU64, + physical_bytes: NonZeroU64, +} + +impl RetentionClosureLimits { + /// Maximum admitted closure node count. + pub const MAXIMUM_NODES: u64 = 1_048_576; + /// Maximum admitted traversal depth. + pub const MAXIMUM_DEPTH: u16 = 8; + /// Maximum admitted encoded bytes. + pub const MAXIMUM_ENCODED_BYTES: u64 = 16_777_216; + /// Maximum admitted physical bytes. + pub const MAXIMUM_PHYSICAL_BYTES: u64 = 1_073_741_824; + + /// Admits one complete positive, ceiling-bounded policy. + /// + /// # Errors + /// + /// Returns the first zero or above-maximum limit in argument order. + pub fn new( + nodes: u64, + depth: u16, + encoded_bytes: u64, + physical_bytes: u64, + ) -> Result { + let nodes = admit_u64(RetentionClosureLimit::Nodes, nodes, Self::MAXIMUM_NODES)?; + let depth = admit_depth(depth)?; + let encoded_bytes = admit_u64( + RetentionClosureLimit::EncodedBytes, + encoded_bytes, + Self::MAXIMUM_ENCODED_BYTES, + )?; + let physical_bytes = admit_u64( + RetentionClosureLimit::PhysicalBytes, + physical_bytes, + Self::MAXIMUM_PHYSICAL_BYTES, + )?; + Ok(Self { + nodes, + depth, + encoded_bytes, + physical_bytes, + }) + } + + /// Returns the positive closure node limit. + #[must_use] + pub const fn nodes(self) -> u64 { + self.nodes.get() + } + + /// Returns the positive closure depth limit. + #[must_use] + pub const fn depth(self) -> u16 { + self.depth.get() + } + + /// Returns the positive encoded-byte limit. + #[must_use] + pub const fn encoded_bytes(self) -> u64 { + self.encoded_bytes.get() + } + + /// Returns the positive physical-byte limit. + #[must_use] + pub const fn physical_bytes(self) -> u64 { + self.physical_bytes.get() + } +} + +fn admit_u64( + limit: RetentionClosureLimit, + observed: u64, + maximum: u64, +) -> Result { + let value = NonZeroU64::new(observed).ok_or(RetentionClosureLimitError::Zero { limit })?; + if observed > maximum { + return Err(RetentionClosureLimitError::AboveMaximum { + limit, + maximum, + observed, + }); + } + Ok(value) +} + +fn admit_depth(observed: u16) -> Result { + let limit = RetentionClosureLimit::Depth; + let value = NonZeroU16::new(observed).ok_or(RetentionClosureLimitError::Zero { limit })?; + if observed > RetentionClosureLimits::MAXIMUM_DEPTH { + return Err(RetentionClosureLimitError::AboveMaximum { + limit, + maximum: u64::from(RetentionClosureLimits::MAXIMUM_DEPTH), + observed: u64::from(observed), + }); + } + Ok(value) +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 43494e9..a03e461 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -6,19 +6,37 @@ //! collection. mod anchor; +mod closure_limit; +mod closure_limit_error; +mod closure_limits; mod liveness_generation; mod liveness_generation_error; mod namespace; mod namespace_digest; mod namespace_error; +mod policy; +mod profile; +mod profile_admission_error; +mod root; +mod root_digest; +mod root_error; mod root_generation; mod root_generation_error; pub use anchor::RetentionAnchor; +pub use closure_limit::RetentionClosureLimit; +pub use closure_limit_error::RetentionClosureLimitError; +pub use closure_limits::RetentionClosureLimits; pub use liveness_generation::LivenessGeneration; pub use liveness_generation_error::LivenessGenerationError; pub use namespace::RetentionNamespace; pub use namespace_digest::RetentionNamespaceDigest; pub use namespace_error::RetentionNamespaceError; +pub use policy::RetentionPolicy; +pub use profile::RegisteredRetentionProfile; +pub use profile_admission_error::RetentionProfileAdmissionError; +pub use root::RetentionRoot; +pub use root_digest::RetentionRootDigest; +pub use root_error::RetentionRootError; pub use root_generation::RootGeneration; pub use root_generation_error::RootGenerationError; diff --git a/src/retention/policy.rs b/src/retention/policy.rs new file mode 100644 index 0000000..04b07a8 --- /dev/null +++ b/src/retention/policy.rs @@ -0,0 +1,28 @@ +//! This module owns one registered, bounded retention realization policy. + +use super::{RegisteredRetentionProfile, RetentionClosureLimits}; + +/// Registered realization semantics paired with caller-selected closure limits. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionPolicy { + profile: RegisteredRetentionProfile, + limits: RetentionClosureLimits, +} + +impl RetentionPolicy { + /// Combines one registered profile with already-admitted closure limits. + pub const fn new(profile: RegisteredRetentionProfile, limits: RetentionClosureLimits) -> Self { + Self { profile, limits } + } + + /// Returns the registered realization profile. + pub const fn profile(self) -> RegisteredRetentionProfile { + self.profile + } + + /// Returns the admitted closure limits. + pub const fn limits(self) -> RetentionClosureLimits { + self.limits + } +} diff --git a/src/retention/profile.rs b/src/retention/profile.rs new file mode 100644 index 0000000..3158037 --- /dev/null +++ b/src/retention/profile.rs @@ -0,0 +1,75 @@ +//! This module owns the closed registered retention realization-profile set. + +use super::RetentionProfileAdmissionError; + +const PROFILE_DIGEST: [u8; 32] = [ + 0xdb, 0x1c, 0x1c, 0x1a, 0x50, 0x61, 0x3e, 0xf1, 0x1f, 0x7c, 0x0e, 0xe0, 0x88, 0x2e, 0x37, 0xb6, + 0xd2, 0x4e, 0x2d, 0xb2, 0xca, 0x57, 0x78, 0x3d, 0x01, 0x19, 0x7b, 0xa5, 0x1b, 0x61, 0xce, 0x59, +]; + +/// One deterministic retention realization profile implemented by Keep. +/// +/// The type has private representation so future registered profiles remain +/// an additive registry change rather than an exhaustive-enum break. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RegisteredRetentionProfile { + identity: u32, + version: u32, + digest: [u8; 32], +} + +impl RegisteredRetentionProfile { + /// The single-canonical-witness version-1 profile. + pub const SINGLE_CANONICAL_WITNESS_V1: Self = Self { + identity: 1, + version: 1, + digest: PROFILE_DIGEST, + }; + + /// Admits an exact registered profile coordinate. + /// + /// # Errors + /// + /// Returns a typed coordinate or definition-digest mismatch. + pub fn admit( + identity: u32, + version: u32, + digest: [u8; 32], + ) -> Result { + let expected = Self::SINGLE_CANONICAL_WITNESS_V1; + if identity != expected.identity || version != expected.version { + return Err(RetentionProfileAdmissionError::UnsupportedCoordinate { + expected_identity: expected.identity, + expected_version: expected.version, + observed_identity: identity, + observed_version: version, + }); + } + if digest != expected.digest { + return Err(RetentionProfileAdmissionError::DefinitionDigestMismatch { + expected: expected.digest, + observed: digest, + }); + } + Ok(expected) + } + + /// Returns the registered integer identity. + #[must_use] + pub const fn identity(self) -> u32 { + self.identity + } + + /// Returns the registered profile version. + #[must_use] + pub const fn version(self) -> u32 { + self.version + } + + /// Returns the exact registered definition digest. + #[must_use] + pub const fn digest(&self) -> &[u8; 32] { + &self.digest + } +} diff --git a/src/retention/profile_admission_error.rs b/src/retention/profile_admission_error.rs new file mode 100644 index 0000000..1be8dc8 --- /dev/null +++ b/src/retention/profile_admission_error.rs @@ -0,0 +1,51 @@ +//! This module owns typed retention-profile admission failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit a retention realization-profile coordinate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionProfileAdmissionError { + /// The identity and version pair is not registered. + UnsupportedCoordinate { + /// Registered identity expected by this Keep version. + expected_identity: u32, + /// Registered version expected by this Keep version. + expected_version: u32, + /// Identity observed at the boundary. + observed_identity: u32, + /// Version observed at the boundary. + observed_version: u32, + }, + /// The registered coordinate carried different definition bytes. + DefinitionDigestMismatch { + /// Exact registered definition digest. + expected: [u8; 32], + /// Digest observed at the boundary. + observed: [u8; 32], + }, +} + +impl fmt::Display for RetentionProfileAdmissionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnsupportedCoordinate { + expected_identity, + expected_version, + observed_identity, + observed_version, + } => write!( + formatter, + "unsupported retention profile {observed_identity}/{observed_version}; \ + expected {expected_identity}/{expected_version}" + ), + Self::DefinitionDigestMismatch { expected, observed } => write!( + formatter, + "retention profile definition digest mismatch: expected {expected:02x?}, \ + observed {observed:02x?}" + ), + } + } +} + +impl Error for RetentionProfileAdmissionError {} diff --git a/src/retention/root.rs b/src/retention/root.rs new file mode 100644 index 0000000..964876c --- /dev/null +++ b/src/retention/root.rs @@ -0,0 +1,128 @@ +//! This module owns one validated semantic retention root generation. + +use super::{ + RegisteredRetentionProfile, RetentionAnchor, RetentionClosureLimits, RetentionNamespace, + RetentionPolicy, RetentionRootDigest, RetentionRootError, RootGeneration, +}; + +/// One canonical namespace root generation before durable byte encoding. +/// +/// Construction canonicalizes the caller's `Vec` in place, rejects duplicate +/// or excessive anchors, and consumes it into an immutable boxed slice. The +/// boxed-slice conversion may discard excess capacity. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetentionRoot { + namespace: RetentionNamespace, + generation: RootGeneration, + policy: RetentionPolicy, + predecessor: Option, + anchors: Box<[RetentionAnchor]>, + anchor_count: u32, +} + +impl RetentionRoot { + /// Maximum anchors admitted in one namespace generation. + pub const MAXIMUM_ANCHOR_COUNT: u32 = 65_536; + + /// Admits one deterministic semantic root. + /// + /// Anchors are sorted into canonical order. Duplicate anchors refuse + /// instead of being silently removed. + /// + /// # Errors + /// + /// Returns a typed predecessor, anchor-count, or duplicate refusal. + pub fn new( + namespace: RetentionNamespace, + generation: RootGeneration, + policy: RetentionPolicy, + predecessor: Option, + mut anchors: Vec, + ) -> Result { + admit_predecessor(generation, predecessor)?; + let observed = anchors.len(); + let anchor_count = + u32::try_from(observed).map_err(|_| RetentionRootError::AnchorCountExceeded { + maximum: Self::MAXIMUM_ANCHOR_COUNT, + observed, + })?; + if anchor_count > Self::MAXIMUM_ANCHOR_COUNT { + return Err(RetentionRootError::AnchorCountExceeded { + maximum: Self::MAXIMUM_ANCHOR_COUNT, + observed, + }); + } + anchors.sort_unstable(); + refuse_duplicate(&anchors)?; + Ok(Self { + namespace, + generation, + policy, + predecessor, + anchors: anchors.into_boxed_slice(), + anchor_count, + }) + } + + /// Returns the exact opaque namespace. + pub const fn namespace(&self) -> &RetentionNamespace { + &self.namespace + } + + /// Returns the per-namespace root generation. + pub const fn generation(&self) -> RootGeneration { + self.generation + } + + /// Returns the registered realization profile. + pub const fn profile(&self) -> RegisteredRetentionProfile { + self.policy.profile() + } + + /// Returns the admitted closure limits. + pub const fn limits(&self) -> RetentionClosureLimits { + self.policy.limits() + } + + /// Returns the exact predecessor, absent only for generation one. + #[must_use] + pub const fn predecessor(&self) -> Option { + self.predecessor + } + + /// Returns the canonical, duplicate-free anchors. + pub fn anchors(&self) -> &[RetentionAnchor] { + &self.anchors + } + + /// Returns the bounded anchor count. + #[must_use] + pub const fn anchor_count(&self) -> u32 { + self.anchor_count + } +} + +const fn admit_predecessor( + generation: RootGeneration, + predecessor: Option, +) -> Result<(), RetentionRootError> { + match (generation.get(), predecessor) { + (1, Some(observed)) => { + Err(RetentionRootError::InitialGenerationHasPredecessor { observed }) + } + (1, None) | (_, Some(_)) => Ok(()), + (_, None) => Err(RetentionRootError::MissingPredecessor { generation }), + } +} + +fn refuse_duplicate(anchors: &[RetentionAnchor]) -> Result<(), RetentionRootError> { + let mut previous = None; + for anchor in anchors { + if previous == Some(*anchor) { + return Err(RetentionRootError::DuplicateAnchor { anchor: *anchor }); + } + previous = Some(*anchor); + } + Ok(()) +} diff --git a/src/retention/root_digest.rs b/src/retention/root_digest.rs new file mode 100644 index 0000000..c398fb7 --- /dev/null +++ b/src/retention/root_digest.rs @@ -0,0 +1,18 @@ +//! This module owns canonical retention root identity. + +/// Canonical BLAKE3-256 identity of one complete retention root record. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionRootDigest([u8; 32]); + +impl RetentionRootDigest { + pub(crate) const fn from_hash(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the exact 32 digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/retention/root_error.rs b/src/retention/root_error.rs new file mode 100644 index 0000000..2360303 --- /dev/null +++ b/src/retention/root_error.rs @@ -0,0 +1,59 @@ +//! This module owns typed semantic retention root failures. + +use std::error::Error; +use std::fmt; + +use super::{RetentionAnchor, RetentionRootDigest, RootGeneration}; + +/// Failure to construct one canonical semantic retention root. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionRootError { + /// Generation one carried an impossible predecessor. + InitialGenerationHasPredecessor { + /// Observed predecessor digest. + observed: RetentionRootDigest, + }, + /// A successor generation omitted its required predecessor. + MissingPredecessor { + /// Successor generation lacking a predecessor. + generation: RootGeneration, + }, + /// The caller supplied too many anchors. + AnchorCountExceeded { + /// Fixed maximum anchor count. + maximum: u32, + /// Observed anchor count. + observed: usize, + }, + /// The caller supplied the same anchor more than once. + DuplicateAnchor { + /// Exact duplicated anchor. + anchor: RetentionAnchor, + }, +} + +impl fmt::Display for RetentionRootError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGenerationHasPredecessor { observed } => write!( + formatter, + "initial retention root has predecessor {:?}", + observed.as_bytes() + ), + Self::MissingPredecessor { generation } => write!( + formatter, + "retention root generation {} requires a predecessor", + generation.get() + ), + Self::AnchorCountExceeded { maximum, observed } => write!( + formatter, + "retention root has {observed} anchors; maximum is {maximum}" + ), + Self::DuplicateAnchor { anchor } => { + write!(formatter, "retention root repeats anchor {anchor:?}") + } + } + } +} + +impl Error for RetentionRootError {} diff --git a/tests/retention_root_encoding.rs b/tests/retention_root_encoding.rs new file mode 100644 index 0000000..446ece2 --- /dev/null +++ b/tests/retention_root_encoding.rs @@ -0,0 +1,180 @@ +//! Public construction laws for canonical version-2 retention roots. + +mod support; + +use std::io; + +use keep::{ + BlobId, CanonicalRetentionRoot, LayoutId, RegisteredRetentionProfile, RetentionAnchor, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionNamespace, + RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootError, + RootGeneration, +}; + +const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const EMPTY_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:0:", + "c0074a279c09f9d019dc10e4c821f79f1450cfb8541ab4627132ab9f3c75e33f" +); +const ONE_ZERO_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:1:", + "1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" +); +const ONE_ZERO_LAYOUT: &str = concat!( + "keep:layout:v1:flat-chunks-v1:blake3-256:220:", + "887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8" +); + +#[test] +fn canonical_root_reproduces_the_frozen_one_anchor_record() -> Result<(), Box> +{ + let root = one_anchor_root()?; + let canonical = CanonicalRetentionRoot::from_root(&root)?; + assert_eq!(canonical.encoded(), fixture_bytes()?); + assert_eq!( + canonical.digest().as_bytes(), + &[ + 0xca, 0x4c, 0x11, 0xf2, 0x65, 0xc3, 0xbe, 0xd0, 0x70, 0x73, 0xbd, 0xc3, 0xb6, 0xae, + 0xf0, 0x03, 0xe9, 0x64, 0xac, 0x8c, 0xb3, 0x6f, 0xcf, 0xcc, 0x92, 0xf2, 0x0f, 0xa6, + 0xf0, 0xb6, 0x00, 0x85, + ] + ); + Ok(()) +} + +#[test] +fn closure_limits_refuse_zero_and_excess_before_root_construction() { + assert_eq!( + RetentionClosureLimits::new(0, 2, 4_096, 4_096), + Err(RetentionClosureLimitError::Zero { + limit: RetentionClosureLimit::Nodes, + }) + ); + assert_eq!( + RetentionClosureLimits::new(4, 2, 4_096, 1_073_741_825), + Err(RetentionClosureLimitError::AboveMaximum { + limit: RetentionClosureLimit::PhysicalBytes, + maximum: 1_073_741_824, + observed: 1_073_741_825, + }) + ); +} + +#[test] +fn realization_profile_admits_only_the_exact_registered_definition() { + let expected = RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1; + let digest = [ + 0xdb, 0x1c, 0x1c, 0x1a, 0x50, 0x61, 0x3e, 0xf1, 0x1f, 0x7c, 0x0e, 0xe0, 0x88, 0x2e, 0x37, + 0xb6, 0xd2, 0x4e, 0x2d, 0xb2, 0xca, 0x57, 0x78, 0x3d, 0x01, 0x19, 0x7b, 0xa5, 0x1b, 0x61, + 0xce, 0x59, + ]; + assert_eq!( + RegisteredRetentionProfile::admit(1, 1, digest), + Ok(expected) + ); + assert_eq!( + RegisteredRetentionProfile::admit(2, 1, digest), + Err(RetentionProfileAdmissionError::UnsupportedCoordinate { + expected_identity: 1, + expected_version: 1, + observed_identity: 2, + observed_version: 1, + }) + ); + assert_eq!( + RegisteredRetentionProfile::admit(1, 1, [0_u8; 32]), + Err(RetentionProfileAdmissionError::DefinitionDigestMismatch { + expected: digest, + observed: [0_u8; 32], + }) + ); +} + +#[test] +fn root_predecessors_and_anchor_sets_have_one_canonical_admission() +-> Result<(), Box> { + let initial = one_anchor_root()?; + let canonical = CanonicalRetentionRoot::from_root(&initial)?; + let predecessor = canonical.digest(); + let namespace = RetentionNamespace::try_from(vec![0x00, 0x2f, 0xff])?; + let limits = RetentionClosureLimits::new(4, 2, 4_096, 4_096)?; + let profile = RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1; + let policy = RetentionPolicy::new(profile, limits); + let anchor = one_zero_anchor()?; + let earlier_anchor = RetentionAnchor::new(EMPTY_BLOB.parse()?, anchor.layout_id()); + + assert_eq!( + RetentionRoot::new( + namespace.clone(), + RootGeneration::new(1)?, + policy, + Some(predecessor), + vec![anchor], + ), + Err(RetentionRootError::InitialGenerationHasPredecessor { + observed: predecessor, + }) + ); + assert_eq!( + RetentionRoot::new( + namespace.clone(), + RootGeneration::new(2)?, + policy, + None, + vec![anchor], + ), + Err(RetentionRootError::MissingPredecessor { + generation: RootGeneration::new(2)?, + }) + ); + assert_eq!( + RetentionRoot::new( + namespace.clone(), + RootGeneration::new(2)?, + policy, + Some(predecessor), + vec![anchor, anchor], + ), + Err(RetentionRootError::DuplicateAnchor { anchor }) + ); + let sorted = RetentionRoot::new( + namespace, + RootGeneration::new(2)?, + policy, + Some(predecessor), + vec![anchor, earlier_anchor], + )?; + assert_eq!(sorted.anchors(), &[earlier_anchor, anchor]); + Ok(()) +} + +fn one_anchor_root() -> Result> { + Ok(RetentionRoot::new( + RetentionNamespace::try_from(vec![0x00, 0x2f, 0xff])?, + RootGeneration::new(1)?, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(4, 2, 4_096, 4_096)?, + ), + None, + vec![one_zero_anchor()?], + )?) +} + +fn one_zero_anchor() -> Result> { + let blob: BlobId = ONE_ZERO_BLOB.parse()?; + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + Ok(RetentionAnchor::new(blob, layout)) +} + +fn fixture_bytes() -> Result, io::Error> { + let encoded = ONE_ANCHOR_ROOT + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention root fixture lacks final newline"))?; + if encoded.contains(['\n', '\r']) { + return Err(io::Error::other( + "retention root fixture contains an embedded line ending", + )); + } + support::decode_hex(encoded) +} From 741068c42870f684832acc040af5881ee4d3f57b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 20:44:17 -0700 Subject: [PATCH 05/50] Add: Decode canonical retention roots --- CHANGELOG.md | 9 +- README.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 5 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 11 + src/adapters/retention/admitted_root.rs | 58 ++++++ src/adapters/retention/root_anchor_decoder.rs | 47 +++++ src/adapters/retention/root_decode_error.rs | 150 ++++++++++++++ .../retention/root_decode_error_display.rs | 122 +++++++++++ src/adapters/retention/root_decoder.rs | 50 +++++ src/adapters/retention/root_field_decoder.rs | 106 ++++++++++ src/adapters/retention/root_header_decoder.rs | 111 ++++++++++ src/adapters/retention/root_integrity.rs | 82 ++++++++ .../retention/root_semantic_header.rs | 55 +++++ src/lib.rs | 15 +- tests/retention_root_decoding.rs | 189 ++++++++++++++++++ 17 files changed, 1008 insertions(+), 16 deletions(-) create mode 100644 src/adapters/retention/admitted_root.rs create mode 100644 src/adapters/retention/root_anchor_decoder.rs create mode 100644 src/adapters/retention/root_decode_error.rs create mode 100644 src/adapters/retention/root_decode_error_display.rs create mode 100644 src/adapters/retention/root_decoder.rs create mode 100644 src/adapters/retention/root_field_decoder.rs create mode 100644 src/adapters/retention/root_header_decoder.rs create mode 100644 src/adapters/retention/root_integrity.rs create mode 100644 src/adapters/retention/root_semantic_header.rs create mode 100644 tests/retention_root_decoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ea1ccb..d9ecd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -309,9 +309,12 @@ after its public API and format compatibility policies are established. `RetentionNamespace`, namespace-digest, `RootGeneration`, `LivenessGeneration`, `RetentionAnchor`, realization profile, closure limits, and semantic root values now establish the core boundary. The canonical root - encoder reproduces the independent version-2 golden bytes. Version-1 - immutable bytes remain authoritative; production version-2 writing remains - unavailable until issue #19's executable evidence is complete. + encoder reproduces the independent version-2 golden bytes, and the decoder + verifies framing, checksum, root digest, anchor-set digest, nested identities, + resource bounds, canonical anchor order, and semantic invariants before + admission. Version-1 immutable bytes remain authoritative; production + version-2 writing remains unavailable until issue #19's executable evidence + is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 3fc5302..6cc0d33 100644 --- a/README.md +++ b/README.md @@ -115,9 +115,10 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Retention, compaction, and garbage collection remain planned. -Presence in the reference CAS does not claim retention, crash recovery, or -durability. +power loss. Version-2 retention values and canonical in-memory root encoding +and decoding are implemented. Retention publication, recovery, compaction, and +garbage collection remain planned. Presence in the reference CAS does not +claim retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index c0ffe91..aeebfad 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -10,7 +10,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder evidence in `tests/retention_root_encoding.rs`; root decoder and manifest/head codecs remain | In progress in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder and decoder evidence in `tests/retention_root_encoding.rs` and `tests/retention_root_decoding.rs`; manifest/head codecs remain | In progress in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 6462ce4..3f57f19 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -159,7 +159,10 @@ retention/roots// -.root ``` -Names with alternate width, case, suffix, generation, or digest refuse. +Names with alternate width, case, suffix, generation, or digest refuse. Keep +implements validated in-memory root encoding and decoding with complete +integrity verification before semantic admission. Filesystem publication, +manifest/head codecs, transitions, recovery, and garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 77c463a..111841a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -449,7 +449,10 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; -pub use retention::{CanonicalRetentionRoot, RetentionRootEncodeError}; +pub use retention::{ + AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionRootDecodeError, + RetentionRootEncodeError, +}; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index d2a8653..5fa3118 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -1,8 +1,19 @@ //! This module owns canonical retention record boundary adapters. +mod admitted_root; mod canonical_root; +mod root_anchor_decoder; +mod root_decode_error; +mod root_decode_error_display; +mod root_decoder; mod root_encode_error; mod root_encoder; +mod root_field_decoder; +mod root_header_decoder; +mod root_integrity; +mod root_semantic_header; +pub use admitted_root::AdmittedRetentionRoot; pub use canonical_root::CanonicalRetentionRoot; +pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/admitted_root.rs b/src/adapters/retention/admitted_root.rs new file mode 100644 index 0000000..ac0f33f --- /dev/null +++ b/src/adapters/retention/admitted_root.rs @@ -0,0 +1,58 @@ +//! This boundary module owns one decoded and admitted retention root. + +use super::{RetentionRootDecodeError, root_decoder}; +use crate::{RetentionRoot, RetentionRootDigest}; + +/// Borrowed canonical bytes paired with their admitted semantic root. +/// +/// Decoding verifies exact framing, the complete-record checksum, the root and +/// anchor-set digests, every nested identity, canonical anchor order, and all +/// semantic invariants. Anchor and namespace allocation is bounded by fields +/// admitted from the record. Decoding performs no I/O. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct AdmittedRetentionRoot<'encoded> { + encoded: &'encoded [u8], + root: RetentionRoot, + digest: RetentionRootDigest, +} + +impl<'encoded> AdmittedRetentionRoot<'encoded> { + /// Decodes and admits one exact canonical version-2 root record. + /// + /// # Errors + /// + /// Returns [`RetentionRootDecodeError`] at the first violated framing, + /// integrity, nested-codec, resource-bound, or semantic invariant. + pub fn decode(encoded: &'encoded [u8]) -> Result { + root_decoder::decode(encoded) + } + + /// Returns the complete verified canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the admitted semantic root. + pub const fn root(&self) -> &RetentionRoot { + &self.root + } + + /// Returns the verified canonical root digest. + pub const fn digest(&self) -> RetentionRootDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + root: RetentionRoot, + digest: RetentionRootDigest, + ) -> Self { + Self { + encoded, + root, + digest, + } + } +} diff --git a/src/adapters/retention/root_anchor_decoder.rs b/src/adapters/retention/root_anchor_decoder.rs new file mode 100644 index 0000000..44d2625 --- /dev/null +++ b/src/adapters/retention/root_anchor_decoder.rs @@ -0,0 +1,47 @@ +//! This boundary module owns canonical retention anchor body decoding. + +use super::RetentionRootDecodeError; +use crate::{BlobId, LayoutId, RetentionAnchor}; + +const BLOB_ID_WIDTH: usize = 59; +const ANCHOR_WIDTH: usize = 119; + +pub(super) fn decode( + encoded: &[u8], + anchor_count: u32, +) -> Result, RetentionRootDecodeError> { + let capacity = + usize::try_from(anchor_count).map_err(|_| RetentionRootDecodeError::LengthOverflow)?; + let mut anchors = Vec::new(); + anchors + .try_reserve_exact(capacity) + .map_err(|source| RetentionRootDecodeError::Allocation { source })?; + let mut previous = None; + for (position, bytes) in encoded.chunks_exact(ANCHOR_WIDTH).enumerate() { + let index = + u32::try_from(position).map_err(|_| RetentionRootDecodeError::LengthOverflow)?; + let (blob_bytes, layout_bytes) = bytes.split_at(BLOB_ID_WIDTH); + let blob_id = BlobId::parse_binary(blob_bytes) + .map_err(|source| RetentionRootDecodeError::BlobId { index, source })?; + let layout_id = LayoutId::parse_binary(layout_bytes) + .map_err(|source| RetentionRootDecodeError::LayoutId { index, source })?; + let observed = RetentionAnchor::new(blob_id, layout_id); + if let Some(prior) = previous + && observed <= prior + { + return Err(RetentionRootDecodeError::NonCanonicalAnchorOrder { index }); + } + anchors.push(observed); + previous = Some(observed); + } + if anchors.len() == capacity { + Ok(anchors) + } else { + Err(RetentionRootDecodeError::Truncated { + expected: capacity + .checked_mul(ANCHOR_WIDTH) + .ok_or(RetentionRootDecodeError::LengthOverflow)?, + observed: encoded.len(), + }) + } +} diff --git a/src/adapters/retention/root_decode_error.rs b/src/adapters/retention/root_decode_error.rs new file mode 100644 index 0000000..4965385 --- /dev/null +++ b/src/adapters/retention/root_decode_error.rs @@ -0,0 +1,150 @@ +//! This boundary module owns typed retention root decoding failures. + +use std::collections::TryReserveError; + +use crate::{ + BlobIdBinaryParseError, LayoutIdBinaryParseError, RetentionClosureLimitError, + RetentionNamespaceError, RetentionProfileAdmissionError, RetentionRootError, + RootGenerationError, +}; + +/// Failure to decode and admit one version-2 retention root. +#[derive(Debug)] +pub enum RetentionRootDecodeError { + /// The byte string ended before its required exact length. + Truncated { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// Bytes followed the required exact record. + TrailingData { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed header width was not canonical. + InvalidHeaderLength { + /// Required header width. + expected: u16, + /// Observed width. + observed: u16, + }, + /// The record carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The declared total length disagreed with canonical field arithmetic. + DeclaredLengthMismatch { + /// Canonical computed length. + expected: u64, + /// Declared length. + observed: u64, + }, + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// The fixed anchor width was not canonical. + InvalidAnchorWidth { + /// Required anchor width. + expected: u16, + /// Observed anchor width. + observed: u16, + }, + /// A reserved field was nonzero. + NonZeroReserved { + /// Protocol field name. + field: &'static str, + }, + /// Root generation admission failed. + Generation { + /// Preserved generation failure. + source: RootGenerationError, + }, + /// Namespace admission failed. + Namespace { + /// Preserved namespace failure. + source: RetentionNamespaceError, + }, + /// The declared anchor count exceeded the fixed bound. + AnchorCountExceeded { + /// Fixed maximum count. + maximum: u32, + /// Observed count. + observed: u32, + }, + /// Realization-profile admission failed. + Profile { + /// Preserved profile failure. + source: RetentionProfileAdmissionError, + }, + /// Closure-limit admission failed. + ClosureLimit { + /// Preserved limit failure. + source: RetentionClosureLimitError, + }, + /// One anchor contained a malformed `BlobId`. + BlobId { + /// Zero-based anchor index. + index: u32, + /// Preserved coordinate failure. + source: BlobIdBinaryParseError, + }, + /// One anchor contained a malformed `LayoutId`. + LayoutId { + /// Zero-based anchor index. + index: u32, + /// Preserved coordinate failure. + source: LayoutIdBinaryParseError, + }, + /// Canonical anchor ordering was violated. + NonCanonicalAnchorOrder { + /// Zero-based index of the observed anchor. + index: u32, + }, + /// Anchor allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// The anchor-set digest did not match the exact body. + AnchorSetDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the header. + observed: [u8; 32], + }, + /// The root digest did not match the exact header and body. + RootDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the record. + observed: [u8; 32], + }, + /// The checksum did not match the complete digest-bearing prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// Final semantic root admission failed. + Semantic { + /// Preserved semantic failure. + source: RetentionRootError, + }, +} diff --git a/src/adapters/retention/root_decode_error_display.rs b/src/adapters/retention/root_decode_error_display.rs new file mode 100644 index 0000000..f95e69f --- /dev/null +++ b/src/adapters/retention/root_decode_error_display.rs @@ -0,0 +1,122 @@ +//! This boundary module owns retention root decode diagnostics and sources. + +use std::{error::Error, fmt}; + +use super::RetentionRootDecodeError; +impl fmt::Display for RetentionRootDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Truncated { expected, observed } => { + write!( + formatter, + "retention root has {observed} bytes; expected {expected}" + ) + } + Self::TrailingData { expected, observed } => write!( + formatter, + "retention root has trailing data: expected {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { observed } => { + write!(formatter, "invalid retention root magic {observed:02x?}") + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported retention root version {observed}; expected {expected}" + ), + Self::InvalidHeaderLength { expected, observed } => write!( + formatter, + "retention root header length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported retention root flags {observed:#010x}" + ) + } + Self::DeclaredLengthMismatch { expected, observed } => write!( + formatter, + "retention root declares {observed} bytes; canonical fields require {expected}" + ), + Self::LengthOverflow => formatter.write_str("retention root length overflow"), + Self::InvalidAnchorWidth { expected, observed } => write!( + formatter, + "retention root anchor width {observed}; expected {expected}" + ), + Self::NonZeroReserved { field } => { + write!( + formatter, + "retention root {field} reserved bytes are nonzero" + ) + } + Self::Generation { source } => write!(formatter, "invalid root generation: {source}"), + Self::Namespace { source } => write!(formatter, "invalid root namespace: {source}"), + Self::AnchorCountExceeded { maximum, observed } => write!( + formatter, + "retention root declares {observed} anchors; maximum is {maximum}" + ), + Self::Profile { source } => write!(formatter, "invalid root profile: {source}"), + Self::ClosureLimit { source } => { + write!(formatter, "invalid root closure limit: {source}") + } + Self::BlobId { index, source } => { + write!( + formatter, + "invalid BlobId in retention anchor {index}: {source}" + ) + } + Self::LayoutId { index, source } => { + write!( + formatter, + "invalid LayoutId in retention anchor {index}: {source}" + ) + } + Self::NonCanonicalAnchorOrder { index, .. } => write!( + formatter, + "retention anchor {index} is not greater than its predecessor" + ), + Self::Allocation { .. } => { + formatter.write_str("retention root anchor allocation failed") + } + Self::AnchorSetDigestMismatch { .. } => { + formatter.write_str("retention root anchor-set digest mismatch") + } + Self::RootDigestMismatch { .. } => { + formatter.write_str("retention root digest mismatch") + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("retention root checksum mismatch") + } + Self::Semantic { source } => write!(formatter, "invalid semantic root: {source}"), + } + } +} + +impl Error for RetentionRootDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Generation { source } => Some(source), + Self::Namespace { source } => Some(source), + Self::Profile { source } => Some(source), + Self::ClosureLimit { source } => Some(source), + Self::BlobId { source, .. } => Some(source), + Self::LayoutId { source, .. } => Some(source), + Self::Allocation { source } => Some(source), + Self::Semantic { source } => Some(source), + Self::Truncated { .. } + | Self::TrailingData { .. } + | Self::InvalidMagic { .. } + | Self::UnsupportedVersion { .. } + | Self::InvalidHeaderLength { .. } + | Self::UnsupportedFlags { .. } + | Self::DeclaredLengthMismatch { .. } + | Self::LengthOverflow + | Self::InvalidAnchorWidth { .. } + | Self::NonZeroReserved { .. } + | Self::AnchorCountExceeded { .. } + | Self::NonCanonicalAnchorOrder { .. } + | Self::AnchorSetDigestMismatch { .. } + | Self::RootDigestMismatch { .. } + | Self::ChecksumMismatch { .. } => None, + } + } +} diff --git a/src/adapters/retention/root_decoder.rs b/src/adapters/retention/root_decoder.rs new file mode 100644 index 0000000..cf4a7b1 --- /dev/null +++ b/src/adapters/retention/root_decoder.rs @@ -0,0 +1,50 @@ +//! This boundary module owns canonical retention root decoding order. + +use super::root_header_decoder::HEADER_LENGTH; +use super::{ + AdmittedRetentionRoot, RetentionRootDecodeError, root_anchor_decoder, root_header_decoder, + root_integrity, root_semantic_header, +}; +use crate::{RetentionNamespace, RetentionPolicy, RetentionRoot, RetentionRootDigest}; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, RetentionRootDecodeError> { + let header = root_header_decoder::decode(encoded)?; + let digest = root_integrity::verify(encoded, header.digest_offset, header.checksum_offset)?; + let namespace_end = HEADER_LENGTH + .checked_add(header.namespace_length) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let namespace_bytes = + encoded + .get(HEADER_LENGTH..namespace_end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: namespace_end, + observed: encoded.len(), + })?; + let anchor_bytes = encoded.get(namespace_end..header.digest_offset).ok_or( + RetentionRootDecodeError::Truncated { + expected: header.digest_offset, + observed: encoded.len(), + }, + )?; + root_integrity::verify_anchor_set(header.anchor_count, anchor_bytes, header.anchor_set_digest)?; + let admitted_header = root_semantic_header::admit(&header)?; + let namespace = RetentionNamespace::try_from(namespace_bytes) + .map_err(|source| RetentionRootDecodeError::Namespace { source })?; + let anchors = root_anchor_decoder::decode(anchor_bytes, header.anchor_count)?; + let policy = RetentionPolicy::new(admitted_header.profile, admitted_header.limits); + let root = RetentionRoot::new( + namespace, + admitted_header.generation, + policy, + admitted_header.predecessor, + anchors, + ) + .map_err(|source| RetentionRootDecodeError::Semantic { source })?; + Ok(AdmittedRetentionRoot::admitted( + encoded, + root, + RetentionRootDigest::from_hash(digest), + )) +} diff --git a/src/adapters/retention/root_field_decoder.rs b/src/adapters/retention/root_field_decoder.rs new file mode 100644 index 0000000..89b9f8c --- /dev/null +++ b/src/adapters/retention/root_field_decoder.rs @@ -0,0 +1,106 @@ +//! This boundary module owns fixed-width retention root field extraction. + +use std::cmp::Ordering; + +use super::RetentionRootDecodeError; + +pub(super) fn require_exact( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionRootDecodeError> { + match encoded.len().cmp(&expected) { + Ordering::Less => Err(RetentionRootDecodeError::Truncated { + expected, + observed: encoded.len(), + }), + Ordering::Equal => Ok(()), + Ordering::Greater => Err(RetentionRootDecodeError::TrailingData { + expected, + observed: encoded.len(), + }), + } +} + +pub(super) const fn require_minimum( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionRootDecodeError> { + if encoded.len() < expected { + Err(RetentionRootDecodeError::Truncated { + expected, + observed: encoded.len(), + }) + } else { + Ok(()) + } +} + +pub(super) fn require_zero( + encoded: &[u8], + offset: usize, + width: usize, + field: &'static str, +) -> Result<(), RetentionRootDecodeError> { + let end = offset + .checked_add(width) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + if bytes.iter().all(|byte| *byte == 0) { + Ok(()) + } else { + Err(RetentionRootDecodeError::NonZeroReserved { field }) + } +} + +pub(super) fn require_u16( + encoded: &[u8], + offset: usize, + expected: u16, + error: F, +) -> Result<(), RetentionRootDecodeError> +where + F: FnOnce(u16, u16) -> RetentionRootDecodeError, +{ + let observed = read_u16(encoded, offset)?; + if observed == expected { + Ok(()) + } else { + Err(error(expected, observed)) + } +} + +pub(super) fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionRootDecodeError> { + let end = offset + .checked_add(WIDTH) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/root_header_decoder.rs b/src/adapters/retention/root_header_decoder.rs new file mode 100644 index 0000000..0de5091 --- /dev/null +++ b/src/adapters/retention/root_header_decoder.rs @@ -0,0 +1,111 @@ +//! This boundary module owns retention root header framing admission. + +use super::RetentionRootDecodeError; +use super::root_field_decoder::{ + read_array, read_u16, read_u32, read_u64, require_exact, require_minimum, require_u16, + require_zero, +}; + +pub(super) const HEADER_LENGTH: usize = 192; +const ANCHOR_WIDTH: usize = 119; +const TRAILER_LENGTH: usize = 64; + +pub(super) struct DecodedRootHeader { + pub(super) generation: u64, + pub(super) namespace_length: usize, + pub(super) anchor_count: u32, + pub(super) profile_identity: u32, + pub(super) profile_version: u32, + pub(super) profile_digest: [u8; 32], + pub(super) closure_nodes: u64, + pub(super) closure_depth: u16, + pub(super) closure_encoded_bytes: u64, + pub(super) closure_physical_bytes: u64, + pub(super) predecessor: [u8; 32], + pub(super) anchor_set_digest: [u8; 32], + pub(super) digest_offset: usize, + pub(super) checksum_offset: usize, +} + +pub(super) fn decode(encoded: &[u8]) -> Result { + require_minimum(encoded, HEADER_LENGTH)?; + validate_fixed_fields(encoded)?; + let namespace_length = usize::from(read_u16(encoded, 40)?); + let anchor_count = read_u32(encoded, 44)?; + let total_length = canonical_length(namespace_length, anchor_count)?; + require_declared_length(encoded, total_length)?; + require_exact(encoded, total_length)?; + let checksum_offset = total_length + .checked_sub(32) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let digest_offset = checksum_offset + .checked_sub(32) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + Ok(DecodedRootHeader { + generation: read_u64(encoded, 32)?, + namespace_length, + anchor_count, + profile_identity: read_u32(encoded, 48)?, + profile_version: read_u32(encoded, 52)?, + profile_digest: read_array(encoded, 56)?, + closure_nodes: read_u64(encoded, 88)?, + closure_depth: read_u16(encoded, 96)?, + closure_encoded_bytes: read_u64(encoded, 100)?, + closure_physical_bytes: read_u64(encoded, 108)?, + predecessor: read_array(encoded, 116)?, + anchor_set_digest: read_array(encoded, 148)?, + digest_offset, + checksum_offset, + }) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), RetentionRootDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != *b"KEEP:RET:ROOT2\0\0" { + return Err(RetentionRootDecodeError::InvalidMagic { observed: magic }); + } + require_u16(encoded, 16, 2, |expected, observed| { + RetentionRootDecodeError::UnsupportedVersion { expected, observed } + })?; + require_u16(encoded, 18, 192, |expected, observed| { + RetentionRootDecodeError::InvalidHeaderLength { expected, observed } + })?; + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(RetentionRootDecodeError::UnsupportedFlags { observed: flags }); + } + require_u16(encoded, 42, 119, |expected, observed| { + RetentionRootDecodeError::InvalidAnchorWidth { expected, observed } + })?; + require_zero(encoded, 98, 2, "limit")?; + require_zero(encoded, 180, 12, "trailing header") +} + +fn canonical_length( + namespace_length: usize, + anchor_count: u32, +) -> Result { + let anchors = usize::try_from(anchor_count) + .map_err(|_| RetentionRootDecodeError::LengthOverflow)? + .checked_mul(ANCHOR_WIDTH) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + HEADER_LENGTH + .checked_add(namespace_length) + .and_then(|length| length.checked_add(anchors)) + .and_then(|length| length.checked_add(TRAILER_LENGTH)) + .ok_or(RetentionRootDecodeError::LengthOverflow) +} + +fn require_declared_length( + encoded: &[u8], + total_length: usize, +) -> Result<(), RetentionRootDecodeError> { + let observed = read_u64(encoded, 24)?; + let expected = + u64::try_from(total_length).map_err(|_| RetentionRootDecodeError::LengthOverflow)?; + if observed == expected { + Ok(()) + } else { + Err(RetentionRootDecodeError::DeclaredLengthMismatch { expected, observed }) + } +} diff --git a/src/adapters/retention/root_integrity.rs b/src/adapters/retention/root_integrity.rs new file mode 100644 index 0000000..71f842d --- /dev/null +++ b/src/adapters/retention/root_integrity.rs @@ -0,0 +1,82 @@ +//! This boundary module owns retention root digest and checksum verification. + +use super::RetentionRootDecodeError; + +pub(super) fn verify( + encoded: &[u8], + digest_offset: usize, + checksum_offset: usize, +) -> Result<[u8; 32], RetentionRootDecodeError> { + let observed_checksum = read_digest(encoded, checksum_offset)?; + let checksum_preimage = + encoded + .get(..checksum_offset) + .ok_or(RetentionRootDecodeError::Truncated { + expected: checksum_offset, + observed: encoded.len(), + })?; + let expected_checksum = hash(b"keep.retention-root-checksum/v2\0", checksum_preimage); + if observed_checksum != expected_checksum { + return Err(RetentionRootDecodeError::ChecksumMismatch { + expected: expected_checksum, + observed: observed_checksum, + }); + } + + let observed_digest = read_digest(encoded, digest_offset)?; + let digest_preimage = + encoded + .get(..digest_offset) + .ok_or(RetentionRootDecodeError::Truncated { + expected: digest_offset, + observed: encoded.len(), + })?; + let expected_digest = hash(b"keep.retention-root/v2\0", digest_preimage); + if observed_digest != expected_digest { + return Err(RetentionRootDecodeError::RootDigestMismatch { + expected: expected_digest, + observed: observed_digest, + }); + } + Ok(expected_digest) +} + +pub(super) fn verify_anchor_set( + anchor_count: u32, + anchors: &[u8], + observed: [u8; 32], +) -> Result<(), RetentionRootDecodeError> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-anchor-set/v2\0"); + hasher.update(&anchor_count.to_be_bytes()); + hasher.update(anchors); + let expected = *hasher.finalize().as_bytes(); + if observed == expected { + Ok(()) + } else { + Err(RetentionRootDecodeError::AnchorSetDigestMismatch { expected, observed }) + } +} + +fn read_digest(encoded: &[u8], offset: usize) -> Result<[u8; 32], RetentionRootDecodeError> { + let end = offset + .checked_add(32) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; 32]>::try_from(bytes).map_err(|_| RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/retention/root_semantic_header.rs b/src/adapters/retention/root_semantic_header.rs new file mode 100644 index 0000000..17e9d9d --- /dev/null +++ b/src/adapters/retention/root_semantic_header.rs @@ -0,0 +1,55 @@ +//! This boundary module owns post-integrity retention header admission. + +use super::RetentionRootDecodeError; +use super::root_header_decoder::DecodedRootHeader; +use crate::{ + RegisteredRetentionProfile, RetentionClosureLimits, RetentionRoot, RetentionRootDigest, + RootGeneration, +}; + +pub(super) struct AdmittedRootHeader { + pub(super) generation: RootGeneration, + pub(super) profile: RegisteredRetentionProfile, + pub(super) limits: RetentionClosureLimits, + pub(super) predecessor: Option, +} + +pub(super) fn admit( + header: &DecodedRootHeader, +) -> Result { + if header.anchor_count > RetentionRoot::MAXIMUM_ANCHOR_COUNT { + return Err(RetentionRootDecodeError::AnchorCountExceeded { + maximum: RetentionRoot::MAXIMUM_ANCHOR_COUNT, + observed: header.anchor_count, + }); + } + let generation = RootGeneration::new(header.generation) + .map_err(|source| RetentionRootDecodeError::Generation { source })?; + let profile = RegisteredRetentionProfile::admit( + header.profile_identity, + header.profile_version, + header.profile_digest, + ) + .map_err(|source| RetentionRootDecodeError::Profile { source })?; + let limits = RetentionClosureLimits::new( + header.closure_nodes, + header.closure_depth, + header.closure_encoded_bytes, + header.closure_physical_bytes, + ) + .map_err(|source| RetentionRootDecodeError::ClosureLimit { source })?; + Ok(AdmittedRootHeader { + generation, + profile, + limits, + predecessor: predecessor(header.predecessor), + }) +} + +fn predecessor(bytes: [u8; 32]) -> Option { + if bytes == [0_u8; 32] { + None + } else { + Some(RetentionRootDigest::from_hash(bytes)) + } +} diff --git a/src/lib.rs b/src/lib.rs index 72dc778..e248621 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,8 +22,9 @@ //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots -//! are validated; canonical root encoding is available. Retention decoding, -//! publication, recovery, and garbage collection remain intentionally absent. +//! are validated; canonical in-memory root encoding and decoding are available. +//! Retention publication, recovery, and garbage collection remain intentionally +//! absent. #[cfg(test)] extern crate self as keep; @@ -41,9 +42,9 @@ mod retention; #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; pub use adapters::{ - AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionRoot, AdmittedSegment, + AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, + CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, @@ -84,8 +85,8 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, diff --git a/tests/retention_root_decoding.rs b/tests/retention_root_decoding.rs new file mode 100644 index 0000000..9405541 --- /dev/null +++ b/tests/retention_root_decoding.rs @@ -0,0 +1,189 @@ +//! Public decoding and integrity laws for version-2 retention roots. + +mod support; + +use std::io; + +use keep::{AdmittedRetentionRoot, RetentionRootDecodeError}; + +const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const ANCHOR_SET_DIGEST_OFFSET: usize = 148; +const ANCHOR_BODY_OFFSET: usize = 195; +const ROOT_DIGEST_OFFSET: usize = 314; +const CHECKSUM_OFFSET: usize = 346; + +#[test] +fn frozen_root_decodes_to_one_complete_semantic_generation() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let admitted = AdmittedRetentionRoot::decode(&bytes)?; + assert_eq!(admitted.encoded(), bytes); + assert_eq!(admitted.root().namespace().as_bytes(), &[0x00, 0x2f, 0xff]); + assert_eq!(admitted.root().generation().get(), 1); + assert_eq!(admitted.root().anchor_count(), 1); + assert_eq!( + admitted.digest().as_bytes(), + bytes.get(314..346).ok_or_else(|| { + io::Error::other("frozen retention root lacks its embedded digest") + })? + ); + Ok(()) +} + +#[test] +fn root_framing_refuses_truncation_trailing_data_and_magic_substitution() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + AdmittedRetentionRoot::decode(&truncated), + Err(RetentionRootDecodeError::Truncated { + expected: 378, + observed: 377, + }) + )); + + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + AdmittedRetentionRoot::decode(&trailing), + Err(RetentionRootDecodeError::TrailingData { + expected: 378, + observed: 379, + }) + )); + + let mut wrong_magic = bytes; + let first = wrong_magic + .first_mut() + .ok_or_else(|| io::Error::other("frozen retention root is empty"))?; + *first ^= 1; + assert!(matches!( + AdmittedRetentionRoot::decode(&wrong_magic), + Err(RetentionRootDecodeError::InvalidMagic { .. }) + )); + Ok(()) +} + +#[test] +fn root_checksum_and_digest_have_distinct_integrity_refusals() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut checksum_corruption = bytes.clone(); + let last = checksum_corruption + .last_mut() + .ok_or_else(|| io::Error::other("frozen retention root is empty"))?; + *last ^= 1; + assert!(matches!( + AdmittedRetentionRoot::decode(&checksum_corruption), + Err(RetentionRootDecodeError::ChecksumMismatch { .. }) + )); + + let mut digest_corruption = bytes; + let digest_byte = digest_corruption + .get_mut(314) + .ok_or_else(|| io::Error::other("frozen retention root lacks digest bytes"))?; + *digest_byte ^= 1; + refresh_checksum(&mut digest_corruption)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&digest_corruption), + Err(RetentionRootDecodeError::RootDigestMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn semantic_fields_are_admitted_only_after_complete_integrity() +-> Result<(), Box> { + let mut bytes = fixture_bytes()?; + bytes + .get_mut(32..40) + .ok_or_else(|| io::Error::other("frozen retention root lacks generation bytes"))? + .fill(0); + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::ChecksumMismatch { .. }) + )); + + refresh_root_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::Generation { .. }) + )); + Ok(()) +} + +#[test] +fn anchor_set_integrity_precedes_nested_identity_admission() +-> Result<(), Box> { + let mut bytes = fixture_bytes()?; + let first_anchor_byte = bytes + .get_mut(ANCHOR_BODY_OFFSET) + .ok_or_else(|| io::Error::other("frozen retention root lacks its anchor body"))?; + *first_anchor_byte ^= 1; + refresh_root_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::AnchorSetDigestMismatch { .. }) + )); + + refresh_anchor_set_digest(&mut bytes)?; + refresh_root_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::BlobId { index: 0, .. }) + )); + Ok(()) +} + +fn fixture_bytes() -> Result, io::Error> { + let encoded = ONE_ANCHOR_ROOT + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention root fixture lacks final newline"))?; + support::decode_hex(encoded) +} + +fn refresh_anchor_set_digest(bytes: &mut [u8]) -> Result<(), io::Error> { + let anchors = bytes + .get(ANCHOR_BODY_OFFSET..ROOT_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention root lacks its anchor body"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-anchor-set/v2\0"); + hasher.update(&1_u32.to_be_bytes()); + hasher.update(anchors); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(ANCHOR_SET_DIGEST_OFFSET..ANCHOR_SET_DIGEST_OFFSET + 32) + .ok_or_else(|| io::Error::other("retention root lacks its anchor-set digest"))? + .copy_from_slice(&digest); + Ok(()) +} + +fn refresh_root_digest_and_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let preimage = bytes + .get(..ROOT_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention root lacks its root digest preimage"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-root/v2\0"); + hasher.update(preimage); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(ROOT_DIGEST_OFFSET..CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention root lacks its root digest"))? + .copy_from_slice(&digest); + refresh_checksum(bytes) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let checksum_offset = bytes + .len() + .checked_sub(32) + .ok_or_else(|| io::Error::other("retention root lacks a checksum"))?; + let (preimage, checksum_slot) = bytes.split_at_mut(checksum_offset); + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-root-checksum/v2\0"); + hasher.update(preimage); + checksum_slot.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} From fe684e2dd9fd776c2e0ae4c4fb5b2fd0ddc4d2ca Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:12:04 -0700 Subject: [PATCH 06/50] Fix: Align segment-store registry contract --- xtask/tests/layout_format_contract.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtask/tests/layout_format_contract.rs b/xtask/tests/layout_format_contract.rs index 6f1340c..71ccd52 100644 --- a/xtask/tests/layout_format_contract.rs +++ b/xtask/tests/layout_format_contract.rs @@ -91,8 +91,8 @@ fn format_registry_reports_flat_layout_as_implemented() { fn format_registry_reports_the_segment_store_implementation_boundary() { const EXPECTED_ROW: &str = "\ | [Durable Segment Store v1](segment-store-v1/README.md) | \ -`keep.segment-store/v1` | Specified in issue #14; segment I/O implemented in \ -issue #15; publication and recovery remain in issues #16–#17 | \ +`keep.segment-store/v1` | Implemented through initialization, publication, \ +restart, and recovery in issues #14–#17 | \ [Golden corpus](../../conformance/segment-store/v1/README.md) |"; assert!( From 99ed9977130ea86d66249047e2ef5030c3d3187b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:13:35 -0700 Subject: [PATCH 07/50] Fix: Refresh segment-store documentation law --- xtask/tests/segment_store_implementation_documentation.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xtask/tests/segment_store_implementation_documentation.rs b/xtask/tests/segment_store_implementation_documentation.rs index dcead01..cb616af 100644 --- a/xtask/tests/segment_store_implementation_documentation.rs +++ b/xtask/tests/segment_store_implementation_documentation.rs @@ -11,7 +11,10 @@ fn living_documentation_names_the_implemented_segment_boundary() { for (document, claim) in [ (ROOT_README, "`StagedSegment`"), (ROOT_README, "`AdmittedSegment`"), - (FORMAT_REGISTRY, "segment I/O implemented in issue #15"), + ( + FORMAT_REGISTRY, + "Implemented through initialization, publication, restart, and recovery in issues #14–#17", + ), ( FORMAT_README, "Segment writing and verified reading are implemented in issue #15", From e5d7521f3a0de640b9d3a44d79e523fdefd12b2a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:15:20 -0700 Subject: [PATCH 08/50] Add: Admit canonical retention manifests --- CHANGELOG.md | 9 +- README.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 6 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 16 ++ src/adapters/retention/admitted_manifest.rs | 58 +++++++ src/adapters/retention/canonical_manifest.rs | 44 ++++++ .../retention/manifest_decode_error.rs | 124 +++++++++++++++ .../manifest_decode_error_display.rs | 109 +++++++++++++ src/adapters/retention/manifest_decoder.rs | 35 ++++ .../retention/manifest_encode_error.rs | 44 ++++++ src/adapters/retention/manifest_encoder.rs | 137 ++++++++++++++++ .../retention/manifest_entry_decoder.rs | 69 ++++++++ .../retention/manifest_field_decoder.rs | 106 +++++++++++++ .../retention/manifest_header_decoder.rs | 91 +++++++++++ src/adapters/retention/manifest_integrity.rs | 82 ++++++++++ .../retention/manifest_semantic_header.rs | 35 ++++ src/lib.rs | 35 ++-- src/retention/manifest.rs | 108 +++++++++++++ src/retention/manifest_digest.rs | 18 +++ src/retention/manifest_entry.rs | 42 +++++ src/retention/manifest_error.rs | 60 +++++++ src/retention/mod.rs | 8 + src/retention/namespace_digest.rs | 2 +- tests/retention_manifest_codec.rs | 92 +++++++++++ .../retention_manifest_codec/refusal_laws.rs | 149 ++++++++++++++++++ 27 files changed, 1464 insertions(+), 29 deletions(-) create mode 100644 src/adapters/retention/admitted_manifest.rs create mode 100644 src/adapters/retention/canonical_manifest.rs create mode 100644 src/adapters/retention/manifest_decode_error.rs create mode 100644 src/adapters/retention/manifest_decode_error_display.rs create mode 100644 src/adapters/retention/manifest_decoder.rs create mode 100644 src/adapters/retention/manifest_encode_error.rs create mode 100644 src/adapters/retention/manifest_encoder.rs create mode 100644 src/adapters/retention/manifest_entry_decoder.rs create mode 100644 src/adapters/retention/manifest_field_decoder.rs create mode 100644 src/adapters/retention/manifest_header_decoder.rs create mode 100644 src/adapters/retention/manifest_integrity.rs create mode 100644 src/adapters/retention/manifest_semantic_header.rs create mode 100644 src/retention/manifest.rs create mode 100644 src/retention/manifest_digest.rs create mode 100644 src/retention/manifest_entry.rs create mode 100644 src/retention/manifest_error.rs create mode 100644 tests/retention_manifest_codec.rs create mode 100644 tests/retention_manifest_codec/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d9ecd3e..233537f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -312,9 +312,12 @@ after its public API and format compatibility policies are established. encoder reproduces the independent version-2 golden bytes, and the decoder verifies framing, checksum, root digest, anchor-set digest, nested identities, resource bounds, canonical anchor order, and semantic invariants before - admission. Version-1 immutable bytes remain authoritative; production - version-2 writing remains unavailable until issue #19's executable evidence - is complete. + admission. Validated global manifest values and their canonical encoder and + decoder now reproduce the independent manifest fixture and enforce liveness + history, namespace uniqueness, bounds, ordering, and all three integrity + layers. Version-1 immutable bytes remain authoritative; production version-2 + writing remains unavailable until issue #19's executable evidence is + complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 6cc0d33..6198612 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,10 @@ recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values and canonical in-memory root encoding -and decoding are implemented. Retention publication, recovery, compaction, and -garbage collection remain planned. Presence in the reference CAS does not -claim retention, crash recovery, or durability. +and decoding are implemented, as are the global manifest values and codec. +Retention-head codecs, publication, recovery, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim +retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index aeebfad..89cd612 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -10,7 +10,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder and decoder evidence in `tests/retention_root_encoding.rs` and `tests/retention_root_decoding.rs`; manifest/head codecs remain | In progress in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root and manifest evidence in `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, and `tests/retention_manifest_codec.rs`; head codec remains | In progress in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 3f57f19..46d6cbc 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root encoding and decoding with complete -integrity verification before semantic admission. Filesystem publication, -manifest/head codecs, transitions, recovery, and garbage collection remain absent. +implements validated in-memory root and manifest codecs with integrity before +semantic admission. Filesystem publication, the head codec, transitions, +recovery, and garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 111841a..7033c63 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -450,8 +450,9 @@ pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutc #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; pub use retention::{ - AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionRootDecodeError, - RetentionRootEncodeError, + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionManifest, + CanonicalRetentionRoot, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, }; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 5fa3118..3cebdd7 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -1,7 +1,19 @@ //! This module owns canonical retention record boundary adapters. +mod admitted_manifest; mod admitted_root; +mod canonical_manifest; mod canonical_root; +mod manifest_decode_error; +mod manifest_decode_error_display; +mod manifest_decoder; +mod manifest_encode_error; +mod manifest_encoder; +mod manifest_entry_decoder; +mod manifest_field_decoder; +mod manifest_header_decoder; +mod manifest_integrity; +mod manifest_semantic_header; mod root_anchor_decoder; mod root_decode_error; mod root_decode_error_display; @@ -13,7 +25,11 @@ mod root_header_decoder; mod root_integrity; mod root_semantic_header; +pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; +pub use canonical_manifest::CanonicalRetentionManifest; pub use canonical_root::CanonicalRetentionRoot; +pub use manifest_decode_error::RetentionManifestDecodeError; +pub use manifest_encode_error::RetentionManifestEncodeError; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/admitted_manifest.rs b/src/adapters/retention/admitted_manifest.rs new file mode 100644 index 0000000..b06aa2d --- /dev/null +++ b/src/adapters/retention/admitted_manifest.rs @@ -0,0 +1,58 @@ +//! This boundary module owns one decoded and admitted retention manifest. + +use super::{RetentionManifestDecodeError, manifest_decoder}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +/// Borrowed canonical bytes paired with their admitted semantic manifest. +/// +/// Decoding verifies exact framing, the complete-record checksum, manifest and +/// entry-set digests, ordered entries, resource bounds, and generation-history +/// invariants. Entry allocation is bounded by a verified count. Decoding +/// performs no I/O. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct AdmittedRetentionManifest<'encoded> { + encoded: &'encoded [u8], + manifest: RetentionManifest, + digest: RetentionManifestDigest, +} + +impl<'encoded> AdmittedRetentionManifest<'encoded> { + /// Decodes and admits one exact canonical version-2 manifest record. + /// + /// # Errors + /// + /// Returns [`RetentionManifestDecodeError`] at the first violated framing, + /// integrity, resource-bound, ordering, or semantic invariant. + pub fn decode(encoded: &'encoded [u8]) -> Result { + manifest_decoder::decode(encoded) + } + + /// Returns the complete verified canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the admitted semantic manifest. + pub const fn manifest(&self) -> &RetentionManifest { + &self.manifest + } + + /// Returns the verified canonical manifest digest. + pub const fn digest(&self) -> RetentionManifestDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + manifest: RetentionManifest, + digest: RetentionManifestDigest, + ) -> Self { + Self { + encoded, + manifest, + digest, + } + } +} diff --git a/src/adapters/retention/canonical_manifest.rs b/src/adapters/retention/canonical_manifest.rs new file mode 100644 index 0000000..8c08a28 --- /dev/null +++ b/src/adapters/retention/canonical_manifest.rs @@ -0,0 +1,44 @@ +//! This boundary module owns materialized canonical retention manifest bytes. + +use super::{RetentionManifestEncodeError, manifest_encoder}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +/// Owned canonical version-2 retention manifest record. +/// +/// The complete record is materialized in memory after semantic bounds are +/// admitted and exact checked length calculation succeeds. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct CanonicalRetentionManifest { + encoded: Vec, + digest: RetentionManifestDigest, +} + +impl CanonicalRetentionManifest { + /// Encodes one validated semantic retention manifest. + /// + /// # Errors + /// + /// Returns [`RetentionManifestEncodeError`] for checked length overflow, + /// allocation refusal, or an internal construction-length mismatch. + pub fn from_manifest( + manifest: &RetentionManifest, + ) -> Result { + manifest_encoder::encode(manifest) + } + + /// Returns the complete canonical manifest bytes. + #[must_use] + pub fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the canonical manifest digest embedded in the record. + pub const fn digest(&self) -> RetentionManifestDigest { + self.digest + } + + pub(super) const fn admitted(encoded: Vec, digest: RetentionManifestDigest) -> Self { + Self { encoded, digest } + } +} diff --git a/src/adapters/retention/manifest_decode_error.rs b/src/adapters/retention/manifest_decode_error.rs new file mode 100644 index 0000000..4f00dc8 --- /dev/null +++ b/src/adapters/retention/manifest_decode_error.rs @@ -0,0 +1,124 @@ +//! This boundary module owns typed retention manifest decoding failures. + +use std::collections::TryReserveError; + +use crate::{LivenessGenerationError, RetentionManifestError, RootGenerationError}; + +/// Failure to decode and admit one version-2 retention manifest. +#[derive(Debug)] +pub enum RetentionManifestDecodeError { + /// The byte string ended before its required exact length. + Truncated { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// Bytes followed the required exact record. + TrailingData { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed header width was not canonical. + InvalidHeaderLength { + /// Required header width. + expected: u16, + /// Observed width. + observed: u16, + }, + /// The record carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The declared total length disagreed with canonical field arithmetic. + DeclaredLengthMismatch { + /// Canonical computed length. + expected: u64, + /// Declared length. + observed: u64, + }, + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// The fixed entry width was not canonical. + InvalidEntryWidth { + /// Required entry width. + expected: u16, + /// Observed entry width. + observed: u16, + }, + /// A reserved field was nonzero. + NonZeroReserved { + /// Protocol field name. + field: &'static str, + }, + /// Liveness-generation admission failed. + LivenessGeneration { + /// Preserved generation failure. + source: LivenessGenerationError, + }, + /// The declared entry count exceeded the fixed bound. + EntryCountExceeded { + /// Fixed maximum count. + maximum: u32, + /// Observed count. + observed: u32, + }, + /// One entry contained an invalid root generation. + RootGeneration { + /// Zero-based entry index. + index: u32, + /// Preserved generation failure. + source: RootGenerationError, + }, + /// Canonical namespace ordering was violated. + NonCanonicalEntryOrder { + /// Zero-based index of the observed entry. + index: u32, + }, + /// Entry allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// The entry-set digest did not match the exact body. + EntrySetDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the header. + observed: [u8; 32], + }, + /// The manifest digest did not match the exact header and body. + ManifestDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the record. + observed: [u8; 32], + }, + /// The checksum did not match the complete digest-bearing prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// Final semantic manifest admission failed. + Semantic { + /// Preserved semantic failure. + source: RetentionManifestError, + }, +} diff --git a/src/adapters/retention/manifest_decode_error_display.rs b/src/adapters/retention/manifest_decode_error_display.rs new file mode 100644 index 0000000..76a81dd --- /dev/null +++ b/src/adapters/retention/manifest_decode_error_display.rs @@ -0,0 +1,109 @@ +//! This boundary module owns retention manifest decode diagnostics and sources. + +use std::{error::Error, fmt}; + +use super::RetentionManifestDecodeError; + +impl fmt::Display for RetentionManifestDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Truncated { expected, observed } => write!( + formatter, + "retention manifest has {observed} bytes; expected {expected}" + ), + Self::TrailingData { expected, observed } => write!( + formatter, + "retention manifest has trailing data: expected {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { observed } => { + write!( + formatter, + "invalid retention manifest magic {observed:02x?}" + ) + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported retention manifest version {observed}; expected {expected}" + ), + Self::InvalidHeaderLength { expected, observed } => write!( + formatter, + "retention manifest header length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported retention manifest flags {observed:#010x}" + ) + } + Self::DeclaredLengthMismatch { expected, observed } => write!( + formatter, + "retention manifest declares {observed} bytes; canonical fields require {expected}" + ), + Self::LengthOverflow => formatter.write_str("retention manifest length overflow"), + Self::InvalidEntryWidth { expected, observed } => write!( + formatter, + "retention manifest entry width {observed}; expected {expected}" + ), + Self::NonZeroReserved { field } => write!( + formatter, + "retention manifest {field} reserved bytes are nonzero" + ), + Self::LivenessGeneration { source } => { + write!(formatter, "invalid liveness generation: {source}") + } + Self::EntryCountExceeded { maximum, observed } => write!( + formatter, + "retention manifest declares {observed} entries; maximum is {maximum}" + ), + Self::RootGeneration { index, source } => write!( + formatter, + "invalid root generation in retention entry {index}: {source}" + ), + Self::NonCanonicalEntryOrder { index } => write!( + formatter, + "retention manifest entry {index} is not greater than its predecessor" + ), + Self::Allocation { .. } => { + formatter.write_str("retention manifest entry allocation failed") + } + Self::EntrySetDigestMismatch { .. } => { + formatter.write_str("retention manifest entry-set digest mismatch") + } + Self::ManifestDigestMismatch { .. } => { + formatter.write_str("retention manifest digest mismatch") + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("retention manifest checksum mismatch") + } + Self::Semantic { source } => { + write!(formatter, "invalid semantic retention manifest: {source}") + } + } + } +} + +impl Error for RetentionManifestDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LivenessGeneration { source } => Some(source), + Self::RootGeneration { source, .. } => Some(source), + Self::Allocation { source } => Some(source), + Self::Semantic { source } => Some(source), + Self::Truncated { .. } + | Self::TrailingData { .. } + | Self::InvalidMagic { .. } + | Self::UnsupportedVersion { .. } + | Self::InvalidHeaderLength { .. } + | Self::UnsupportedFlags { .. } + | Self::DeclaredLengthMismatch { .. } + | Self::LengthOverflow + | Self::InvalidEntryWidth { .. } + | Self::NonZeroReserved { .. } + | Self::EntryCountExceeded { .. } + | Self::NonCanonicalEntryOrder { .. } + | Self::EntrySetDigestMismatch { .. } + | Self::ManifestDigestMismatch { .. } + | Self::ChecksumMismatch { .. } => None, + } + } +} diff --git a/src/adapters/retention/manifest_decoder.rs b/src/adapters/retention/manifest_decoder.rs new file mode 100644 index 0000000..6169c40 --- /dev/null +++ b/src/adapters/retention/manifest_decoder.rs @@ -0,0 +1,35 @@ +//! This boundary module owns canonical retention manifest decoding order. + +use super::manifest_header_decoder::HEADER_LENGTH; +use super::{ + AdmittedRetentionManifest, RetentionManifestDecodeError, manifest_entry_decoder, + manifest_header_decoder, manifest_integrity, manifest_semantic_header, +}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, RetentionManifestDecodeError> { + let header = manifest_header_decoder::decode(encoded)?; + let digest = manifest_integrity::verify(encoded, header.digest_offset, header.checksum_offset)?; + let entry_bytes = encoded.get(HEADER_LENGTH..header.digest_offset).ok_or( + RetentionManifestDecodeError::Truncated { + expected: header.digest_offset, + observed: encoded.len(), + }, + )?; + manifest_integrity::verify_entry_set(header.entry_count, entry_bytes, header.entry_set_digest)?; + let admitted_header = manifest_semantic_header::admit(&header)?; + let entries = manifest_entry_decoder::decode(entry_bytes, header.entry_count)?; + let manifest = RetentionManifest::new( + admitted_header.generation, + admitted_header.predecessor, + entries, + ) + .map_err(|source| RetentionManifestDecodeError::Semantic { source })?; + Ok(AdmittedRetentionManifest::admitted( + encoded, + manifest, + RetentionManifestDigest::from_hash(digest), + )) +} diff --git a/src/adapters/retention/manifest_encode_error.rs b/src/adapters/retention/manifest_encode_error.rs new file mode 100644 index 0000000..90a4385 --- /dev/null +++ b/src/adapters/retention/manifest_encode_error.rs @@ -0,0 +1,44 @@ +//! This boundary module owns retention manifest encoding failures. + +use std::{collections::TryReserveError, error::Error, fmt}; + +/// Failure to encode one canonical retention manifest record. +#[derive(Debug)] +pub enum RetentionManifestEncodeError { + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// Canonical byte allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// Internal construction produced a noncanonical length. + ConstructionLength { + /// Required length. + expected: usize, + /// Constructed length. + observed: usize, + }, +} + +impl fmt::Display for RetentionManifestEncodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LengthOverflow => formatter.write_str("retention manifest length overflow"), + Self::Allocation { .. } => formatter.write_str("retention manifest allocation failed"), + Self::ConstructionLength { expected, observed } => write!( + formatter, + "retention manifest construction produced {observed} bytes; expected {expected}" + ), + } + } +} + +impl Error for RetentionManifestEncodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Allocation { source } => Some(source), + Self::LengthOverflow | Self::ConstructionLength { .. } => None, + } + } +} diff --git a/src/adapters/retention/manifest_encoder.rs b/src/adapters/retention/manifest_encoder.rs new file mode 100644 index 0000000..1c53399 --- /dev/null +++ b/src/adapters/retention/manifest_encoder.rs @@ -0,0 +1,137 @@ +//! This boundary module owns canonical version-2 retention manifest encoding. + +use super::{CanonicalRetentionManifest, RetentionManifestEncodeError}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +const HEADER_LENGTH: usize = 160; +const ENTRY_WIDTH: usize = 72; +const TRAILER_LENGTH: usize = 64; + +struct EncodingPlan { + total_length: usize, + digest_preimage_length: usize, + entry_set_digest: [u8; 32], +} + +pub(super) fn encode( + manifest: &RetentionManifest, +) -> Result { + let plan = plan(manifest)?; + let mut encoded = Vec::new(); + encoded + .try_reserve_exact(plan.total_length) + .map_err(|source| RetentionManifestEncodeError::Allocation { source })?; + write_header(&mut encoded, manifest, &plan)?; + write_entries(&mut encoded, manifest); + require_length(&encoded, plan.digest_preimage_length)?; + let digest = hash(b"keep.retention-manifest/v2\0", &encoded); + encoded.extend_from_slice(&digest); + let checksum = hash(b"keep.retention-manifest-checksum/v2\0", &encoded); + encoded.extend_from_slice(&checksum); + require_length(&encoded, plan.total_length)?; + Ok(CanonicalRetentionManifest::admitted( + encoded, + RetentionManifestDigest::from_hash(digest), + )) +} + +fn plan(manifest: &RetentionManifest) -> Result { + let entry_bytes = usize::try_from(manifest.entry_count()) + .map_err(|_| RetentionManifestEncodeError::LengthOverflow)? + .checked_mul(ENTRY_WIDTH) + .ok_or(RetentionManifestEncodeError::LengthOverflow)?; + let digest_preimage_length = HEADER_LENGTH + .checked_add(entry_bytes) + .ok_or(RetentionManifestEncodeError::LengthOverflow)?; + let total_length = digest_preimage_length + .checked_add(TRAILER_LENGTH) + .ok_or(RetentionManifestEncodeError::LengthOverflow)?; + Ok(EncodingPlan { + total_length, + digest_preimage_length, + entry_set_digest: entry_set_digest(manifest), + }) +} + +fn entry_set_digest(manifest: &RetentionManifest) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-entries/v2\0"); + hasher.update(&manifest.entry_count().to_be_bytes()); + for entry in manifest.entries() { + hasher.update(entry.namespace().as_bytes()); + hasher.update(&entry.root_generation().get().to_be_bytes()); + hasher.update(entry.root_digest().as_bytes()); + } + *hasher.finalize().as_bytes() +} + +fn write_header( + encoded: &mut Vec, + manifest: &RetentionManifest, + plan: &EncodingPlan, +) -> Result<(), RetentionManifestEncodeError> { + encoded.extend_from_slice(b"KEEP:RET:LIVE2\0\0"); + push_u16(encoded, 2); + push_u16(encoded, 160); + push_u32(encoded, 0); + push_u64( + encoded, + u64::try_from(plan.total_length) + .map_err(|_| RetentionManifestEncodeError::LengthOverflow)?, + ); + push_u64(encoded, manifest.generation().get()); + push_u16(encoded, 72); + push_u16(encoded, 0); + push_u32(encoded, manifest.entry_count()); + encoded.extend_from_slice(&predecessor_bytes(manifest)); + encoded.extend_from_slice(&plan.entry_set_digest); + encoded.extend_from_slice(&[0_u8; 48]); + require_length(encoded, HEADER_LENGTH) +} + +fn predecessor_bytes(manifest: &RetentionManifest) -> [u8; 32] { + manifest + .predecessor() + .map_or([0_u8; 32], |digest| *digest.as_bytes()) +} + +fn write_entries(encoded: &mut Vec, manifest: &RetentionManifest) { + for entry in manifest.entries() { + encoded.extend_from_slice(entry.namespace().as_bytes()); + push_u64(encoded, entry.root_generation().get()); + encoded.extend_from_slice(entry.root_digest().as_bytes()); + } +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} + +fn push_u16(encoded: &mut Vec, value: u16) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(encoded: &mut Vec, value: u32) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u64(encoded: &mut Vec, value: u64) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +const fn require_length( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionManifestEncodeError> { + if encoded.len() == expected { + Ok(()) + } else { + Err(RetentionManifestEncodeError::ConstructionLength { + expected, + observed: encoded.len(), + }) + } +} diff --git a/src/adapters/retention/manifest_entry_decoder.rs b/src/adapters/retention/manifest_entry_decoder.rs new file mode 100644 index 0000000..46d09fb --- /dev/null +++ b/src/adapters/retention/manifest_entry_decoder.rs @@ -0,0 +1,69 @@ +//! This boundary module owns canonical retention manifest entry decoding. + +use super::RetentionManifestDecodeError; +use super::manifest_field_decoder::require_exact; +use crate::{ + RetentionManifestEntry, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, +}; + +const ENTRY_WIDTH: usize = 72; + +pub(super) fn decode( + encoded: &[u8], + entry_count: u32, +) -> Result, RetentionManifestDecodeError> { + let capacity = + usize::try_from(entry_count).map_err(|_| RetentionManifestDecodeError::LengthOverflow)?; + let expected_length = capacity + .checked_mul(ENTRY_WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + require_exact(encoded, expected_length)?; + let mut entries = Vec::new(); + entries + .try_reserve_exact(capacity) + .map_err(|source| RetentionManifestDecodeError::Allocation { source })?; + let mut previous = None; + for (position, bytes) in encoded.chunks_exact(ENTRY_WIDTH).enumerate() { + let index = + u32::try_from(position).map_err(|_| RetentionManifestDecodeError::LengthOverflow)?; + let namespace = RetentionNamespaceDigest::from_hash(read_array(bytes, 0)?); + let root_generation = RootGeneration::new(read_u64(bytes, 32)?) + .map_err(|source| RetentionManifestDecodeError::RootGeneration { index, source })?; + let root_digest = RetentionRootDigest::from_hash(read_array(bytes, 40)?); + if let Some(prior) = previous + && namespace <= prior + { + return Err(RetentionManifestDecodeError::NonCanonicalEntryOrder { index }); + } + entries.push(RetentionManifestEntry::new( + namespace, + root_generation, + root_digest, + )); + previous = Some(namespace); + } + Ok(entries) +} + +fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionManifestDecodeError> { + let end = offset + .checked_add(WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/manifest_field_decoder.rs b/src/adapters/retention/manifest_field_decoder.rs new file mode 100644 index 0000000..80b1ae4 --- /dev/null +++ b/src/adapters/retention/manifest_field_decoder.rs @@ -0,0 +1,106 @@ +//! This boundary module owns fixed-width retention manifest field extraction. + +use std::cmp::Ordering; + +use super::RetentionManifestDecodeError; + +pub(super) fn require_exact( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionManifestDecodeError> { + match encoded.len().cmp(&expected) { + Ordering::Less => Err(RetentionManifestDecodeError::Truncated { + expected, + observed: encoded.len(), + }), + Ordering::Equal => Ok(()), + Ordering::Greater => Err(RetentionManifestDecodeError::TrailingData { + expected, + observed: encoded.len(), + }), + } +} + +pub(super) const fn require_minimum( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionManifestDecodeError> { + if encoded.len() < expected { + Err(RetentionManifestDecodeError::Truncated { + expected, + observed: encoded.len(), + }) + } else { + Ok(()) + } +} + +pub(super) fn require_zero( + encoded: &[u8], + offset: usize, + width: usize, + field: &'static str, +) -> Result<(), RetentionManifestDecodeError> { + let end = offset + .checked_add(width) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + if bytes.iter().all(|byte| *byte == 0) { + Ok(()) + } else { + Err(RetentionManifestDecodeError::NonZeroReserved { field }) + } +} + +pub(super) fn require_u16( + encoded: &[u8], + offset: usize, + expected: u16, + error: F, +) -> Result<(), RetentionManifestDecodeError> +where + F: FnOnce(u16, u16) -> RetentionManifestDecodeError, +{ + let observed = read_u16(encoded, offset)?; + if observed == expected { + Ok(()) + } else { + Err(error(expected, observed)) + } +} + +pub(super) fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionManifestDecodeError> { + let end = offset + .checked_add(WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/manifest_header_decoder.rs b/src/adapters/retention/manifest_header_decoder.rs new file mode 100644 index 0000000..d577564 --- /dev/null +++ b/src/adapters/retention/manifest_header_decoder.rs @@ -0,0 +1,91 @@ +//! This boundary module owns retention manifest header framing admission. + +use super::RetentionManifestDecodeError; +use super::manifest_field_decoder::{ + read_array, read_u32, read_u64, require_exact, require_minimum, require_u16, require_zero, +}; + +pub(super) const HEADER_LENGTH: usize = 160; +const ENTRY_WIDTH: usize = 72; +const TRAILER_LENGTH: usize = 64; + +pub(super) struct DecodedManifestHeader { + pub(super) generation: u64, + pub(super) entry_count: u32, + pub(super) predecessor: [u8; 32], + pub(super) entry_set_digest: [u8; 32], + pub(super) digest_offset: usize, + pub(super) checksum_offset: usize, +} + +pub(super) fn decode( + encoded: &[u8], +) -> Result { + require_minimum(encoded, HEADER_LENGTH)?; + validate_fixed_fields(encoded)?; + let entry_count = read_u32(encoded, 44)?; + let total_length = canonical_length(entry_count)?; + require_declared_length(encoded, total_length)?; + require_exact(encoded, total_length)?; + let checksum_offset = total_length + .checked_sub(32) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let digest_offset = checksum_offset + .checked_sub(32) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + Ok(DecodedManifestHeader { + generation: read_u64(encoded, 32)?, + entry_count, + predecessor: read_array(encoded, 48)?, + entry_set_digest: read_array(encoded, 80)?, + digest_offset, + checksum_offset, + }) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), RetentionManifestDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != *b"KEEP:RET:LIVE2\0\0" { + return Err(RetentionManifestDecodeError::InvalidMagic { observed: magic }); + } + require_u16(encoded, 16, 2, |expected, observed| { + RetentionManifestDecodeError::UnsupportedVersion { expected, observed } + })?; + require_u16(encoded, 18, 160, |expected, observed| { + RetentionManifestDecodeError::InvalidHeaderLength { expected, observed } + })?; + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(RetentionManifestDecodeError::UnsupportedFlags { observed: flags }); + } + require_u16(encoded, 40, 72, |expected, observed| { + RetentionManifestDecodeError::InvalidEntryWidth { expected, observed } + })?; + require_zero(encoded, 42, 2, "entry")?; + require_zero(encoded, 112, 48, "trailing header") +} + +fn canonical_length(entry_count: u32) -> Result { + let entries = usize::try_from(entry_count) + .map_err(|_| RetentionManifestDecodeError::LengthOverflow)? + .checked_mul(ENTRY_WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + HEADER_LENGTH + .checked_add(entries) + .and_then(|length| length.checked_add(TRAILER_LENGTH)) + .ok_or(RetentionManifestDecodeError::LengthOverflow) +} + +fn require_declared_length( + encoded: &[u8], + total_length: usize, +) -> Result<(), RetentionManifestDecodeError> { + let observed = read_u64(encoded, 24)?; + let expected = + u64::try_from(total_length).map_err(|_| RetentionManifestDecodeError::LengthOverflow)?; + if observed == expected { + Ok(()) + } else { + Err(RetentionManifestDecodeError::DeclaredLengthMismatch { expected, observed }) + } +} diff --git a/src/adapters/retention/manifest_integrity.rs b/src/adapters/retention/manifest_integrity.rs new file mode 100644 index 0000000..9e4e741 --- /dev/null +++ b/src/adapters/retention/manifest_integrity.rs @@ -0,0 +1,82 @@ +//! This boundary module owns retention manifest integrity verification. + +use super::RetentionManifestDecodeError; + +pub(super) fn verify( + encoded: &[u8], + digest_offset: usize, + checksum_offset: usize, +) -> Result<[u8; 32], RetentionManifestDecodeError> { + let observed_checksum = read_digest(encoded, checksum_offset)?; + let checksum_preimage = + encoded + .get(..checksum_offset) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: checksum_offset, + observed: encoded.len(), + })?; + let expected_checksum = hash(b"keep.retention-manifest-checksum/v2\0", checksum_preimage); + if observed_checksum != expected_checksum { + return Err(RetentionManifestDecodeError::ChecksumMismatch { + expected: expected_checksum, + observed: observed_checksum, + }); + } + + let observed_digest = read_digest(encoded, digest_offset)?; + let digest_preimage = + encoded + .get(..digest_offset) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: digest_offset, + observed: encoded.len(), + })?; + let expected_digest = hash(b"keep.retention-manifest/v2\0", digest_preimage); + if observed_digest != expected_digest { + return Err(RetentionManifestDecodeError::ManifestDigestMismatch { + expected: expected_digest, + observed: observed_digest, + }); + } + Ok(expected_digest) +} + +pub(super) fn verify_entry_set( + entry_count: u32, + entries: &[u8], + observed: [u8; 32], +) -> Result<(), RetentionManifestDecodeError> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-entries/v2\0"); + hasher.update(&entry_count.to_be_bytes()); + hasher.update(entries); + let expected = *hasher.finalize().as_bytes(); + if observed == expected { + Ok(()) + } else { + Err(RetentionManifestDecodeError::EntrySetDigestMismatch { expected, observed }) + } +} + +fn read_digest(encoded: &[u8], offset: usize) -> Result<[u8; 32], RetentionManifestDecodeError> { + let end = offset + .checked_add(32) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; 32]>::try_from(bytes).map_err(|_| RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/retention/manifest_semantic_header.rs b/src/adapters/retention/manifest_semantic_header.rs new file mode 100644 index 0000000..8135a08 --- /dev/null +++ b/src/adapters/retention/manifest_semantic_header.rs @@ -0,0 +1,35 @@ +//! This boundary module owns post-integrity retention manifest header admission. + +use super::RetentionManifestDecodeError; +use super::manifest_header_decoder::DecodedManifestHeader; +use crate::{LivenessGeneration, RetentionManifest, RetentionManifestDigest}; + +pub(super) struct AdmittedManifestHeader { + pub(super) generation: LivenessGeneration, + pub(super) predecessor: Option, +} + +pub(super) fn admit( + header: &DecodedManifestHeader, +) -> Result { + if header.entry_count > RetentionManifest::MAXIMUM_ENTRY_COUNT { + return Err(RetentionManifestDecodeError::EntryCountExceeded { + maximum: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed: header.entry_count, + }); + } + let generation = LivenessGeneration::new(header.generation) + .map_err(|source| RetentionManifestDecodeError::LivenessGeneration { source })?; + Ok(AdmittedManifestHeader { + generation, + predecessor: predecessor(header.predecessor), + }) +} + +fn predecessor(bytes: [u8; 32]) -> Option { + if bytes == [0_u8; 32] { + None + } else { + Some(RetentionManifestDigest::from_hash(bytes)) + } +} diff --git a/src/lib.rs b/src/lib.rs index e248621..8e07d58 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,9 +22,9 @@ //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots -//! are validated; canonical in-memory root encoding and decoding are available. -//! Retention publication, recovery, and garbage collection remain intentionally -//! absent. +//! are validated; canonical in-memory root and manifest encoding and decoding +//! are available. Retention-head codecs, publication, recovery, and garbage +//! collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -42,16 +42,17 @@ mod retention; #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; pub use adapters::{ - AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionRoot, AdmittedSegment, - AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, - CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, - CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, - CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionManifest, AdmittedRetentionRoot, + AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, + CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionManifest, + CanonicalRetentionRoot, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, @@ -85,8 +86,9 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, + RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, @@ -125,7 +127,8 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionNamespace, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionManifest, + RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, RootGeneration, RootGenerationError, diff --git a/src/retention/manifest.rs b/src/retention/manifest.rs new file mode 100644 index 0000000..5095cdb --- /dev/null +++ b/src/retention/manifest.rs @@ -0,0 +1,108 @@ +//! This module owns one canonical semantic global retention manifest. + +use super::{ + LivenessGeneration, RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, +}; + +/// Complete namespace-to-root view at one global liveness generation. +/// +/// Entries are stored in strict namespace-digest order. Construction sorts +/// caller input, refuses duplicate namespaces, and allocates no additional +/// buffer beyond the supplied `Vec`. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetentionManifest { + generation: LivenessGeneration, + predecessor: Option, + entries: Vec, + entry_count: u32, +} + +impl RetentionManifest { + /// Maximum admitted namespace entries. + pub const MAXIMUM_ENTRY_COUNT: u32 = 4_096; + + /// Admits one complete semantic manifest. + /// + /// # Errors + /// + /// Returns a typed generation-history, entry-count, or duplicate-namespace + /// failure before the value is admitted. + pub fn new( + generation: LivenessGeneration, + predecessor: Option, + mut entries: Vec, + ) -> Result { + admit_predecessor(generation, predecessor)?; + let observed = entries.len(); + let entry_count = + u32::try_from(observed).map_err(|_| RetentionManifestError::EntryCountExceeded { + maximum: Self::MAXIMUM_ENTRY_COUNT, + observed, + })?; + if entry_count > Self::MAXIMUM_ENTRY_COUNT { + return Err(RetentionManifestError::EntryCountExceeded { + maximum: Self::MAXIMUM_ENTRY_COUNT, + observed, + }); + } + entries.sort_unstable_by_key(|entry| entry.namespace()); + refuse_duplicate(&entries)?; + Ok(Self { + generation, + predecessor, + entries, + entry_count, + }) + } + + /// Returns the exact global liveness generation. + pub const fn generation(&self) -> LivenessGeneration { + self.generation + } + + /// Returns the preceding manifest digest, if this is a successor. + pub const fn predecessor(&self) -> Option { + self.predecessor + } + + /// Returns entries in strict namespace-digest order. + pub fn entries(&self) -> &[RetentionManifestEntry] { + &self.entries + } + + /// Returns the bounded entry count. + pub const fn entry_count(&self) -> u32 { + self.entry_count + } +} + +fn admit_predecessor( + generation: LivenessGeneration, + predecessor: Option, +) -> Result<(), RetentionManifestError> { + if generation.get() == 1 { + return predecessor.map_or(Ok(()), |observed| { + Err(RetentionManifestError::InitialGenerationHasPredecessor { observed }) + }); + } + if predecessor.is_some() { + Ok(()) + } else { + Err(RetentionManifestError::MissingPredecessor { generation }) + } +} + +fn refuse_duplicate(entries: &[RetentionManifestEntry]) -> Result<(), RetentionManifestError> { + for pair in entries.windows(2) { + let [previous, observed] = pair else { + continue; + }; + if previous.namespace() == observed.namespace() { + return Err(RetentionManifestError::DuplicateNamespace { + namespace: previous.namespace(), + }); + } + } + Ok(()) +} diff --git a/src/retention/manifest_digest.rs b/src/retention/manifest_digest.rs new file mode 100644 index 0000000..bfcc292 --- /dev/null +++ b/src/retention/manifest_digest.rs @@ -0,0 +1,18 @@ +//! This module owns canonical global retention manifest identity. + +/// Canonical BLAKE3-256 identity of one complete retention manifest record. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionManifestDigest([u8; 32]); + +impl RetentionManifestDigest { + pub(crate) const fn from_hash(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the exact 32 digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/retention/manifest_entry.rs b/src/retention/manifest_entry.rs new file mode 100644 index 0000000..dcb0941 --- /dev/null +++ b/src/retention/manifest_entry.rs @@ -0,0 +1,42 @@ +//! This module owns one semantic retention manifest entry. + +use super::{RetentionNamespaceDigest, RetentionRootDigest, RootGeneration}; + +/// Exact current root coordinate for one retention namespace. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionManifestEntry { + namespace: RetentionNamespaceDigest, + root_generation: RootGeneration, + root_digest: RetentionRootDigest, +} + +impl RetentionManifestEntry { + /// Combines already-validated namespace and root coordinates. + pub const fn new( + namespace: RetentionNamespaceDigest, + root_generation: RootGeneration, + root_digest: RetentionRootDigest, + ) -> Self { + Self { + namespace, + root_generation, + root_digest, + } + } + + /// Returns the namespace digest selected by this entry. + pub const fn namespace(self) -> RetentionNamespaceDigest { + self.namespace + } + + /// Returns the exact current namespace root generation. + pub const fn root_generation(self) -> RootGeneration { + self.root_generation + } + + /// Returns the exact current namespace root digest. + pub const fn root_digest(self) -> RetentionRootDigest { + self.root_digest + } +} diff --git a/src/retention/manifest_error.rs b/src/retention/manifest_error.rs new file mode 100644 index 0000000..1f4ba84 --- /dev/null +++ b/src/retention/manifest_error.rs @@ -0,0 +1,60 @@ +//! This module owns typed semantic retention manifest failures. + +use std::{error::Error, fmt}; + +use super::{LivenessGeneration, RetentionManifestDigest, RetentionNamespaceDigest}; + +/// Failure to construct one canonical semantic retention manifest. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionManifestError { + /// Generation one carried an impossible predecessor. + InitialGenerationHasPredecessor { + /// Observed predecessor digest. + observed: RetentionManifestDigest, + }, + /// A successor generation omitted its required predecessor. + MissingPredecessor { + /// Successor generation lacking a predecessor. + generation: LivenessGeneration, + }, + /// The caller supplied too many namespace entries. + EntryCountExceeded { + /// Fixed maximum entry count. + maximum: u32, + /// Observed entry count. + observed: usize, + }, + /// The caller supplied one namespace more than once. + DuplicateNamespace { + /// Exact duplicated namespace digest. + namespace: RetentionNamespaceDigest, + }, +} + +impl fmt::Display for RetentionManifestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGenerationHasPredecessor { observed } => write!( + formatter, + "initial retention manifest has predecessor {:?}", + observed.as_bytes() + ), + Self::MissingPredecessor { generation } => write!( + formatter, + "retention manifest generation {} requires a predecessor", + generation.get() + ), + Self::EntryCountExceeded { maximum, observed } => write!( + formatter, + "retention manifest has {observed} entries; maximum is {maximum}" + ), + Self::DuplicateNamespace { namespace } => write!( + formatter, + "retention manifest repeats namespace {:?}", + namespace.as_bytes() + ), + } + } +} + +impl Error for RetentionManifestError {} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index a03e461..38ddba9 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -11,6 +11,10 @@ mod closure_limit_error; mod closure_limits; mod liveness_generation; mod liveness_generation_error; +mod manifest; +mod manifest_digest; +mod manifest_entry; +mod manifest_error; mod namespace; mod namespace_digest; mod namespace_error; @@ -29,6 +33,10 @@ pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; pub use liveness_generation::LivenessGeneration; pub use liveness_generation_error::LivenessGenerationError; +pub use manifest::RetentionManifest; +pub use manifest_digest::RetentionManifestDigest; +pub use manifest_entry::RetentionManifestEntry; +pub use manifest_error::RetentionManifestError; pub use namespace::RetentionNamespace; pub use namespace_digest::RetentionNamespaceDigest; pub use namespace_error::RetentionNamespaceError; diff --git a/src/retention/namespace_digest.rs b/src/retention/namespace_digest.rs index d68dda4..a695432 100644 --- a/src/retention/namespace_digest.rs +++ b/src/retention/namespace_digest.rs @@ -9,7 +9,7 @@ pub struct RetentionNamespaceDigest([u8; 32]); impl RetentionNamespaceDigest { - pub(super) const fn from_hash(bytes: [u8; 32]) -> Self { + pub(crate) const fn from_hash(bytes: [u8; 32]) -> Self { Self(bytes) } diff --git a/tests/retention_manifest_codec.rs b/tests/retention_manifest_codec.rs new file mode 100644 index 0000000..a17fb86 --- /dev/null +++ b/tests/retention_manifest_codec.rs @@ -0,0 +1,92 @@ +//! Public semantic and canonical-codec laws for retention manifests. + +#[path = "retention_manifest_codec/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::io; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionManifest, + LivenessGeneration, RetentionManifest, RetentionManifestEntry, RetentionManifestError, +}; + +pub(crate) const ONE_ANCHOR_ROOT: &str = + include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +pub(crate) const ONE_ROOT_MANIFEST: &str = + include_str!("../conformance/segment-store/v2/one-root-manifest.hex"); +pub(crate) const ENTRY_SET_DIGEST_OFFSET: usize = 80; +pub(crate) const ENTRY_BODY_OFFSET: usize = 160; +pub(crate) const MANIFEST_DIGEST_OFFSET: usize = 232; +pub(crate) const CHECKSUM_OFFSET: usize = 264; + +#[test] +fn one_root_manifest_has_one_semantic_and_canonical_representation() +-> Result<(), Box> { + let root_bytes = fixture_bytes(ONE_ANCHOR_ROOT)?; + let root = AdmittedRetentionRoot::decode(&root_bytes)?; + let entry = RetentionManifestEntry::new( + root.root().namespace().digest(), + root.root().generation(), + root.digest(), + ); + let generation = LivenessGeneration::new(1)?; + let manifest = RetentionManifest::new(generation, None, vec![entry])?; + let canonical = CanonicalRetentionManifest::from_manifest(&manifest)?; + let manifest_bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + assert_eq!(canonical.encoded(), manifest_bytes); + + let admitted = AdmittedRetentionManifest::decode(&manifest_bytes)?; + assert_eq!(admitted.encoded(), manifest_bytes); + assert_eq!(admitted.manifest(), &manifest); + assert_eq!(admitted.digest(), canonical.digest()); + assert_eq!( + admitted.digest().as_bytes(), + manifest_bytes + .get(MANIFEST_DIGEST_OFFSET..MANIFEST_DIGEST_OFFSET + 32) + .ok_or_else(|| io::Error::other("frozen manifest lacks its digest"))? + ); + Ok(()) +} + +#[test] +fn manifest_history_and_namespace_set_are_admitted_canonically() +-> Result<(), Box> { + let entry = fixture_entry()?; + let manifest_bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let predecessor = AdmittedRetentionManifest::decode(&manifest_bytes)?.digest(); + assert!(matches!( + RetentionManifest::new(LivenessGeneration::new(1)?, Some(predecessor), vec![entry]), + Err(RetentionManifestError::InitialGenerationHasPredecessor { .. }) + )); + assert!(matches!( + RetentionManifest::new(LivenessGeneration::new(2)?, None, vec![entry]), + Err(RetentionManifestError::MissingPredecessor { .. }) + )); + assert!(matches!( + RetentionManifest::new( + LivenessGeneration::new(2)?, + Some(predecessor), + vec![entry, entry], + ), + Err(RetentionManifestError::DuplicateNamespace { .. }) + )); + Ok(()) +} + +fn fixture_entry() -> Result> { + let root_bytes = fixture_bytes(ONE_ANCHOR_ROOT)?; + let root = AdmittedRetentionRoot::decode(&root_bytes)?; + Ok(RetentionManifestEntry::new( + root.root().namespace().digest(), + root.root().generation(), + root.digest(), + )) +} + +pub(crate) fn fixture_bytes(fixture: &str) -> Result, io::Error> { + let encoded = fixture + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention fixture lacks final newline"))?; + support::decode_hex(encoded) +} diff --git a/tests/retention_manifest_codec/refusal_laws.rs b/tests/retention_manifest_codec/refusal_laws.rs new file mode 100644 index 0000000..8fc11a2 --- /dev/null +++ b/tests/retention_manifest_codec/refusal_laws.rs @@ -0,0 +1,149 @@ +//! Framing, integrity, and semantic refusal laws for retention manifests. + +use std::io; + +use keep::{AdmittedRetentionManifest, RetentionManifestDecodeError}; + +use super::{ + CHECKSUM_OFFSET, ENTRY_BODY_OFFSET, ENTRY_SET_DIGEST_OFFSET, MANIFEST_DIGEST_OFFSET, + ONE_ROOT_MANIFEST, fixture_bytes, +}; + +#[test] +fn manifest_framing_and_integrity_have_exact_first_refusals() +-> Result<(), Box> { + let bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + AdmittedRetentionManifest::decode(&truncated), + Err(RetentionManifestDecodeError::Truncated { + expected: 296, + observed: 295, + }) + )); + + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + AdmittedRetentionManifest::decode(&trailing), + Err(RetentionManifestDecodeError::TrailingData { + expected: 296, + observed: 297, + }) + )); + + let mut checksum_corruption = bytes.clone(); + let last = checksum_corruption + .last_mut() + .ok_or_else(|| io::Error::other("frozen manifest is empty"))?; + *last ^= 1; + assert!(matches!( + AdmittedRetentionManifest::decode(&checksum_corruption), + Err(RetentionManifestDecodeError::ChecksumMismatch { .. }) + )); + + let mut digest_corruption = bytes; + let digest_byte = digest_corruption + .get_mut(MANIFEST_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("frozen manifest lacks its digest"))?; + *digest_byte ^= 1; + refresh_checksum(&mut digest_corruption)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&digest_corruption), + Err(RetentionManifestDecodeError::ManifestDigestMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn complete_integrity_precedes_manifest_semantics() -> Result<(), Box> { + let mut bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + bytes + .get_mut(32..40) + .ok_or_else(|| io::Error::other("frozen manifest lacks generation bytes"))? + .fill(0); + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::ChecksumMismatch { .. }) + )); + + refresh_manifest_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::LivenessGeneration { .. }) + )); + Ok(()) +} + +#[test] +fn entry_set_integrity_precedes_nested_root_generation_admission() +-> Result<(), Box> { + let mut bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let first_entry_byte = bytes + .get_mut(ENTRY_BODY_OFFSET) + .ok_or_else(|| io::Error::other("frozen manifest lacks its entry body"))?; + *first_entry_byte ^= 1; + refresh_manifest_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::EntrySetDigestMismatch { .. }) + )); + + bytes + .get_mut(ENTRY_BODY_OFFSET + 32..ENTRY_BODY_OFFSET + 40) + .ok_or_else(|| io::Error::other("frozen manifest lacks root generation bytes"))? + .fill(0); + refresh_entry_set_digest(&mut bytes)?; + refresh_manifest_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::RootGeneration { index: 0, .. }) + )); + Ok(()) +} + +fn refresh_entry_set_digest(bytes: &mut [u8]) -> Result<(), io::Error> { + let entries = bytes + .get(ENTRY_BODY_OFFSET..MANIFEST_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks its entry body"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-entries/v2\0"); + hasher.update(&1_u32.to_be_bytes()); + hasher.update(entries); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(ENTRY_SET_DIGEST_OFFSET..ENTRY_SET_DIGEST_OFFSET + 32) + .ok_or_else(|| io::Error::other("retention manifest lacks its entry-set digest"))? + .copy_from_slice(&digest); + Ok(()) +} + +fn refresh_manifest_digest_and_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let preimage = bytes + .get(..MANIFEST_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks its digest preimage"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest/v2\0"); + hasher.update(preimage); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(MANIFEST_DIGEST_OFFSET..CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks its digest"))? + .copy_from_slice(&digest); + refresh_checksum(bytes) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, trailer) = bytes + .split_at_mut_checked(CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks a checksum"))?; + let checksum_slot = trailer + .get_mut(..blake3::OUT_LEN) + .ok_or_else(|| io::Error::other("retention manifest checksum is truncated"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-checksum/v2\0"); + hasher.update(preimage); + checksum_slot.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} From 109466777065cade453549017c11a2ebefc7044a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:30:02 -0700 Subject: [PATCH 09/50] Add: Admit canonical retention heads --- CHANGELOG.md | 8 +- README.md | 9 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 6 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 9 + src/adapters/retention/canonical_head.rs | 28 +++ src/adapters/retention/checksummed_head.rs | 45 +++++ src/adapters/retention/head_decode_error.rs | 66 +++++++ .../retention/head_decode_error_display.rs | 68 +++++++ src/adapters/retention/head_decoder.rs | 134 +++++++++++++ src/adapters/retention/head_encoder.rs | 32 +++ src/lib.rs | 87 ++++---- src/retention/head.rs | 74 +++++++ src/retention/head_error.rs | 39 ++++ src/retention/manifest_length.rs | 55 ++++++ src/retention/manifest_length_error.rs | 45 +++++ src/retention/mod.rs | 8 + tests/retention_head_codec.rs | 187 ++++++++++++++++++ 19 files changed, 850 insertions(+), 57 deletions(-) create mode 100644 src/adapters/retention/canonical_head.rs create mode 100644 src/adapters/retention/checksummed_head.rs create mode 100644 src/adapters/retention/head_decode_error.rs create mode 100644 src/adapters/retention/head_decode_error_display.rs create mode 100644 src/adapters/retention/head_decoder.rs create mode 100644 src/adapters/retention/head_encoder.rs create mode 100644 src/retention/head.rs create mode 100644 src/retention/head_error.rs create mode 100644 src/retention/manifest_length.rs create mode 100644 src/retention/manifest_length_error.rs create mode 100644 tests/retention_head_codec.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 233537f..5acdfb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -315,9 +315,11 @@ after its public API and format compatibility policies are established. admission. Validated global manifest values and their canonical encoder and decoder now reproduce the independent manifest fixture and enforce liveness history, namespace uniqueness, bounds, ordering, and all three integrity - layers. Version-1 immutable bytes remain authoritative; production version-2 - writing remains unavailable until issue #19's executable evidence is - complete. + layers. Typed manifest lengths and semantic global heads now reproduce and + admit the exact 144-byte head fixture with fixed framing, checksum-first + semantic admission, and explicit generation-history laws. Version-1 + immutable bytes remain authoritative; production version-2 writing remains + unavailable until issue #19's executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 6198612..465fdf7 100644 --- a/README.md +++ b/README.md @@ -115,11 +115,10 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Version-2 retention values and canonical in-memory root encoding -and decoding are implemented, as are the global manifest values and codec. -Retention-head codecs, publication, recovery, compaction, and garbage -collection remain planned. Presence in the reference CAS does not claim -retention, crash recovery, or durability. +power loss. Version-2 retention values and canonical in-memory root, global +manifest, and retention-head codecs are implemented. Retention publication, +recovery, compaction, and garbage collection remain planned. Presence in the +reference CAS does not claim retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 89cd612..9defde7 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -10,7 +10,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root and manifest evidence in `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, and `tests/retention_manifest_codec.rs`; head codec remains | In progress in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 46d6cbc..1823e50 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root and manifest codecs with integrity before -semantic admission. Filesystem publication, the head codec, transitions, -recovery, and garbage collection remain absent. +implements validated in-memory root, manifest, and head codecs with integrity +before semantic admission. Filesystem publication, transitions, recovery, and +garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 7033c63..b49e065 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -450,8 +450,9 @@ pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutc #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; pub use retention::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionManifest, - CanonicalRetentionRoot, RetentionManifestDecodeError, RetentionManifestEncodeError, + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, }; pub use sealed_segment::SealedSegment; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 3cebdd7..ec3f2ff 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -2,8 +2,14 @@ mod admitted_manifest; mod admitted_root; +mod canonical_head; mod canonical_manifest; mod canonical_root; +mod checksummed_head; +mod head_decode_error; +mod head_decode_error_display; +mod head_decoder; +mod head_encoder; mod manifest_decode_error; mod manifest_decode_error_display; mod manifest_decoder; @@ -27,8 +33,11 @@ mod root_semantic_header; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; +pub use canonical_head::CanonicalRetentionHead; pub use canonical_manifest::CanonicalRetentionManifest; pub use canonical_root::CanonicalRetentionRoot; +pub use checksummed_head::ChecksummedRetentionHead; +pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use root_decode_error::RetentionRootDecodeError; diff --git a/src/adapters/retention/canonical_head.rs b/src/adapters/retention/canonical_head.rs new file mode 100644 index 0000000..0f70013 --- /dev/null +++ b/src/adapters/retention/canonical_head.rs @@ -0,0 +1,28 @@ +//! This boundary module owns materialized canonical retention-head bytes. + +use super::head_encoder; +use crate::RetentionHead; + +/// Owned canonical version-2 global retention-head record. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CanonicalRetentionHead { + encoded: [u8; 144], +} + +impl CanonicalRetentionHead { + /// Encodes one validated semantic retention head. + pub fn from_head(head: &RetentionHead) -> Self { + head_encoder::encode(head) + } + + /// Returns the complete exact canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &[u8; 144] { + &self.encoded + } + + pub(super) const fn admitted(encoded: [u8; 144]) -> Self { + Self { encoded } + } +} diff --git a/src/adapters/retention/checksummed_head.rs b/src/adapters/retention/checksummed_head.rs new file mode 100644 index 0000000..6630847 --- /dev/null +++ b/src/adapters/retention/checksummed_head.rs @@ -0,0 +1,45 @@ +//! This boundary module owns a framing- and checksum-verified retention head. + +use super::{RetentionHeadDecodeError, head_decoder}; +use crate::RetentionHead; + +/// Borrowed canonical retention-head bytes with admitted semantic coordinates. +/// +/// This state does not prove that the named manifest exists or that its entries +/// name admitted namespace roots. A reader must bind those artifacts before +/// treating this value as a complete retention snapshot. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ChecksummedRetentionHead<'encoded> { + encoded: &'encoded [u8], + head: RetentionHead, +} + +impl<'encoded> ChecksummedRetentionHead<'encoded> { + /// Decodes exact version-2 framing and verifies the head checksum. + /// + /// This operation performs no allocation or I/O. + /// + /// # Errors + /// + /// Returns [`RetentionHeadDecodeError`] for wrong framing, unsupported or + /// noncanonical fields, checksum disagreement, or invalid coordinates. + pub fn decode(encoded: &'encoded [u8]) -> Result { + head_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the admitted semantic head. + pub const fn head(&self) -> &RetentionHead { + &self.head + } + + pub(super) const fn admitted(encoded: &'encoded [u8], head: RetentionHead) -> Self { + Self { encoded, head } + } +} diff --git a/src/adapters/retention/head_decode_error.rs b/src/adapters/retention/head_decode_error.rs new file mode 100644 index 0000000..dee202b --- /dev/null +++ b/src/adapters/retention/head_decode_error.rs @@ -0,0 +1,66 @@ +//! This boundary module owns typed retention-head decoding failures. + +use crate::{LivenessGenerationError, RetentionHeadError, RetentionManifestLengthError}; + +/// Failure to decode and admit one version-2 retention head. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionHeadDecodeError { + /// The input was not exactly one complete fixed-width head. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed record length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The record carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// Reserved bytes were nonzero. + NonZeroReserved { + /// Observed reserved bytes. + observed: [u8; 8], + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// Liveness-generation admission failed. + LivenessGeneration { + /// Preserved generation failure. + source: LivenessGenerationError, + }, + /// Manifest-length admission failed. + ManifestLength { + /// Preserved manifest-length failure. + source: RetentionManifestLengthError, + }, + /// Final semantic head admission failed. + Semantic { + /// Preserved semantic failure. + source: RetentionHeadError, + }, +} diff --git a/src/adapters/retention/head_decode_error_display.rs b/src/adapters/retention/head_decode_error_display.rs new file mode 100644 index 0000000..601e2c9 --- /dev/null +++ b/src/adapters/retention/head_decode_error_display.rs @@ -0,0 +1,68 @@ +//! This boundary module owns retention-head decode diagnostics and sources. + +use std::{error::Error, fmt}; + +use super::RetentionHeadDecodeError; + +impl fmt::Display for RetentionHeadDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "retention head has {observed} bytes; expected {expected}" + ), + Self::InvalidMagic { observed } => { + write!(formatter, "invalid retention head magic {observed:02x?}") + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported retention head version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "retention head record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported retention head flags {observed:#010x}" + ) + } + Self::NonZeroReserved { observed } => { + write!( + formatter, + "retention head reserved bytes are nonzero: {observed:02x?}" + ) + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("retention head checksum mismatch") + } + Self::LivenessGeneration { source } => { + write!(formatter, "invalid retention-head generation: {source}") + } + Self::ManifestLength { source } => { + write!(formatter, "invalid retention manifest length: {source}") + } + Self::Semantic { source } => { + write!(formatter, "invalid semantic retention head: {source}") + } + } + } +} + +impl Error for RetentionHeadDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LivenessGeneration { source } => Some(source), + Self::ManifestLength { source } => Some(source), + Self::Semantic { source } => Some(source), + Self::WrongLength { .. } + | Self::InvalidMagic { .. } + | Self::UnsupportedVersion { .. } + | Self::InvalidRecordLength { .. } + | Self::UnsupportedFlags { .. } + | Self::NonZeroReserved { .. } + | Self::ChecksumMismatch { .. } => None, + } + } +} diff --git a/src/adapters/retention/head_decoder.rs b/src/adapters/retention/head_decoder.rs new file mode 100644 index 0000000..76d9462 --- /dev/null +++ b/src/adapters/retention/head_decoder.rs @@ -0,0 +1,134 @@ +//! This boundary module owns canonical retention-head decoding order. + +use super::{ChecksummedRetentionHead, RetentionHeadDecodeError}; +use crate::{LivenessGeneration, RetentionHead, RetentionManifestDigest, RetentionManifestLength}; + +pub(super) const ENCODED_LENGTH: usize = 144; +pub(super) const CHECKSUM_OFFSET: usize = 112; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:RET:HEAD2\0\0"; +pub(super) const VERSION: u16 = 2; +pub(super) const RECORD_LENGTH: u16 = 144; +const CHECKSUM_DOMAIN: &[u8] = b"keep.retention-head-checksum/v2\0"; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, RetentionHeadDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let generation = LivenessGeneration::new(read_u64(encoded, 24)?) + .map_err(|source| RetentionHeadDecodeError::LivenessGeneration { source })?; + let manifest_length = RetentionManifestLength::new(read_u64(encoded, 32)?) + .map_err(|source| RetentionHeadDecodeError::ManifestLength { source })?; + let manifest_digest = RetentionManifestDigest::from_hash(read_array(encoded, 40)?); + let predecessor = predecessor(read_array(encoded, 72)?); + let head = RetentionHead::new(generation, manifest_length, manifest_digest, predecessor) + .map_err(|source| RetentionHeadDecodeError::Semantic { source })?; + Ok(ChecksummedRetentionHead::admitted(encoded, head)) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), RetentionHeadDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(RetentionHeadDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(RetentionHeadDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(RetentionHeadDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(RetentionHeadDecodeError::UnsupportedFlags { observed: flags }); + } + let reserved = read_array(encoded, 104)?; + if reserved != [0_u8; 8] { + return Err(RetentionHeadDecodeError::NonZeroReserved { observed: reserved }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), RetentionHeadDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + })?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = checksum(preimage); + if observed == expected { + Ok(()) + } else { + Err(RetentionHeadDecodeError::ChecksumMismatch { expected, observed }) + } +} + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(CHECKSUM_DOMAIN); + hasher.update(preimage); + *hasher.finalize().as_bytes() +} + +fn predecessor(bytes: [u8; 32]) -> Option { + if bytes == [0_u8; 32] { + None + } else { + Some(RetentionManifestDigest::from_hash(bytes)) + } +} + +const fn require_length(encoded: &[u8]) -> Result<(), RetentionHeadDecodeError> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }) + } +} + +fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionHeadDecodeError> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }); + }; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/head_encoder.rs b/src/adapters/retention/head_encoder.rs new file mode 100644 index 0000000..6565476 --- /dev/null +++ b/src/adapters/retention/head_encoder.rs @@ -0,0 +1,32 @@ +//! This boundary module owns canonical version-2 retention-head encoding. + +use super::{CanonicalRetentionHead, head_decoder as format}; +use crate::RetentionHead; + +pub(super) fn encode(head: &RetentionHead) -> CanonicalRetentionHead { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + let (magic, remaining) = preimage.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, remaining) = remaining.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, remaining) = remaining.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, remaining) = remaining.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (generation, remaining) = remaining.split_at_mut(8); + generation.copy_from_slice(&head.generation().get().to_be_bytes()); + let (manifest_length, remaining) = remaining.split_at_mut(8); + manifest_length.copy_from_slice(&head.manifest_length().get().to_be_bytes()); + let (manifest_digest, remaining) = remaining.split_at_mut(32); + manifest_digest.copy_from_slice(head.manifest_digest().as_bytes()); + let (predecessor, remaining) = remaining.split_at_mut(32); + predecessor.copy_from_slice( + &head + .predecessor() + .map_or([0_u8; 32], |digest| *digest.as_bytes()), + ); + let (_reserved, _complete) = remaining.split_at_mut(8); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + CanonicalRetentionHead::admitted(encoded) +} diff --git a/src/lib.rs b/src/lib.rs index 8e07d58..957b5ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,8 +22,8 @@ //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots -//! are validated; canonical in-memory root and manifest encoding and decoding -//! are available. Retention-head codecs, publication, recovery, and garbage +//! are validated; canonical in-memory root, manifest, and head encoding and +//! decoding are available. Retention publication, recovery, and garbage //! collection remain intentionally absent. #[cfg(test)] @@ -44,41 +44,41 @@ pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionManifest, - CanonicalRetentionRoot, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, - FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, - FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, CatalogAdmissionError, + CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, + CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedRetentionHead, + ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, + FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, + RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -86,9 +86,9 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, - RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, @@ -127,9 +127,10 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionManifest, - RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionNamespace, - RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionHead, + RetentionHeadError, RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, + RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, + RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, RootGeneration, RootGenerationError, }; diff --git a/src/retention/head.rs b/src/retention/head.rs new file mode 100644 index 0000000..907a080 --- /dev/null +++ b/src/retention/head.rs @@ -0,0 +1,74 @@ +//! This module owns one semantic global retention head. + +use super::{ + LivenessGeneration, RetentionHeadError, RetentionManifestDigest, RetentionManifestLength, +}; + +/// Exact coordinate of the globally selected retention manifest. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionHead { + generation: LivenessGeneration, + manifest_length: RetentionManifestLength, + manifest_digest: RetentionManifestDigest, + predecessor: Option, +} + +impl RetentionHead { + /// Admits one semantic global retention-head coordinate. + /// + /// # Errors + /// + /// Returns [`RetentionHeadError`] when initial or successor history is + /// malformed. + pub fn new( + generation: LivenessGeneration, + manifest_length: RetentionManifestLength, + manifest_digest: RetentionManifestDigest, + predecessor: Option, + ) -> Result { + admit_predecessor(generation, predecessor)?; + Ok(Self { + generation, + manifest_length, + manifest_digest, + predecessor, + }) + } + + /// Returns the selected global liveness generation. + pub const fn generation(self) -> LivenessGeneration { + self.generation + } + + /// Returns the exact selected manifest length. + pub const fn manifest_length(self) -> RetentionManifestLength { + self.manifest_length + } + + /// Returns the exact selected manifest digest. + pub const fn manifest_digest(self) -> RetentionManifestDigest { + self.manifest_digest + } + + /// Returns the preceding manifest digest, if this is a successor. + pub const fn predecessor(self) -> Option { + self.predecessor + } +} + +fn admit_predecessor( + generation: LivenessGeneration, + predecessor: Option, +) -> Result<(), RetentionHeadError> { + if generation.get() == 1 { + return predecessor.map_or(Ok(()), |observed| { + Err(RetentionHeadError::InitialGenerationHasPredecessor { observed }) + }); + } + if predecessor.is_some() { + Ok(()) + } else { + Err(RetentionHeadError::MissingPredecessor { generation }) + } +} diff --git a/src/retention/head_error.rs b/src/retention/head_error.rs new file mode 100644 index 0000000..9b4dfd0 --- /dev/null +++ b/src/retention/head_error.rs @@ -0,0 +1,39 @@ +//! This module owns semantic retention head failures. + +use std::{error::Error, fmt}; + +use super::{LivenessGeneration, RetentionManifestDigest}; + +/// Failure to construct one semantic retention head. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionHeadError { + /// Generation one carried an impossible predecessor. + InitialGenerationHasPredecessor { + /// Observed predecessor digest. + observed: RetentionManifestDigest, + }, + /// A successor generation omitted its required predecessor. + MissingPredecessor { + /// Successor generation lacking a predecessor. + generation: LivenessGeneration, + }, +} + +impl fmt::Display for RetentionHeadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGenerationHasPredecessor { observed } => write!( + formatter, + "initial retention head has predecessor {:?}", + observed.as_bytes() + ), + Self::MissingPredecessor { generation } => write!( + formatter, + "retention head generation {} requires a predecessor", + generation.get() + ), + } + } +} + +impl Error for RetentionHeadError {} diff --git a/src/retention/manifest_length.rs b/src/retention/manifest_length.rs new file mode 100644 index 0000000..311d2cc --- /dev/null +++ b/src/retention/manifest_length.rs @@ -0,0 +1,55 @@ +//! This module owns canonical retention manifest byte lengths. + +use super::RetentionManifestLengthError; + +const HEADER_LENGTH: u64 = 160; +const ENTRY_LENGTH: u64 = 72; +const TRAILER_LENGTH: u64 = 64; +const MINIMUM_VALUE: u64 = HEADER_LENGTH + TRAILER_LENGTH; +const MAXIMUM_VALUE: u64 = 295_136; + +/// Exact canonical byte length of one complete version-2 retention manifest. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionManifestLength(u64); + +impl RetentionManifestLength { + /// Smallest complete version-2 retention manifest length. + pub const MINIMUM: Self = Self(MINIMUM_VALUE); + + /// Largest complete version-2 retention manifest length. + pub const MAXIMUM: Self = Self(MAXIMUM_VALUE); + + /// Admits one complete canonical retention manifest length. + /// + /// # Errors + /// + /// Returns [`RetentionManifestLengthError`] when `value` exceeds the + /// format bound or cannot contain a whole number of fixed-width entries. + pub const fn new(value: u64) -> Result { + if value < MINIMUM_VALUE || value > MAXIMUM_VALUE { + return Err(RetentionManifestLengthError::OutOfBounds { + minimum: MINIMUM_VALUE, + maximum: MAXIMUM_VALUE, + observed: value, + }); + } + let Some(entry_bytes) = value.checked_sub(MINIMUM_VALUE) else { + return Err(RetentionManifestLengthError::OutOfBounds { + minimum: MINIMUM_VALUE, + maximum: MAXIMUM_VALUE, + observed: value, + }); + }; + if !entry_bytes.is_multiple_of(ENTRY_LENGTH) { + return Err(RetentionManifestLengthError::NotCongruent { observed: value }); + } + Ok(Self(value)) + } + + /// Returns the exact admitted byte length. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} diff --git a/src/retention/manifest_length_error.rs b/src/retention/manifest_length_error.rs new file mode 100644 index 0000000..d3a4ca2 --- /dev/null +++ b/src/retention/manifest_length_error.rs @@ -0,0 +1,45 @@ +//! This module owns retention manifest length admission failures. + +use std::{error::Error, fmt}; + +/// Failure to admit a canonical retention manifest byte length. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionManifestLengthError { + /// The length is outside the version-2 manifest bounds. + OutOfBounds { + /// Smallest complete manifest length. + minimum: u64, + /// Largest permitted manifest length. + maximum: u64, + /// Length supplied by the boundary. + observed: u64, + }, + /// The length cannot contain a whole number of fixed-width entries. + NotCongruent { + /// Length supplied by the boundary. + observed: u64, + }, +} + +impl fmt::Display for RetentionManifestLengthError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::OutOfBounds { + minimum, + maximum, + observed, + } => write!( + formatter, + "retention manifest length {observed} is outside {minimum}..={maximum}" + ), + Self::NotCongruent { observed } => { + write!( + formatter, + "retention manifest length {observed} is not congruent" + ) + } + } + } +} + +impl Error for RetentionManifestLengthError {} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 38ddba9..1b6edba 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -9,12 +9,16 @@ mod anchor; mod closure_limit; mod closure_limit_error; mod closure_limits; +mod head; +mod head_error; mod liveness_generation; mod liveness_generation_error; mod manifest; mod manifest_digest; mod manifest_entry; mod manifest_error; +mod manifest_length; +mod manifest_length_error; mod namespace; mod namespace_digest; mod namespace_error; @@ -31,12 +35,16 @@ pub use anchor::RetentionAnchor; pub use closure_limit::RetentionClosureLimit; pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; +pub use head::RetentionHead; +pub use head_error::RetentionHeadError; pub use liveness_generation::LivenessGeneration; pub use liveness_generation_error::LivenessGenerationError; pub use manifest::RetentionManifest; pub use manifest_digest::RetentionManifestDigest; pub use manifest_entry::RetentionManifestEntry; pub use manifest_error::RetentionManifestError; +pub use manifest_length::RetentionManifestLength; +pub use manifest_length_error::RetentionManifestLengthError; pub use namespace::RetentionNamespace; pub use namespace_digest::RetentionNamespaceDigest; pub use namespace_error::RetentionNamespaceError; diff --git a/tests/retention_head_codec.rs b/tests/retention_head_codec.rs new file mode 100644 index 0000000..b3bd0ce --- /dev/null +++ b/tests/retention_head_codec.rs @@ -0,0 +1,187 @@ +//! Public semantic and canonical-codec laws for the retention head. + +mod support; + +use std::io; + +use keep::{ + CanonicalRetentionHead, ChecksummedRetentionHead, LivenessGeneration, RetentionHead, + RetentionHeadDecodeError, RetentionHeadError, RetentionManifestLength, + RetentionManifestLengthError, +}; + +const ONE_ROOT_MANIFEST: &str = + include_str!("../conformance/segment-store/v2/one-root-manifest.hex"); +const ONE_ROOT_HEAD: &str = include_str!("../conformance/segment-store/v2/one-root-head.hex"); +const CHECKSUM_OFFSET: usize = 112; + +#[test] +fn one_root_head_has_one_semantic_and_canonical_representation() +-> Result<(), Box> { + let manifest_bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let manifest = keep::AdmittedRetentionManifest::decode(&manifest_bytes)?; + let manifest_length = RetentionManifestLength::new(u64::try_from(manifest_bytes.len())?)?; + let head = RetentionHead::new( + manifest.manifest().generation(), + manifest_length, + manifest.digest(), + manifest.manifest().predecessor(), + )?; + + let canonical = CanonicalRetentionHead::from_head(&head); + let head_bytes = fixture_bytes(ONE_ROOT_HEAD)?; + assert_eq!(canonical.encoded(), head_bytes.as_slice()); + + let checksummed = ChecksummedRetentionHead::decode(&head_bytes)?; + assert_eq!(checksummed.encoded(), head_bytes); + assert_eq!(checksummed.head(), &head); + Ok(()) +} + +#[test] +fn manifest_length_and_head_history_are_admitted_exactly() -> Result<(), Box> +{ + assert_eq!(RetentionManifestLength::new(224)?.get(), 224); + assert_eq!(RetentionManifestLength::new(295_136)?.get(), 295_136); + assert!(matches!( + RetentionManifestLength::new(223), + Err(RetentionManifestLengthError::OutOfBounds { .. }) + )); + assert!(matches!( + RetentionManifestLength::new(225), + Err(RetentionManifestLengthError::NotCongruent { .. }) + )); + + let head_bytes = fixture_bytes(ONE_ROOT_HEAD)?; + let head = ChecksummedRetentionHead::decode(&head_bytes)?; + assert!(matches!( + RetentionHead::new( + LivenessGeneration::new(1)?, + head.head().manifest_length(), + head.head().manifest_digest(), + Some(head.head().manifest_digest()), + ), + Err(RetentionHeadError::InitialGenerationHasPredecessor { .. }) + )); + assert!(matches!( + RetentionHead::new( + LivenessGeneration::new(2)?, + head.head().manifest_length(), + head.head().manifest_digest(), + None, + ), + Err(RetentionHeadError::MissingPredecessor { .. }) + )); + Ok(()) +} + +#[test] +fn head_framing_and_integrity_have_exact_first_refusals() -> Result<(), Box> +{ + let bytes = fixture_bytes(ONE_ROOT_HEAD)?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + ChecksummedRetentionHead::decode(&truncated), + Err(RetentionHeadDecodeError::WrongLength { + expected: 144, + observed: 143, + }) + )); + + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + ChecksummedRetentionHead::decode(&trailing), + Err(RetentionHeadDecodeError::WrongLength { + expected: 144, + observed: 145, + }) + )); + + let mut wrong_magic = bytes.clone(); + let first = wrong_magic + .first_mut() + .ok_or_else(|| io::Error::other("frozen retention head is empty"))?; + *first ^= 1; + assert!(matches!( + ChecksummedRetentionHead::decode(&wrong_magic), + Err(RetentionHeadDecodeError::InvalidMagic { .. }) + )); + + let mut checksum_corruption = bytes; + let last = checksum_corruption + .last_mut() + .ok_or_else(|| io::Error::other("frozen retention head is empty"))?; + *last ^= 1; + assert!(matches!( + ChecksummedRetentionHead::decode(&checksum_corruption), + Err(RetentionHeadDecodeError::ChecksumMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn complete_integrity_precedes_head_semantics() -> Result<(), Box> { + let mut bytes = fixture_bytes(ONE_ROOT_HEAD)?; + bytes + .get_mut(24..32) + .ok_or_else(|| io::Error::other("frozen retention head lacks generation bytes"))? + .fill(0); + assert!(matches!( + ChecksummedRetentionHead::decode(&bytes), + Err(RetentionHeadDecodeError::ChecksumMismatch { .. }) + )); + + refresh_checksum(&mut bytes)?; + assert!(matches!( + ChecksummedRetentionHead::decode(&bytes), + Err(RetentionHeadDecodeError::LivenessGeneration { .. }) + )); + + let mut noncanonical_length = fixture_bytes(ONE_ROOT_HEAD)?; + noncanonical_length + .get_mut(32..40) + .ok_or_else(|| io::Error::other("frozen retention head lacks manifest length bytes"))? + .copy_from_slice(&225_u64.to_be_bytes()); + refresh_checksum(&mut noncanonical_length)?; + assert!(matches!( + ChecksummedRetentionHead::decode(&noncanonical_length), + Err(RetentionHeadDecodeError::ManifestLength { .. }) + )); + + let mut missing_predecessor = fixture_bytes(ONE_ROOT_HEAD)?; + missing_predecessor + .get_mut(24..32) + .ok_or_else(|| io::Error::other("frozen retention head lacks generation bytes"))? + .copy_from_slice(&2_u64.to_be_bytes()); + refresh_checksum(&mut missing_predecessor)?; + assert!(matches!( + ChecksummedRetentionHead::decode(&missing_predecessor), + Err(RetentionHeadDecodeError::Semantic { + source: RetentionHeadError::MissingPredecessor { .. }, + }) + )); + Ok(()) +} + +fn fixture_bytes(fixture: &str) -> Result, io::Error> { + let encoded = fixture + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention fixture lacks final newline"))?; + support::decode_hex(encoded) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, trailer) = bytes + .split_at_mut_checked(CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention head lacks its checksum"))?; + let checksum = trailer + .get_mut(..blake3::OUT_LEN) + .ok_or_else(|| io::Error::other("retention head checksum is truncated"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-head-checksum/v2\0"); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} From 500b45250ebca9f5a7c8a7eb108d4c87cdb71c51 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:34:19 -0700 Subject: [PATCH 10/50] Refactor: Isolate retention adapter exports --- src/adapters/mod.rs | 8 +--- src/lib.rs | 109 +++++++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 60 deletions(-) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index b49e065..2e72a15 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -234,7 +234,7 @@ mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; -mod retention; +pub(crate) mod retention; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -449,12 +449,6 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; -pub use retention::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, - CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, -}; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/lib.rs b/src/lib.rs index 957b5ca..e63b8f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,44 +41,49 @@ mod retention; #[cfg(feature = "repository-tasks")] #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; +pub use adapters::retention::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, +}; pub use adapters::{ - AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionManifest, AdmittedRetentionRoot, - AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionHead, - CanonicalRetentionManifest, CanonicalRetentionRoot, CatalogAdmissionError, - CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, - CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedRetentionHead, - ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, - FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, - FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, - PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, - RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, - RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, - RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, - RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, - RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, - RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, + BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, + CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, + LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -86,22 +91,20 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, - SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, - SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, - SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, - SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, - SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, - StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, - WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, - classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, execute_recovery_next_head_finalization, - execute_recovery_segment_resume, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, - plan_recovery_next_head_finalization, plan_recovery_segment_resume, + ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, + SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, + StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, + execute_recovery_next_head_finalization, execute_recovery_segment_resume, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; From c98682e159845f5e194122e8b3f0afda9bf0cee2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:40:53 -0700 Subject: [PATCH 11/50] Fix: Route retention exports through adapters --- src/adapters/mod.rs | 3 ++- src/lib.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 2e72a15..d55d271 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -234,7 +234,7 @@ mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; -pub(crate) mod retention; +mod retention; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -449,6 +449,7 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; +pub use retention::*; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/lib.rs b/src/lib.rs index e63b8f6..5dde8b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,12 +41,6 @@ mod retention; #[cfg(feature = "repository-tasks")] #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; -pub use adapters::retention::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, - CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, -}; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, @@ -108,6 +102,12 @@ pub use adapters::{ plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; +pub use adapters::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, +}; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, ByteRange, ByteRangeError, From a4e3837bb56104dee8ea14b0d75490aa347d13c9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:48:23 -0700 Subject: [PATCH 12/50] Add: Plan retention root transitions --- CHANGELOG.md | 10 +- README.md | 5 +- docs/formats/segment-store-v2/requirements.md | 4 +- docs/formats/segment-store-v2/retention.md | 6 +- src/adapters/retention.rs | 6 + src/adapters/retention/transition_error.rs | 112 ++++++++++++++++ src/adapters/retention/transition_planner.rs | 120 ++++++++++++++++++ .../retention/transition_readiness.rs | 35 +++++ src/lib.rs | 17 ++- src/retention/generation_expectation.rs | 13 ++ src/retention/mod.rs | 2 + src/retention/root_generation.rs | 3 + tests/retention_transition.rs | 96 ++++++++++++++ tests/retention_transition/refusal_laws.rs | 98 ++++++++++++++ 14 files changed, 510 insertions(+), 17 deletions(-) create mode 100644 src/adapters/retention/transition_error.rs create mode 100644 src/adapters/retention/transition_planner.rs create mode 100644 src/adapters/retention/transition_readiness.rs create mode 100644 src/retention/generation_expectation.rs create mode 100644 tests/retention_transition.rs create mode 100644 tests/retention_transition/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5acdfb9..792cd93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -317,9 +317,13 @@ after its public API and format compatibility policies are established. history, namespace uniqueness, bounds, ordering, and all three integrity layers. Typed manifest lengths and semantic global heads now reproduce and admit the exact 144-byte head fixture with fixed framing, checksum-first - semantic admission, and explicit generation-history laws. Version-1 - immutable bytes remain authoritative; production version-2 writing remains - unavailable until issue #19's executable evidence is complete. + semantic admission, and explicit generation-history laws. Storage-independent + transition planning now compares absent or exact-generation expectations, + admits only same-namespace exact successors, preserves expected and observed + stale coordinates, and distinguishes byte-identical already-committed + replay. Version-1 immutable bytes remain authoritative; production version-2 + writing remains unavailable until issue #19's executable evidence is + complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 465fdf7..1b8c6a5 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,9 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Version-2 retention values and canonical in-memory root, global -manifest, and retention-head codecs are implemented. Retention publication, +power loss. Version-2 retention values; canonical in-memory root, global +manifest, and retention-head codecs; and storage-independent expected-state +transition planning are implemented. Closure verification, publication, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 9defde7..bba03ed 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,12 +12,12 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | retry and stale-successor tests | Planned in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 1823e50..1601d98 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root, manifest, and head codecs with integrity -before semantic admission. Filesystem publication, transitions, recovery, and -garbage collection remain absent. +implements validated in-memory root, manifest, and head codecs plus +storage-independent expected-state transition planning. Closure verification, +filesystem publication, recovery, and garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index ec3f2ff..f2d09c2 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -30,6 +30,9 @@ mod root_field_decoder; mod root_header_decoder; mod root_integrity; mod root_semantic_header; +mod transition_error; +mod transition_planner; +mod transition_readiness; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; @@ -42,3 +45,6 @@ pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; +pub use transition_error::RetentionTransitionError; +pub use transition_planner::plan_retention_transition; +pub use transition_readiness::RetentionTransitionReadiness; diff --git a/src/adapters/retention/transition_error.rs b/src/adapters/retention/transition_error.rs new file mode 100644 index 0000000..1b8d887 --- /dev/null +++ b/src/adapters/retention/transition_error.rs @@ -0,0 +1,112 @@ +//! This boundary module owns exact retention transition planning failures. + +use std::{error::Error, fmt}; + +use crate::retention::RetentionGenerationExpectation; +use crate::{RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, RootGenerationError}; + +/// Failure to admit one candidate retention root transition. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionTransitionError { + /// The observed namespace state disagreed with the caller expectation. + StaleGeneration { + /// Caller-supplied expected state. + expected: RetentionGenerationExpectation, + /// Exact observed current generation, or normal absence. + observed: Option, + }, + /// The candidate named a different namespace from the current root. + NamespaceMismatch { + /// Current namespace digest. + expected: RetentionNamespaceDigest, + /// Candidate namespace digest. + observed: RetentionNamespaceDigest, + }, + /// The current generation has no representable successor. + GenerationExhausted { + /// Preserved checked-generation failure. + source: RootGenerationError, + }, + /// The candidate generation was not the exact required successor. + CandidateGeneration { + /// Required candidate generation. + expected: RootGeneration, + /// Observed candidate generation. + observed: RootGeneration, + }, + /// The candidate did not name the current root digest. + CandidatePredecessor { + /// Required predecessor digest. + expected: RetentionRootDigest, + /// Candidate predecessor coordinate. + observed: Option, + }, +} + +impl fmt::Display for RetentionTransitionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Absent, + observed: Some(observed), + } => write!( + formatter, + "retention generation is stale: expected absence, observed generation {}", + observed.get() + ), + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Current(expected), + observed: None, + } => write!( + formatter, + "retention generation is stale: expected generation {}, observed absence", + expected.get() + ), + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Current(expected), + observed: Some(observed), + } => write!( + formatter, + "retention generation is stale: expected generation {}, observed generation {}", + expected.get(), + observed.get() + ), + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Absent, + observed: None, + } => formatter.write_str( + "retention generation stale-state error carried matching absent coordinates", + ), + Self::NamespaceMismatch { .. } => { + formatter.write_str("retention candidate namespace mismatch") + } + Self::GenerationExhausted { source } => { + write!( + formatter, + "retention root generation is exhausted: {source}" + ) + } + Self::CandidateGeneration { expected, observed } => write!( + formatter, + "retention candidate generation must be {}, observed {}", + expected.get(), + observed.get() + ), + Self::CandidatePredecessor { .. } => { + formatter.write_str("retention candidate predecessor digest mismatch") + } + } + } +} + +impl Error for RetentionTransitionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::GenerationExhausted { source } => Some(source), + Self::StaleGeneration { .. } + | Self::NamespaceMismatch { .. } + | Self::CandidateGeneration { .. } + | Self::CandidatePredecessor { .. } => None, + } + } +} diff --git a/src/adapters/retention/transition_planner.rs b/src/adapters/retention/transition_planner.rs new file mode 100644 index 0000000..894e0a4 --- /dev/null +++ b/src/adapters/retention/transition_planner.rs @@ -0,0 +1,120 @@ +//! This boundary module owns storage-independent retention transition planning. + +use super::{AdmittedRetentionRoot, RetentionTransitionError, RetentionTransitionReadiness}; +use crate::RootGeneration; +use crate::retention::RetentionGenerationExpectation; + +/// Compares one expected, observed, and fully admitted candidate root. +/// +/// The returned readiness performs no I/O and proves no closure availability +/// or durability. Exact byte-identical replay is admitted only while the +/// candidate remains the current root and the expectation names its prior +/// state. +/// +/// # Errors +/// +/// Returns [`RetentionTransitionError`] for stale state, namespace mismatch, +/// generation exhaustion, or a non-successor candidate. +pub fn plan_retention_transition<'encoded>( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, + candidate: AdmittedRetentionRoot<'encoded>, +) -> Result, RetentionTransitionError> { + if is_exact_replay(expected, current, &candidate)? { + return Ok(RetentionTransitionReadiness::AlreadyCommitted { candidate }); + } + require_expected_state(expected, current)?; + match current { + Some(current) => validate_successor(current, &candidate)?, + None => validate_initial(&candidate)?, + } + Ok(RetentionTransitionReadiness::Publish { candidate }) +} + +fn is_exact_replay( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, + candidate: &AdmittedRetentionRoot<'_>, +) -> Result { + let Some(current) = current else { + return Ok(false); + }; + if current.encoded() != candidate.encoded() { + return Ok(false); + } + let candidate_generation = candidate.root().generation(); + match expected { + RetentionGenerationExpectation::Absent => { + Ok(candidate_generation == RootGeneration::INITIAL) + } + RetentionGenerationExpectation::Current(generation) => { + let successor = generation + .successor() + .map_err(|source| RetentionTransitionError::GenerationExhausted { source })?; + Ok(candidate_generation == successor) + } + } +} + +fn require_expected_state( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, +) -> Result<(), RetentionTransitionError> { + let observed = current.map(|root| root.root().generation()); + let matches = match expected { + RetentionGenerationExpectation::Absent => observed.is_none(), + RetentionGenerationExpectation::Current(generation) => observed == Some(generation), + }; + if matches { + Ok(()) + } else { + Err(RetentionTransitionError::StaleGeneration { expected, observed }) + } +} + +fn validate_initial(candidate: &AdmittedRetentionRoot<'_>) -> Result<(), RetentionTransitionError> { + let observed = candidate.root().generation(); + if observed == RootGeneration::INITIAL { + Ok(()) + } else { + Err(RetentionTransitionError::CandidateGeneration { + expected: RootGeneration::INITIAL, + observed, + }) + } +} + +fn validate_successor( + current: &AdmittedRetentionRoot<'_>, + candidate: &AdmittedRetentionRoot<'_>, +) -> Result<(), RetentionTransitionError> { + let expected_namespace = current.root().namespace().digest(); + let observed_namespace = candidate.root().namespace().digest(); + if observed_namespace != expected_namespace { + return Err(RetentionTransitionError::NamespaceMismatch { + expected: expected_namespace, + observed: observed_namespace, + }); + } + let expected_generation = current + .root() + .generation() + .successor() + .map_err(|source| RetentionTransitionError::GenerationExhausted { source })?; + let observed_generation = candidate.root().generation(); + if observed_generation != expected_generation { + return Err(RetentionTransitionError::CandidateGeneration { + expected: expected_generation, + observed: observed_generation, + }); + } + let expected_predecessor = current.digest(); + let observed_predecessor = candidate.root().predecessor(); + if observed_predecessor != Some(expected_predecessor) { + return Err(RetentionTransitionError::CandidatePredecessor { + expected: expected_predecessor, + observed: observed_predecessor, + }); + } + Ok(()) +} diff --git a/src/adapters/retention/transition_readiness.rs b/src/adapters/retention/transition_readiness.rs new file mode 100644 index 0000000..2231f15 --- /dev/null +++ b/src/adapters/retention/transition_readiness.rs @@ -0,0 +1,35 @@ +//! This boundary module owns admitted retention transition readiness. + +use super::AdmittedRetentionRoot; + +/// Result of comparing one expected, observed, and candidate root. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub enum RetentionTransitionReadiness<'encoded> { + /// The candidate is the exact next root and still requires publication. + Publish { + /// Fully admitted candidate root. + candidate: AdmittedRetentionRoot<'encoded>, + }, + /// The exact candidate bytes are already the current published root. + AlreadyCommitted { + /// Fully admitted byte-identical replay candidate. + candidate: AdmittedRetentionRoot<'encoded>, + }, +} + +impl<'encoded> RetentionTransitionReadiness<'encoded> { + /// Borrows the fully admitted candidate root. + pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { + match self { + Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, + } + } + + /// Consumes the readiness proof and returns the admitted candidate root. + pub fn into_candidate(self) -> AdmittedRetentionRoot<'encoded> { + match self { + Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 5dde8b2..ee12705 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,8 +23,9 @@ //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots //! are validated; canonical in-memory root, manifest, and head encoding and -//! decoding are available. Retention publication, recovery, and garbage -//! collection remain intentionally absent. +//! decoding plus storage-independent expected-state transition planning are +//! available. Closure verification, retention publication, recovery, and +//! garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -106,7 +107,8 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionError, + RetentionTransitionReadiness, plan_retention_transition, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, @@ -130,10 +132,11 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionHead, - RetentionHeadError, RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, - RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, - RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, + RetentionGenerationExpectation, RetentionHead, RetentionHeadError, RetentionManifest, + RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, + RetentionManifestLength, RetentionManifestLengthError, RetentionNamespace, + RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, RootGeneration, RootGenerationError, }; diff --git a/src/retention/generation_expectation.rs b/src/retention/generation_expectation.rs new file mode 100644 index 0000000..75dfb0b --- /dev/null +++ b/src/retention/generation_expectation.rs @@ -0,0 +1,13 @@ +//! This module owns caller-supplied retention generation expectations. + +use super::RootGeneration; + +/// Expected current state of one retention namespace. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionGenerationExpectation { + /// The namespace must not yet have a published root. + Absent, + /// The namespace must have exactly this current root generation. + Current(RootGeneration), +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 1b6edba..72567e3 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -9,6 +9,7 @@ mod anchor; mod closure_limit; mod closure_limit_error; mod closure_limits; +mod generation_expectation; mod head; mod head_error; mod liveness_generation; @@ -35,6 +36,7 @@ pub use anchor::RetentionAnchor; pub use closure_limit::RetentionClosureLimit; pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; +pub use generation_expectation::RetentionGenerationExpectation; pub use head::RetentionHead; pub use head_error::RetentionHeadError; pub use liveness_generation::LivenessGeneration; diff --git a/src/retention/root_generation.rs b/src/retention/root_generation.rs index 488565c..e1b3a4d 100644 --- a/src/retention/root_generation.rs +++ b/src/retention/root_generation.rs @@ -13,6 +13,9 @@ use super::RootGenerationError; pub struct RootGeneration(NonZeroU64); impl RootGeneration { + /// Initial published root generation. + pub const INITIAL: Self = Self(NonZeroU64::MIN); + /// Admits one positive root generation. /// /// # Errors diff --git a/tests/retention_transition.rs b/tests/retention_transition.rs new file mode 100644 index 0000000..a673e45 --- /dev/null +++ b/tests/retention_transition.rs @@ -0,0 +1,96 @@ +//! Storage-independent retention namespace transition laws. + +#[path = "retention_transition/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::io; + +use keep::{ + AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionGenerationExpectation, + RetentionNamespace, RetentionRoot, RetentionRootDigest, RetentionTransitionReadiness, + RootGeneration, plan_retention_transition, +}; + +const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); + +#[test] +fn absent_namespace_admits_only_the_initial_candidate() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&bytes)?; + let readiness = + plan_retention_transition(RetentionGenerationExpectation::Absent, None, candidate)?; + assert!(matches!( + readiness, + RetentionTransitionReadiness::Publish { candidate } + if candidate.root().generation().get() == 1 + )); + Ok(()) +} + +#[test] +fn exact_successor_and_byte_identical_replay_have_distinct_readiness() +-> Result<(), Box> { + let initial_bytes = fixture_bytes()?; + let current = AdmittedRetentionRoot::decode(&initial_bytes)?; + let successor = successor(¤t)?; + let candidate = AdmittedRetentionRoot::decode(successor.encoded())?; + let readiness = plan_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(¤t), + candidate, + )?; + assert!(matches!( + readiness, + RetentionTransitionReadiness::Publish { candidate } + if candidate.root().generation().get() == 2 + )); + + let published = AdmittedRetentionRoot::decode(successor.encoded())?; + let replay = AdmittedRetentionRoot::decode(successor.encoded())?; + let readiness = plan_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(&published), + replay, + )?; + assert!(matches!( + readiness, + RetentionTransitionReadiness::AlreadyCommitted { candidate } + if candidate.encoded() == successor.encoded() + )); + Ok(()) +} + +fn successor( + current: &AdmittedRetentionRoot<'_>, +) -> Result> { + candidate( + current, + current.root().namespace().as_bytes(), + current.root().generation().successor()?, + Some(current.digest()), + ) +} + +fn candidate( + current: &AdmittedRetentionRoot<'_>, + namespace: &[u8], + generation: RootGeneration, + predecessor: Option, +) -> Result> { + let root = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + generation, + keep::RetentionPolicy::new(current.root().profile(), current.root().limits()), + predecessor, + current.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +fn fixture_bytes() -> Result, io::Error> { + let encoded = ONE_ANCHOR_ROOT + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention root fixture lacks final newline"))?; + support::decode_hex(encoded) +} diff --git a/tests/retention_transition/refusal_laws.rs b/tests/retention_transition/refusal_laws.rs new file mode 100644 index 0000000..3e4c14d --- /dev/null +++ b/tests/retention_transition/refusal_laws.rs @@ -0,0 +1,98 @@ +//! Exact stale, mismatch, and exhaustion transition refusals. + +use keep::{ + AdmittedRetentionRoot, RetentionGenerationExpectation, RetentionTransitionError, + RootGeneration, RootGenerationError, plan_retention_transition, +}; + +use super::{candidate, fixture_bytes}; + +#[test] +fn stale_expected_state_reports_both_coordinates() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&bytes)?; + let expected = RetentionGenerationExpectation::Current(RootGeneration::new(1)?); + assert!(matches!( + plan_retention_transition(expected, None, candidate), + Err(RetentionTransitionError::StaleGeneration { + expected: error_expected, + observed: None, + }) if error_expected == expected + )); + Ok(()) +} + +#[test] +fn successor_requires_the_same_namespace_generation_and_predecessor() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let current = AdmittedRetentionRoot::decode(&bytes)?; + let expected = RetentionGenerationExpectation::Current(current.root().generation()); + + let wrong_namespace = candidate( + ¤t, + b"different", + current.root().generation().successor()?, + Some(current.digest()), + )?; + let candidate_root = AdmittedRetentionRoot::decode(wrong_namespace.encoded())?; + assert!(matches!( + plan_retention_transition(expected, Some(¤t), candidate_root), + Err(RetentionTransitionError::NamespaceMismatch { .. }) + )); + + let wrong_generation = candidate( + ¤t, + current.root().namespace().as_bytes(), + current.root().generation().successor()?.successor()?, + Some(current.digest()), + )?; + let candidate_root = AdmittedRetentionRoot::decode(wrong_generation.encoded())?; + assert!(matches!( + plan_retention_transition(expected, Some(¤t), candidate_root), + Err(RetentionTransitionError::CandidateGeneration { + expected, + observed, + }) if expected.get() == 2 && observed.get() == 3 + )); + + let other_initial = candidate(¤t, b"other", RootGeneration::new(1)?, None)?; + let other = AdmittedRetentionRoot::decode(other_initial.encoded())?; + let wrong_predecessor = candidate( + ¤t, + current.root().namespace().as_bytes(), + current.root().generation().successor()?, + Some(other.digest()), + )?; + let candidate_root = AdmittedRetentionRoot::decode(wrong_predecessor.encoded())?; + assert!(matches!( + plan_retention_transition(expected, Some(¤t), candidate_root), + Err(RetentionTransitionError::CandidatePredecessor { .. }) + )); + Ok(()) +} + +#[test] +fn maximum_current_generation_has_no_transition() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let initial = AdmittedRetentionRoot::decode(&bytes)?; + let maximum = candidate( + &initial, + initial.root().namespace().as_bytes(), + RootGeneration::new(u64::MAX)?, + Some(initial.digest()), + )?; + let current = AdmittedRetentionRoot::decode(maximum.encoded())?; + let candidate = AdmittedRetentionRoot::decode(&bytes)?; + assert!(matches!( + plan_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(¤t), + candidate, + ), + Err(RetentionTransitionError::GenerationExhausted { + source: RootGenerationError::Exhausted { current: u64::MAX }, + }) + )); + Ok(()) +} From ca0b410f9bd5bcdc67c2d2d94994371b2683784b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:57:53 -0700 Subject: [PATCH 13/50] Docs: Define retention closure accounting --- docs/formats/segment-store-v2/README.md | 4 +- docs/formats/segment-store-v2/closure.md | 180 ++++++++++++++++++ docs/formats/segment-store-v2/rationale.md | 10 + docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 19 +- .../retention_store_v2_protocol_contract.rs | 33 ++++ 6 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 docs/formats/segment-store-v2/closure.md diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 63e3731..59a14d2 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -39,7 +39,9 @@ Version 2 retains every version-1 physical law and adds these: The following pages form one protocol: - [Retention records and publication](retention.md) owns canonical namespace, - root-generation, manifest, retention-head, closure, and transition rules. + root-generation, manifest, retention-head, and transition rules. +- [Closure verification](closure.md) owns deterministic traversal, exact + resource accounting, authenticated reconstruction, and closure evidence. - [GC and disposition records](gc.md) owns the canonical planned intent, completion, and recovery-disposition byte grammars. - [Migration and recovery](recovery.md) owns the exact root namespace, diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md new file mode 100644 index 0000000..8af1486 --- /dev/null +++ b/docs/formats/segment-store-v2/closure.md @@ -0,0 +1,180 @@ +# Closure Verification + +- Status: Normative version-2 protocol; production verifier planned in issue + [#19](https://github.com/flyingrobots/keep/issues/19) +- Format coordinate: `keep.segment-store/v2` +- Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) +- Decision record: + [ADR-0009](../../adr/0009-retention-roots-release-and-gc-liveness.md) + +This page defines deterministic closure traversal, exact resource accounting, +authenticated reconstruction, and canonical closure evidence. The +[retention record specification](retention.md) owns the limits stored in each +root generation. The [format rationale](rationale.md) explains why evidence +cardinality and reconstruction work use separate counters. + +## Verification boundary + +Closure verification receives: + +- one admitted root generation with a canonical anchor sequence; +- its exact registered retention-realization profile; +- one pinned, completely verified catalog generation; and +- the root's admitted closure limits. + +Profile and limit admission completes before traversal. The verifier does not +read paths, enumerate a filesystem, consult a clock, invoke a caller callback, +or replace a missing witness. Version 2 selects the single record bound to each +logical identity by the pinned catalog. + +The catalog has already admitted each bound segment record's framing, checksum, +logical identity, and payload. Closure verification consumes those proofs, +decodes layouts again under the closure budget, and authenticates each complete +logical blob. + +## Deterministic traversal + +Anchors are visited in their canonical `BlobId`, then `LayoutId`, order. For +each anchor: + +1. Schedule the anchor's layout at depth `1`. +2. Resolve the exact layout record and charge its resource units. +3. Decode the canonical layout with the named `LayoutId` as an independent + expectation. +4. Require the layout target to equal the anchor's `BlobId`. +5. Visit layout entries in logical-offset order. +6. Schedule each entry's chunk at depth `2`, resolve its exact record, and + charge its resource units. +7. Reconstruct entries in layout order, replay the exact registered storage + profile, and authenticate the complete `BlobId`. + +Version-2 flat layouts cannot exceed depth `2`. The stored depth limit may be +larger so a successor layout grammar can be represented without weakening the +format ceiling. Version 2 refuses an unknown mandatory edge instead of +interpreting it as a deeper known node. + +The visited set is keyed by `SegmentRecordIdentity`. A node is inserted when +its identity is first scheduled, before catalog lookup. An anchor is not a +closure node because the root format bounds anchors separately. Each unique +`SegmentRecordIdentity` contributes one node even when layouts or chunks are +shared. Missing members still consume their scheduled node and depth budget +before the typed missing-member refusal. + +## Exact resource accounting + +Every counter starts at zero. Every increase uses checked addition before the +corresponding lookup, decode, record consumption, or reconstruction step. An +arithmetic overflow is a typed refusal, not an implied limit breach. + +### Nodes + +The node count is the number of unique catalog record identities first +scheduled across the complete root. It includes layout and chunk identities. +It excludes anchors, catalog entries not reached by an anchor, and a repeated +logical occurrence of an already visited identity. + +### Depth + +Depth is the number of catalog record identities on the active edge path. +The layout is depth `1`; one of its chunks is depth `2`. The verifier checks +the candidate depth before scheduling the identity. The observed depth in +successful evidence is the maximum reached across the complete root, or zero +for an empty anchor set. + +### Encoded bytes + +Encoded bytes count structured closure metadata decoded by the verifier. +Version 2 charges the canonical layout payload length once for each unique +layout identity, before decoding that payload. Chunk payloads, segment framing, +root bytes, manifest bytes, catalog bytes, and profile-definition bytes do not +contribute to this counter. + +### Physical bytes + +Physical bytes bound record-backed reconstruction work rather than unique +storage footprint. The verifier charges: + +- the complete segment-record length once when each layout is consumed; and +- the complete segment-record length for every chunk occurrence consumed in + layout order. + +A repeated logical occurrence therefore consumes physical bytes again even +though it does not add a node or another canonical closure-member entry. This +rule bounds replay and blob-authentication work for layouts that repeat one +small chunk many times. Shared physical evidence is not a license for +unbounded logical reconstruction. + +## Fail-closed order + +For one first-scheduled identity, checks occur in this order: + +1. admit the candidate depth; +2. checked-add and admit the node count; +3. resolve the identity from the pinned catalog; +4. checked-add and admit the complete segment-record length; +5. for a layout, checked-add and admit its canonical layout payload length; +6. consume the already admitted record proof; and +7. decode or reconstruct its semantic content. + +An already visited chunk skips steps 2, 3, 5, and canonical-member insertion, +but each logical occurrence repeats the physical-byte check in step 4 before +its bytes enter profile replay and blob authentication. + +The first failed check in deterministic traversal order is returned. Missing, +wrong-kind, unsupported-profile, limit, overflow, layout, anchor-target, +chunk, profile-boundary, and final-blob failures remain distinct typed errors +with expected and observed state where applicable. No failure yields partial +closure evidence. + +## Canonical closure digest + +Successful verification emits 96-byte closure-member entries, one for each +unique record identity: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 1 | record kind | `1` for chunk; `2` for layout | +| 1 | 3 | reserved | zero | +| 4 | 60 | identity slot | canonical encoding below | +| 64 | 32 | record checksum | exact admitted segment-record checksum | + +The chunk identity slot is its four-byte big-endian length, then its 32-byte +digest, then 24 zero bytes. The layout identity slot is the exact 60-byte +canonical binary `LayoutId`. Entries use canonical typed-identity order: +chunks by identity slot, followed by layouts by identity slot. There are no +duplicate entries. + +The closure digest is: + +```text +BLAKE3-256( + "keep.retention-closure/v2\0" || + profile-identity-u32 || + profile-version-u32 || + profile-definition-digest || + catalog-generation-u64 || + catalog-digest || + node-count-u64 || + maximum-depth-u16 || + six-zero-reserved-bytes || + encoded-bytes-u64 || + physical-bytes-u64 || + canonical-closure-member-entries +) +``` + +All integers are unsigned big-endian. `node-count-u64` is also the entry count. +The digest binds the exact profile, catalog, observed resource use, logical +member set, and record checksums. The transition receipt binds it beside the +root's separate anchor-set digest; neither digest substitutes for the other. + +## Evidence and nonclaims + +Successful evidence records the closure digest, all four observed counters, +the exact profile coordinate, and the exact catalog generation and digest. +It proves that every anchor reconstructed and authenticated at verification +time under those coordinates. + +It does not prove application meaning, future reachability after another +generation commits, unique physical ownership, retained byte count on disk, +secure erasure, or a faster verification path than the accounted traversal. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 315bee8..48213b1 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -63,6 +63,16 @@ Pretending to support multiple representation policies would add an unproved abstraction. The registered single-witness profile states the current law exactly; another profile requires a successor specification and evidence. +## Charge closure evidence and reconstruction work separately + +Counting unique closure members alone would let a layout repeat one small +chunk into an effectively unbounded reconstruction. Counting every repeated +identity as another node would misstate the canonical physical evidence. +Version 2 therefore deduplicates node, encoded-metadata, and member-digest +accounting by logical identity, while charging physical record length for +every chunk occurrence consumed during reconstruction. This keeps both the +evidence set and the work bound truthful. + ## Use a kernel reader fence A durable reader registry, lease, clock, and process liveness inference were diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index bba03ed..1ac7da9 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md`; property, corruption, and adversarial catalog tests remain | Specified; implementation planned in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 1601d98..e7e9529 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -240,20 +240,11 @@ The checksum domain is `keep.retention-head-checksum/v2\0`. ## Closure admission Before publication, Keep pins one completely verified catalog generation and -derives the complete closure for every anchor: - -1. Resolve and admit the exact layout record named by `LayoutId`. -2. Require its embedded `BlobId` to equal the anchor `BlobId`. -3. Resolve and admit every ordered chunk identity required by that layout. -4. Verify each physical record, identity, checksum, digest, and catalog - coordinate under the stored realization profile. -5. Enforce the stored limits with checked counters and a visited set. -6. Reconstruct and authenticate the complete blob identity. - -A missing or corrupt closure member, ambiguous catalog claim, unsupported -profile, limit breach, cycle, unknown mandatory edge, identity mismatch, or -ordering error refuses the entire transition. Keep never omits one failed -member and continues with a smaller live set. +applies the exact deterministic traversal, counter units, failure order, +authenticated reconstruction, and canonical digest defined by +[Closure verification](closure.md). Any closure failure refuses the entire +transition. Keep never omits one failed member and continues with a smaller +live set. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index 34ebdcb..d413b2c 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -42,6 +42,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "`keep.segment-store/v2`", "successor to `keep.segment-store/v1`", "[Retention records and publication](retention.md)", + "[Closure verification](closure.md)", "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", "[Migration crash points](migration-crash.md)", @@ -93,6 +94,37 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box> { + let closure = normalized(&read(&format!("{FORMAT_ROOT}/closure.md"))?); + + for required in [ + "one pinned, completely verified catalog generation", + "first scheduled", + "anchor is not a closure node", + "unique `SegmentRecordIdentity`", + "depth `1`", + "depth `2`", + "canonical layout payload length", + "complete segment-record length", + "checked addition before", + "repeated logical occurrence", + "replay the exact registered storage profile", + "authenticate the complete `BlobId`", + "keep.retention-closure/v2\\0", + "96-byte closure-member entries", + "canonical typed-identity order", + "Missing members still consume", + ] { + assert!( + closure.contains(required), + "segment-store v2 closure contract omits `{required}`" + ); + } + Ok(()) +} + #[test] fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> { @@ -221,6 +253,7 @@ fn requirement_ledger_names_planned_and_executable_evidence() fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box> { for name in [ "README.md", + "closure.md", "gc.md", "migration-crash.md", "migration-inventory.md", From f2f910e7c516206f8068fa88f2692d00eeb9f16a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:13:57 -0700 Subject: [PATCH 14/50] Add: Verify pinned retention closures --- CHANGELOG.md | 10 +- README.md | 9 +- docs/formats/segment-store-v2/README.md | 16 +- docs/formats/segment-store-v2/closure.md | 3 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 11 ++ src/adapters/retention/closure_accounting.rs | 110 +++++++++++ src/adapters/retention/closure_digest.rs | 39 ++++ src/adapters/retention/closure_error.rs | 129 +++++++++++++ .../retention/closure_error_display.rs | 123 ++++++++++++ src/adapters/retention/closure_member.rs | 38 ++++ .../retention/closure_profile_error.rs | 28 +++ src/adapters/retention/closure_verifier.rs | 182 ++++++++++++++++++ src/adapters/retention/verified_closure.rs | 60 ++++++ src/lib.rs | 17 +- .../boundary.rs} | 2 +- src/profile/mod.rs | 14 ++ src/profile/verification.rs | 102 ++++++++++ src/profile/verification_error.rs | 31 +++ src/reference/mod.rs | 3 +- src/reference/profile_verification.rs | 122 ++++-------- src/retention/closure_counter.rs | 27 +++ src/retention/closure_digest.rs | 18 ++ src/retention/closure_usage.rs | 51 +++++ src/retention/mod.rs | 6 + tests/retention_closure.rs | 150 +++++++++++++++ 26 files changed, 1191 insertions(+), 112 deletions(-) create mode 100644 src/adapters/retention/closure_accounting.rs create mode 100644 src/adapters/retention/closure_digest.rs create mode 100644 src/adapters/retention/closure_error.rs create mode 100644 src/adapters/retention/closure_error_display.rs create mode 100644 src/adapters/retention/closure_member.rs create mode 100644 src/adapters/retention/closure_profile_error.rs create mode 100644 src/adapters/retention/closure_verifier.rs create mode 100644 src/adapters/retention/verified_closure.rs rename src/{reference/profile_boundary.rs => profile/boundary.rs} (94%) create mode 100644 src/profile/verification.rs create mode 100644 src/profile/verification_error.rs create mode 100644 src/retention/closure_counter.rs create mode 100644 src/retention/closure_digest.rs create mode 100644 src/retention/closure_usage.rs create mode 100644 tests/retention_closure.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 792cd93..74f5188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -321,9 +321,13 @@ after its public API and format compatibility policies are established. transition planning now compares absent or exact-generation expectations, admits only same-namespace exact successors, preserves expected and observed stale coordinates, and distinguishes byte-identical already-committed - replay. Version-1 immutable bytes remain authoritative; production version-2 - writing remains unavailable until issue #19's executable evidence is - complete. + replay. Deterministic storage-independent closure verification now derives + unique catalog members, enforces exact node, depth, encoded-byte, and + physical-byte accounting, replays the registered storage profile, + authenticates each complete retained blob, and emits a catalog-bound + canonical closure digest. Version-1 immutable bytes remain authoritative; + production version-2 writing remains unavailable until issue #19's + executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 1b8c6a5..7194bed 100644 --- a/README.md +++ b/README.md @@ -116,10 +116,11 @@ recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values; canonical in-memory root, global -manifest, and retention-head codecs; and storage-independent expected-state -transition planning are implemented. Closure verification, publication, -recovery, compaction, and garbage collection remain planned. Presence in the -reference CAS does not claim retention, crash recovery, or durability. +manifest, and retention-head codecs; storage-independent expected-state +transition planning; and deterministic bounded closure verification against a +pinned catalog are implemented. Publication, recovery, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim +retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 59a14d2..8ba96bf 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -71,10 +71,12 @@ re-encode them. The format contract is frozen by ADR-0009 and this specification. Public core types now admit exact namespace bytes, namespace digests, root and liveness generations, registered realization profiles, bounded closure policies, -reconstruction anchors, and semantic roots. The canonical root encoder matches -the independent golden record. No production version-2 decoder, transition, -migration, or writer exists yet. Requirements that remain marked as planned or -in progress in issue #19 or issue #21 are not complete implementation evidence. -A store must refuse version-2 state until the relevant parser, corruption, -golden-format, model-based, crash-injection, recovery, and fuzz evidence is -implemented. +reconstruction anchors, and semantic roots. Canonical root, manifest, and head +codecs match their independent golden records. Storage-independent transition +planning and deterministic bounded closure verification against one pinned +catalog are available. Production filesystem retention publication, recovery, +migration, and garbage collection do not exist yet. Requirements that remain +planned or in progress in issue #19 or issue #21 are not complete +implementation evidence. A store must refuse unsupported version-2 state until +the relevant corruption, model-based, crash-injection, recovery, and fuzz +evidence is implemented. diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 8af1486..0f88d6c 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -1,6 +1,7 @@ # Closure Verification -- Status: Normative version-2 protocol; production verifier planned in issue +- Status: Normative version-2 protocol; storage-independent verifier + implemented; publication integration planned in issue [#19](https://github.com/flyingrobots/keep/issues/19) - Format coordinate: `keep.segment-store/v2` - Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 1ac7da9..fa09482 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md`; property, corruption, and adversarial catalog tests remain | Specified; implementation planned in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md` and one-anchor success in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index f2d09c2..74d5b77 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -6,6 +6,13 @@ mod canonical_head; mod canonical_manifest; mod canonical_root; mod checksummed_head; +mod closure_accounting; +mod closure_digest; +mod closure_error; +mod closure_error_display; +mod closure_member; +mod closure_profile_error; +mod closure_verifier; mod head_decode_error; mod head_decode_error_display; mod head_decoder; @@ -33,6 +40,7 @@ mod root_semantic_header; mod transition_error; mod transition_planner; mod transition_readiness; +mod verified_closure; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; @@ -40,6 +48,8 @@ pub use canonical_head::CanonicalRetentionHead; pub use canonical_manifest::CanonicalRetentionManifest; pub use canonical_root::CanonicalRetentionRoot; pub use checksummed_head::ChecksummedRetentionHead; +pub use closure_error::RetentionClosureVerificationError; +pub use closure_verifier::verify_retention_closure; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; @@ -48,3 +58,4 @@ pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; pub use transition_planner::plan_retention_transition; pub use transition_readiness::RetentionTransitionReadiness; +pub use verified_closure::VerifiedRetentionClosure; diff --git a/src/adapters/retention/closure_accounting.rs b/src/adapters/retention/closure_accounting.rs new file mode 100644 index 0000000..c5bb0ec --- /dev/null +++ b/src/adapters/retention/closure_accounting.rs @@ -0,0 +1,110 @@ +//! This module owns checked retention-closure resource accounting. + +use crate::{ + RetentionClosureCounter, RetentionClosureLimits, RetentionClosureUsage, + RetentionClosureVerificationError, +}; + +pub(super) struct ClosureAccounting { + limits: RetentionClosureLimits, + nodes: u64, + maximum_depth: u16, + encoded_bytes: u64, + physical_bytes: u64, +} + +impl ClosureAccounting { + pub(super) const fn new(limits: RetentionClosureLimits) -> Self { + Self { + limits, + nodes: 0, + maximum_depth: 0, + encoded_bytes: 0, + physical_bytes: 0, + } + } + + pub(super) fn admit_depth( + &mut self, + observed: u16, + ) -> Result<(), RetentionClosureVerificationError> { + let maximum = self.limits.depth(); + if observed > maximum { + return Err(RetentionClosureVerificationError::LimitExceeded { + counter: RetentionClosureCounter::Depth, + maximum: u64::from(maximum), + observed: u64::from(observed), + }); + } + self.maximum_depth = self.maximum_depth.max(observed); + Ok(()) + } + + pub(super) fn add_node(&mut self) -> Result<(), RetentionClosureVerificationError> { + self.nodes = checked_add( + RetentionClosureCounter::Nodes, + self.nodes, + 1, + self.limits.nodes(), + )?; + Ok(()) + } + + pub(super) fn add_encoded( + &mut self, + incoming: u64, + ) -> Result<(), RetentionClosureVerificationError> { + self.encoded_bytes = checked_add( + RetentionClosureCounter::EncodedBytes, + self.encoded_bytes, + incoming, + self.limits.encoded_bytes(), + )?; + Ok(()) + } + + pub(super) fn add_physical( + &mut self, + incoming: u64, + ) -> Result<(), RetentionClosureVerificationError> { + self.physical_bytes = checked_add( + RetentionClosureCounter::PhysicalBytes, + self.physical_bytes, + incoming, + self.limits.physical_bytes(), + )?; + Ok(()) + } + + pub(super) const fn usage(&self) -> RetentionClosureUsage { + RetentionClosureUsage::from_verified( + self.nodes, + self.maximum_depth, + self.encoded_bytes, + self.physical_bytes, + ) + } +} + +fn checked_add( + counter: RetentionClosureCounter, + current: u64, + incoming: u64, + maximum: u64, +) -> Result { + let observed = current.checked_add(incoming).ok_or( + RetentionClosureVerificationError::CounterOverflow { + counter, + current, + incoming, + }, + )?; + if observed > maximum { + return Err(RetentionClosureVerificationError::LimitExceeded { + counter, + maximum, + observed, + }); + } + Ok(observed) +} diff --git a/src/adapters/retention/closure_digest.rs b/src/adapters/retention/closure_digest.rs new file mode 100644 index 0000000..40e0f63 --- /dev/null +++ b/src/adapters/retention/closure_digest.rs @@ -0,0 +1,39 @@ +//! This module owns canonical retention-closure transcript hashing. + +use std::collections::BTreeMap; + +use blake3::Hasher; + +use crate::{ + AdmittedSegmentRecord, CatalogDigest, CatalogGeneration, RegisteredRetentionProfile, + RetentionClosureDigest, RetentionClosureUsage, SegmentRecordIdentity, +}; + +use super::closure_member::ClosureMember; + +const DOMAIN: &[u8] = b"keep.retention-closure/v2\0"; + +pub(super) fn calculate( + profile: RegisteredRetentionProfile, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, + usage: RetentionClosureUsage, + records: &BTreeMap>, +) -> RetentionClosureDigest { + let mut hasher = Hasher::new(); + hasher.update(DOMAIN); + hasher.update(&profile.identity().to_be_bytes()); + hasher.update(&profile.version().to_be_bytes()); + hasher.update(profile.digest()); + hasher.update(&catalog_generation.get().to_be_bytes()); + hasher.update(catalog_digest.as_bytes()); + hasher.update(&usage.node_count().to_be_bytes()); + hasher.update(&usage.maximum_depth().to_be_bytes()); + hasher.update(&[0_u8; 6]); + hasher.update(&usage.encoded_bytes().to_be_bytes()); + hasher.update(&usage.physical_bytes().to_be_bytes()); + for (identity, record) in records { + hasher.update(ClosureMember::new(*identity, *record).as_bytes()); + } + RetentionClosureDigest::from_verified(*hasher.finalize().as_bytes()) +} diff --git a/src/adapters/retention/closure_error.rs b/src/adapters/retention/closure_error.rs new file mode 100644 index 0000000..34a66d4 --- /dev/null +++ b/src/adapters/retention/closure_error.rs @@ -0,0 +1,129 @@ +//! This module owns typed failures from retention-closure verification. + +use std::error::Error; +use std::fmt; + +use crate::{ + BlobHashError, BlobId, ChunkingError, LayoutDecodeError, LayoutEntryLimitError, LayoutId, + ProfileBoundary, RetentionClosureCounter, SegmentRecordIdentity, StorageProfileId, +}; + +/// Failure to derive and authenticate one complete retained root closure. +#[derive(Debug)] +pub enum RetentionClosureVerificationError { + /// A checked resource counter overflowed before the next operation. + CounterOverflow { + /// Counter whose addition failed. + counter: RetentionClosureCounter, + /// Value before the failed addition. + current: u64, + /// Requested increment. + incoming: u64, + }, + /// A candidate resource observation exceeds the stored admitted limit. + LimitExceeded { + /// Counter whose limit was exceeded. + counter: RetentionClosureCounter, + /// Stored admitted maximum. + maximum: u64, + /// Candidate observed value. + observed: u64, + }, + /// The admitted node limit could not become a host-independent entry cap. + LayoutEntryLimitHostWidth { + /// Admitted node limit that did not fit the layout cap width. + observed: u64, + }, + /// The derived layout entry cap violated the layout protocol bound. + LayoutEntryLimit { + /// Exact layout-bound refusal. + source: LayoutEntryLimitError, + }, + /// The pinned catalog omits a first-scheduled closure member. + MissingMember { + /// Exact missing logical record identity. + identity: SegmentRecordIdentity, + }, + /// A selected layout failed bounded canonical decoding. + LayoutDecode { + /// Layout named by the retained anchor. + layout: LayoutId, + /// Exact decoding refusal. + source: LayoutDecodeError, + }, + /// A selected layout names another logical blob. + AnchorTargetMismatch { + /// Layout named by the retained anchor. + layout: LayoutId, + /// Blob named by the anchor. + expected: BlobId, + /// Blob embedded in the admitted layout. + observed: BlobId, + }, + /// No replay verifier implements the layout's registered storage profile. + ProfileVerifierUnavailable { + /// Layout whose profile could not be replayed. + layout: LayoutId, + /// Registered profile without a verifier. + profile: StorageProfileId, + }, + /// Replaying the registered storage profile failed. + ProfileChunking { + /// Layout whose profile was replayed. + layout: LayoutId, + /// Exact detector failure. + source: ChunkingError, + }, + /// Replayed profile boundaries differ from the admitted layout. + ProfileBoundaryMismatch { + /// Layout whose profile was replayed. + layout: LayoutId, + /// Zero-based boundary index. + index: usize, + /// Boundary committed by the layout, or absence for an extra boundary. + expected: Option, + /// Replayed boundary, or absence for a missing boundary. + observed: Option, + }, + /// Complete logical identity calculation failed. + BlobHash { + /// Layout whose bytes were hashed. + layout: LayoutId, + /// Exact hashing failure. + source: BlobHashError, + }, + /// Reconstructed bytes do not authenticate as the retained blob. + BlobIdentityMismatch { + /// Layout whose complete stream was verified. + layout: LayoutId, + /// Blob named by the retained anchor. + expected: BlobId, + /// Blob calculated from the selected chunks. + observed: BlobId, + }, +} + +impl Error for RetentionClosureVerificationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LayoutEntryLimit { source } => Some(source), + Self::LayoutDecode { source, .. } => Some(source), + Self::ProfileChunking { source, .. } => Some(source), + Self::BlobHash { source, .. } => Some(source), + Self::CounterOverflow { .. } + | Self::LimitExceeded { .. } + | Self::LayoutEntryLimitHostWidth { .. } + | Self::MissingMember { .. } + | Self::AnchorTargetMismatch { .. } + | Self::ProfileVerifierUnavailable { .. } + | Self::ProfileBoundaryMismatch { .. } + | Self::BlobIdentityMismatch { .. } => None, + } + } +} + +impl fmt::Display for RetentionClosureVerificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + super::closure_error_display::display(self, formatter) + } +} diff --git a/src/adapters/retention/closure_error_display.rs b/src/adapters/retention/closure_error_display.rs new file mode 100644 index 0000000..d36d38a --- /dev/null +++ b/src/adapters/retention/closure_error_display.rs @@ -0,0 +1,123 @@ +//! This module owns stable retention-closure verification diagnostics. + +use std::fmt; + +use super::RetentionClosureVerificationError; + +pub(super) fn display( + error: &RetentionClosureVerificationError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + RetentionClosureVerificationError::CounterOverflow { + counter, + current, + incoming, + } => write!( + formatter, + "{counter} overflowed while adding {incoming} to {current}" + ), + RetentionClosureVerificationError::LimitExceeded { + counter, + maximum, + observed, + } => write!( + formatter, + "{counter} limit {maximum} was exceeded by observed value {observed}" + ), + RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } => write!( + formatter, + "closure node limit {observed} does not fit the layout entry-limit width" + ), + RetentionClosureVerificationError::LayoutEntryLimit { source } => { + write!( + formatter, + "closure-derived layout entry limit is invalid: {source}" + ) + } + RetentionClosureVerificationError::MissingMember { identity } => match identity { + crate::SegmentRecordIdentity::Chunk(chunk) => write!( + formatter, + "pinned catalog omits closure chunk length {} digest {}", + chunk.length(), + DigestHex(chunk.digest()) + ), + crate::SegmentRecordIdentity::Layout(layout) => { + write!(formatter, "pinned catalog omits closure layout {layout}") + } + }, + RetentionClosureVerificationError::LayoutDecode { layout, source } => { + write!( + formatter, + "retained layout {layout} is not admissible: {source}" + ) + } + RetentionClosureVerificationError::AnchorTargetMismatch { + layout, + expected, + observed, + } => write!( + formatter, + "retained layout {layout} names blob {observed}, not anchor blob {expected}" + ), + RetentionClosureVerificationError::ProfileVerifierUnavailable { layout, profile } => { + write!( + formatter, + "retained layout {layout} has no replay verifier for storage profile {profile}" + ) + } + RetentionClosureVerificationError::ProfileChunking { layout, source } => { + write!( + formatter, + "storage-profile replay failed for retained layout {layout}: {source}" + ) + } + RetentionClosureVerificationError::ProfileBoundaryMismatch { + layout, + index, + expected, + observed, + } => write!( + formatter, + "storage-profile boundary {index} for retained layout {layout} expected {} but observed {}", + BoundaryDisplay(*expected), + BoundaryDisplay(*observed) + ), + RetentionClosureVerificationError::BlobHash { layout, source } => { + write!( + formatter, + "blob hashing failed for retained layout {layout}: {source}" + ) + } + RetentionClosureVerificationError::BlobIdentityMismatch { + layout, + expected, + observed, + } => write!( + formatter, + "retained layout {layout} reconstructs {observed}, not anchor blob {expected}" + ), + } +} + +struct BoundaryDisplay(Option); + +impl fmt::Display for BoundaryDisplay { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(boundary) => write!(formatter, "{boundary}"), + None => formatter.write_str("no boundary"), + } + } +} + +struct DigestHex<'a>(&'a [u8; 32]); + +impl fmt::Display for DigestHex<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} diff --git a/src/adapters/retention/closure_member.rs b/src/adapters/retention/closure_member.rs new file mode 100644 index 0000000..75c67bb --- /dev/null +++ b/src/adapters/retention/closure_member.rs @@ -0,0 +1,38 @@ +//! This module owns one canonical retention-closure member entry. + +use crate::{AdmittedSegmentRecord, SegmentRecordIdentity}; + +const ENTRY_LENGTH: usize = 96; +const CHUNK_KIND: u8 = 1; +const LAYOUT_KIND: u8 = 2; + +pub(super) struct ClosureMember([u8; ENTRY_LENGTH]); + +impl ClosureMember { + pub(super) const fn new( + identity: SegmentRecordIdentity, + record: AdmittedSegmentRecord<'_>, + ) -> Self { + let mut encoded = [0_u8; ENTRY_LENGTH]; + let (kind_slot, remainder) = encoded.split_at_mut(1); + kind_slot.copy_from_slice(&[kind(identity)]); + let (_reserved, remainder) = remainder.split_at_mut(3); + let (identity_slot, checksum_slot) = remainder.split_at_mut(60); + identity_slot.copy_from_slice(&crate::adapters::segment_record_identity_encoding::encode( + identity, + )); + checksum_slot.copy_from_slice(record.checksum().as_bytes()); + Self(encoded) + } + + pub(super) const fn as_bytes(&self) -> &[u8; ENTRY_LENGTH] { + &self.0 + } +} + +const fn kind(identity: SegmentRecordIdentity) -> u8 { + match identity { + SegmentRecordIdentity::Chunk(_) => CHUNK_KIND, + SegmentRecordIdentity::Layout(_) => LAYOUT_KIND, + } +} diff --git a/src/adapters/retention/closure_profile_error.rs b/src/adapters/retention/closure_profile_error.rs new file mode 100644 index 0000000..5e4d286 --- /dev/null +++ b/src/adapters/retention/closure_profile_error.rs @@ -0,0 +1,28 @@ +//! This module owns retention mapping for storage-profile replay failures. + +use crate::profile::StorageProfileVerificationError; +use crate::{LayoutId, RetentionClosureVerificationError}; + +pub(super) const fn map( + layout: LayoutId, + error: StorageProfileVerificationError, +) -> RetentionClosureVerificationError { + match error { + StorageProfileVerificationError::Unsupported { profile } => { + RetentionClosureVerificationError::ProfileVerifierUnavailable { layout, profile } + } + StorageProfileVerificationError::Chunking { source } => { + RetentionClosureVerificationError::ProfileChunking { layout, source } + } + StorageProfileVerificationError::BoundaryMismatch { + index, + expected, + observed, + } => RetentionClosureVerificationError::ProfileBoundaryMismatch { + layout, + index, + expected, + observed, + }, + } +} diff --git a/src/adapters/retention/closure_verifier.rs b/src/adapters/retention/closure_verifier.rs new file mode 100644 index 0000000..30d9dc1 --- /dev/null +++ b/src/adapters/retention/closure_verifier.rs @@ -0,0 +1,182 @@ +//! This module owns deterministic verification of one retained-root closure. + +use std::collections::BTreeMap; + +use crate::profile::StorageProfileVerifier; +use crate::{ + AdmittedLayout, AdmittedSegmentRecord, BlobHasher, CatalogSnapshot, LayoutDecodePolicy, + LayoutEntryLimit, RetentionAnchor, RetentionClosureVerificationError, RetentionRoot, + SegmentRecordIdentity, +}; + +use super::{ + VerifiedRetentionClosure, closure_accounting::ClosureAccounting, closure_digest, + closure_profile_error, +}; + +const LAYOUT_DEPTH: u16 = 1; +const CHUNK_DEPTH: u16 = 2; + +/// Verifies every anchor against one pinned admitted catalog. +/// +/// Verification performs no I/O. It allocates one bounded ordered record index +/// and one bounded decoded entry set per anchor. Every selected chunk is +/// scanned to replay its storage profile and authenticate the complete blob. +/// +/// # Errors +/// +/// Returns the first deterministic resource, catalog-member, layout, profile, +/// or reconstructed-identity refusal. No failure returns partial evidence. +pub fn verify_retention_closure( + root: &RetentionRoot, + catalog: &CatalogSnapshot<'_, '_, '_>, +) -> Result { + let entry_limit = layout_entry_limit(root)?; + let mut verifier = ClosureVerifier::new(root, catalog, entry_limit); + for anchor in root.anchors().iter().copied() { + verifier.verify_anchor(anchor)?; + } + Ok(verifier.finish()) +} + +struct ClosureVerifier<'snapshot, 'head, 'catalog, 'records> { + root: &'snapshot RetentionRoot, + catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, + entry_limit: LayoutEntryLimit, + accounting: ClosureAccounting, + records: BTreeMap>, +} + +impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'catalog, 'records> { + const fn new( + root: &'snapshot RetentionRoot, + catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, + entry_limit: LayoutEntryLimit, + ) -> Self { + Self { + root, + catalog, + entry_limit, + accounting: ClosureAccounting::new(root.limits()), + records: BTreeMap::new(), + } + } + + fn verify_anchor( + &mut self, + anchor: RetentionAnchor, + ) -> Result<(), RetentionClosureVerificationError> { + let layout_id = anchor.layout_id(); + let identity = SegmentRecordIdentity::Layout(layout_id); + let (record, first_scheduled) = self.resolve(identity, LAYOUT_DEPTH)?; + self.accounting + .add_physical(record.header().record_length().get())?; + if first_scheduled { + self.accounting + .add_encoded(record.header().payload_length().get())?; + } + let policy = LayoutDecodePolicy::new(self.entry_limit).with_expected_id(layout_id); + let layout = AdmittedLayout::decode_record(record.payload(), policy).map_err(|source| { + RetentionClosureVerificationError::LayoutDecode { + layout: layout_id, + source, + } + })?; + require_anchor_target(anchor, &layout)?; + self.verify_reconstruction(anchor, &layout) + } + + fn verify_reconstruction( + &mut self, + anchor: RetentionAnchor, + layout: &AdmittedLayout, + ) -> Result<(), RetentionClosureVerificationError> { + let layout_id = anchor.layout_id(); + let mut profile = StorageProfileVerifier::new(layout) + .map_err(|error| closure_profile_error::map(layout_id, error))?; + let mut blob = BlobHasher::new(); + for entry in layout.entries().iter().copied() { + let identity = SegmentRecordIdentity::Chunk(entry.chunk_id()); + let (record, _first_scheduled) = self.resolve(identity, CHUNK_DEPTH)?; + self.accounting + .add_physical(record.header().record_length().get())?; + let bytes = record.payload(); + profile + .feed(bytes) + .map_err(|error| closure_profile_error::map(layout_id, error))?; + blob.update(bytes) + .map_err(|source| RetentionClosureVerificationError::BlobHash { + layout: layout_id, + source, + })?; + } + profile + .finish() + .map_err(|error| closure_profile_error::map(layout_id, error))?; + let observed = blob.finish(); + let expected = anchor.blob_id(); + if observed != expected { + return Err(RetentionClosureVerificationError::BlobIdentityMismatch { + layout: layout_id, + expected, + observed, + }); + } + Ok(()) + } + + fn resolve( + &mut self, + identity: SegmentRecordIdentity, + depth: u16, + ) -> Result<(AdmittedSegmentRecord<'records>, bool), RetentionClosureVerificationError> { + self.accounting.admit_depth(depth)?; + if let Some(record) = self.records.get(&identity).copied() { + return Ok((record, false)); + } + self.accounting.add_node()?; + let record = self + .catalog + .record(identity) + .ok_or(RetentionClosureVerificationError::MissingMember { identity })?; + self.records.insert(identity, record); + Ok((record, true)) + } + + fn finish(self) -> VerifiedRetentionClosure { + let profile = self.root.profile(); + let generation = self.catalog.generation(); + let catalog_digest = self.catalog.catalog_digest(); + let usage = self.accounting.usage(); + let digest = + closure_digest::calculate(profile, generation, catalog_digest, usage, &self.records); + VerifiedRetentionClosure::new(profile, generation, catalog_digest, usage, digest) + } +} + +fn layout_entry_limit( + root: &RetentionRoot, +) -> Result { + let observed = root.limits().nodes(); + let value = u32::try_from(observed).map_err(|_source| { + RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } + })?; + LayoutEntryLimit::new(value) + .map_err(|source| RetentionClosureVerificationError::LayoutEntryLimit { source }) +} + +fn require_anchor_target( + anchor: RetentionAnchor, + layout: &AdmittedLayout, +) -> Result<(), RetentionClosureVerificationError> { + let expected = anchor.blob_id(); + let observed = layout.target(); + if observed == expected { + return Ok(()); + } + Err(RetentionClosureVerificationError::AnchorTargetMismatch { + layout: anchor.layout_id(), + expected, + observed, + }) +} diff --git a/src/adapters/retention/verified_closure.rs b/src/adapters/retention/verified_closure.rs new file mode 100644 index 0000000..ecae6ca --- /dev/null +++ b/src/adapters/retention/verified_closure.rs @@ -0,0 +1,60 @@ +//! This module owns successful retention-closure verification evidence. + +use crate::{ + CatalogDigest, CatalogGeneration, RegisteredRetentionProfile, RetentionClosureDigest, + RetentionClosureUsage, +}; + +/// Exact coordinates and accounting for one completely verified closure. +#[must_use = "verified closure evidence binds the transition's physical claim"] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VerifiedRetentionClosure { + profile: RegisteredRetentionProfile, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, + usage: RetentionClosureUsage, + digest: RetentionClosureDigest, +} + +impl VerifiedRetentionClosure { + /// Returns the exact registered retention-realization profile. + pub const fn profile(self) -> RegisteredRetentionProfile { + self.profile + } + + /// Returns the pinned catalog generation used for verification. + pub const fn catalog_generation(self) -> CatalogGeneration { + self.catalog_generation + } + + /// Returns the pinned catalog digest used for verification. + pub const fn catalog_digest(self) -> CatalogDigest { + self.catalog_digest + } + + /// Returns the exact successful resource accounting. + pub const fn usage(self) -> RetentionClosureUsage { + self.usage + } + + /// Returns the canonical closure transcript digest. + pub const fn digest(self) -> RetentionClosureDigest { + self.digest + } + + pub(super) const fn new( + profile: RegisteredRetentionProfile, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, + usage: RetentionClosureUsage, + digest: RetentionClosureDigest, + ) -> Self { + Self { + profile, + catalog_generation, + catalog_digest, + usage, + digest, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index ee12705..12b90ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,9 +23,10 @@ //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots //! are validated; canonical in-memory root, manifest, and head encoding and -//! decoding plus storage-independent expected-state transition planning are -//! available. Closure verification, retention publication, recovery, and -//! garbage collection remain intentionally absent. +//! decoding, storage-independent expected-state transition planning, and +//! deterministic bounded closure verification against a pinned catalog are +//! available. Retention publication, recovery, and garbage collection remain +//! intentionally absent. #[cfg(test)] extern crate self as keep; @@ -106,9 +107,10 @@ pub use adapters::{ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionError, - RetentionTransitionReadiness, plan_retention_transition, + RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, + RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionTransitionError, RetentionTransitionReadiness, VerifiedRetentionClosure, + plan_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, @@ -132,7 +134,8 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, + RetentionClosureCounter, RetentionClosureDigest, RetentionClosureLimit, + RetentionClosureLimitError, RetentionClosureLimits, RetentionClosureUsage, RetentionGenerationExpectation, RetentionHead, RetentionHeadError, RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, RetentionNamespace, diff --git a/src/reference/profile_boundary.rs b/src/profile/boundary.rs similarity index 94% rename from src/reference/profile_boundary.rs rename to src/profile/boundary.rs index 5eabba9..16969fd 100644 --- a/src/reference/profile_boundary.rs +++ b/src/profile/boundary.rs @@ -1,4 +1,4 @@ -//! Compact semantic coordinate for profile-replay diagnostics. +//! This module owns one semantic storage-profile boundary coordinate. use std::fmt; diff --git a/src/profile/mod.rs b/src/profile/mod.rs index 4a6f5b5..05987fe 100644 --- a/src/profile/mod.rs +++ b/src/profile/mod.rs @@ -5,9 +5,23 @@ //! profile selection policy, storage, or application metadata. mod admission_error; +mod boundary; mod id; mod registered; +mod verification; +mod verification_error; pub use admission_error::StorageProfileAdmissionError; +pub use boundary::ProfileBoundary; pub use id::StorageProfileId; pub use registered::RegisteredStorageProfile; +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters share profile replay without exposing it publicly" +)] +pub(crate) use verification::StorageProfileVerifier; +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters map the same domain replay failures" +)] +pub(crate) use verification_error::StorageProfileVerificationError; diff --git a/src/profile/verification.rs b/src/profile/verification.rs new file mode 100644 index 0000000..3803e34 --- /dev/null +++ b/src/profile/verification.rs @@ -0,0 +1,102 @@ +//! This module owns streaming replay of one admitted storage profile. + +use crate::{AdmittedLayout, ChunkSpan, FastCdc, LayoutEntry, RegisteredStorageProfile}; + +use super::{ProfileBoundary, StorageProfileVerificationError}; + +/// Streaming verifier for the profile and boundaries bound by one layout. +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters share profile replay without depending on each other" +)] +pub(crate) struct StorageProfileVerifier<'a> { + detector: FastCdc, + observation: BoundaryObservation<'a>, +} + +impl<'a> StorageProfileVerifier<'a> { + /// Starts replay for one already admitted layout. + pub(crate) fn new(layout: &'a AdmittedLayout) -> Result { + if layout.profile() != RegisteredStorageProfile::FAST_CDC_64K_V1 { + return Err(StorageProfileVerificationError::Unsupported { + profile: layout.profile().id(), + }); + } + Ok(Self { + detector: FastCdc::new(), + observation: BoundaryObservation::new(layout.entries()), + }) + } + + /// Feeds the next exact logical byte span in layout order. + pub(crate) fn feed(&mut self, bytes: &[u8]) -> Result<(), StorageProfileVerificationError> { + let observation = &mut self.observation; + self.detector + .feed(bytes, |span| observation.observe(span)) + .map_err(|source| StorageProfileVerificationError::Chunking { source })?; + observation.check() + } + + /// Finishes replay and requires the exact admitted boundary sequence. + pub(crate) fn finish(mut self) -> Result<(), StorageProfileVerificationError> { + let final_span = self + .detector + .finish() + .map_err(|source| StorageProfileVerificationError::Chunking { source })?; + if let Some(span) = final_span { + self.observation.observe(span); + } + self.observation.finish() + } +} + +struct BoundaryObservation<'a> { + expected: &'a [LayoutEntry], + next: usize, + mismatch: Option, +} + +impl<'a> BoundaryObservation<'a> { + const fn new(expected: &'a [LayoutEntry]) -> Self { + Self { + expected, + next: 0, + mismatch: None, + } + } + + fn observe(&mut self, span: ChunkSpan) { + if self.mismatch.is_some() { + return; + } + let observed = LayoutEntry::from(span); + let expected = self.expected.get(self.next).copied(); + if expected == Some(observed) + && let Some(accepted) = self.expected.get(..=self.next) + { + self.next = accepted.len(); + return; + } + self.mismatch = Some(StorageProfileVerificationError::BoundaryMismatch { + index: self.next, + expected: expected.map(ProfileBoundary::from), + observed: Some(ProfileBoundary::from(observed)), + }); + } + + fn check(&mut self) -> Result<(), StorageProfileVerificationError> { + self.mismatch.take().map_or(Ok(()), Err) + } + + fn finish(mut self) -> Result<(), StorageProfileVerificationError> { + self.check()?; + if let Some(expected) = self.expected.get(self.next).copied() { + return Err(StorageProfileVerificationError::BoundaryMismatch { + index: self.next, + expected: Some(ProfileBoundary::from(expected)), + observed: None, + }); + } + Ok(()) + } +} diff --git a/src/profile/verification_error.rs b/src/profile/verification_error.rs new file mode 100644 index 0000000..733d018 --- /dev/null +++ b/src/profile/verification_error.rs @@ -0,0 +1,31 @@ +//! This module owns storage-profile replay failures independent of adapters. + +use crate::{ChunkingError, ProfileBoundary, StorageProfileId}; + +/// Failure while replaying one admitted storage profile over logical bytes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters map the same domain replay failures" +)] +pub(crate) enum StorageProfileVerificationError { + /// No replay verifier implements the admitted profile. + Unsupported { + /// Registered profile without a verifier. + profile: StorageProfileId, + }, + /// The registered detector refused the supplied byte stream. + Chunking { + /// Exact detector failure. + source: ChunkingError, + }, + /// Replayed boundaries differ from the admitted layout. + BoundaryMismatch { + /// Zero-based boundary index. + index: usize, + /// Boundary committed by the layout, or absence for an extra boundary. + expected: Option, + /// Replayed boundary, or absence for a missing boundary. + observed: Option, + }, +} diff --git a/src/reference/mod.rs b/src/reference/mod.rs index 9ef043d..d612492 100644 --- a/src/reference/mod.rs +++ b/src/reference/mod.rs @@ -10,7 +10,6 @@ mod chunk_verification; mod ingestion; mod ingestion_error; mod output_write; -mod profile_boundary; mod profile_verification; mod publish_error; mod published_blob; @@ -27,9 +26,9 @@ mod reconstruction_receipt; mod staged_blob; mod store; +pub use crate::profile::ProfileBoundary; pub use capacity::ReferenceStoreCapacity; pub use ingestion_error::{IngestionAllocation, IngestionError}; -pub use profile_boundary::ProfileBoundary; pub use publish_error::PublishError; pub use published_blob::PublishedBlob; pub use range_read_error::RangeReadError; diff --git a/src/reference/profile_verification.rs b/src/reference/profile_verification.rs index 97b6357..e14e061 100644 --- a/src/reference/profile_verification.rs +++ b/src/reference/profile_verification.rs @@ -1,108 +1,58 @@ -//! Registered storage-profile replay during reconstruction. +//! Reference-adapter mapping for domain-owned storage-profile replay. -use crate::{AdmittedLayout, ChunkSpan, FastCdc, LayoutEntry, LayoutId, RegisteredStorageProfile}; +use crate::profile::{StorageProfileVerificationError, StorageProfileVerifier}; +use crate::{AdmittedLayout, LayoutId}; -use super::{ProfileBoundary, ReconstructionError}; +use super::ReconstructionError; pub(super) struct ProfileVerifier<'a> { - detector: FastCdc, - observation: BoundaryObservation<'a>, + layout: LayoutId, + verifier: StorageProfileVerifier<'a>, } impl<'a> ProfileVerifier<'a> { pub(super) fn new( - layout_id: LayoutId, - layout: &'a AdmittedLayout, + layout: LayoutId, + admitted: &'a AdmittedLayout, ) -> Result { - if layout.profile() != RegisteredStorageProfile::FAST_CDC_64K_V1 { - return Err(ReconstructionError::ProfileVerifierUnavailable { - layout: layout_id, - profile: layout.profile().id(), - }); - } - Ok(Self { - detector: FastCdc::new(), - observation: BoundaryObservation::new(layout_id, layout.entries()), - }) + let verifier = + StorageProfileVerifier::new(admitted).map_err(|error| map_error(layout, error))?; + Ok(Self { layout, verifier }) } pub(super) fn feed(&mut self, bytes: &[u8]) -> Result<(), ReconstructionError> { - let observation = &mut self.observation; - self.detector - .feed(bytes, |span| observation.observe(span)) - .map_err(|source| ReconstructionError::ProfileChunking { - layout: observation.layout, - source, - })?; - observation.check() + self.verifier + .feed(bytes) + .map_err(|error| map_error(self.layout, error)) } - pub(super) fn finish(mut self) -> Result<(), ReconstructionError> { - let final_span = - self.detector - .finish() - .map_err(|source| ReconstructionError::ProfileChunking { - layout: self.observation.layout, - source, - })?; - if let Some(span) = final_span { - self.observation.observe(span); - } - self.observation.finish() + pub(super) fn finish(self) -> Result<(), ReconstructionError> { + self.verifier + .finish() + .map_err(|error| map_error(self.layout, error)) } } -struct BoundaryObservation<'a> { +const fn map_error( layout: LayoutId, - expected: &'a [LayoutEntry], - next: usize, - mismatch: Option, -} - -impl<'a> BoundaryObservation<'a> { - const fn new(layout: LayoutId, expected: &'a [LayoutEntry]) -> Self { - Self { - layout, - expected, - next: 0, - mismatch: None, - } - } - - fn observe(&mut self, span: ChunkSpan) { - if self.mismatch.is_some() { - return; + error: StorageProfileVerificationError, +) -> ReconstructionError { + match error { + StorageProfileVerificationError::Unsupported { profile } => { + ReconstructionError::ProfileVerifierUnavailable { layout, profile } } - let observed = LayoutEntry::from(span); - let expected = self.expected.get(self.next).copied(); - if expected == Some(observed) - && let Some(accepted) = self.expected.get(..=self.next) - { - self.next = accepted.len(); - return; + StorageProfileVerificationError::Chunking { source } => { + ReconstructionError::ProfileChunking { layout, source } } - self.mismatch = Some(ReconstructionError::ProfileBoundaryMismatch { - layout: self.layout, - index: self.next, - expected: expected.map(ProfileBoundary::from), - observed: Some(ProfileBoundary::from(observed)), - }); - } - - fn check(&mut self) -> Result<(), ReconstructionError> { - self.mismatch.take().map_or(Ok(()), Err) - } - - fn finish(mut self) -> Result<(), ReconstructionError> { - self.check()?; - if let Some(expected) = self.expected.get(self.next).copied() { - return Err(ReconstructionError::ProfileBoundaryMismatch { - layout: self.layout, - index: self.next, - expected: Some(ProfileBoundary::from(expected)), - observed: None, - }); - } - Ok(()) + StorageProfileVerificationError::BoundaryMismatch { + index, + expected, + observed, + } => ReconstructionError::ProfileBoundaryMismatch { + layout, + index, + expected, + observed, + }, } } diff --git a/src/retention/closure_counter.rs b/src/retention/closure_counter.rs new file mode 100644 index 0000000..172ce0a --- /dev/null +++ b/src/retention/closure_counter.rs @@ -0,0 +1,27 @@ +//! This module owns typed retention-closure resource dimensions. + +use std::fmt; + +/// Resource dimension enforced during retention-closure verification. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum RetentionClosureCounter { + /// Unique first-scheduled catalog record identities. + Nodes, + /// Maximum catalog-record edge depth. + Depth, + /// Unique structured layout payload bytes decoded. + EncodedBytes, + /// Complete record bytes charged to reconstruction work. + PhysicalBytes, +} + +impl fmt::Display for RetentionClosureCounter { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Nodes => "closure nodes", + Self::Depth => "closure depth", + Self::EncodedBytes => "encoded closure bytes", + Self::PhysicalBytes => "physical closure bytes", + }) + } +} diff --git a/src/retention/closure_digest.rs b/src/retention/closure_digest.rs new file mode 100644 index 0000000..d4ee12d --- /dev/null +++ b/src/retention/closure_digest.rs @@ -0,0 +1,18 @@ +//! This module owns one verified version-2 retention-closure digest. + +/// BLAKE3-256 digest of one canonical verified closure transcript. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionClosureDigest([u8; 32]); + +impl RetentionClosureDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(crate) const fn from_verified(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/retention/closure_usage.rs b/src/retention/closure_usage.rs new file mode 100644 index 0000000..06f015f --- /dev/null +++ b/src/retention/closure_usage.rs @@ -0,0 +1,51 @@ +//! This module owns observed resource use for one verified retention closure. + +/// Exact successful resource accounting for one complete retained root. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionClosureUsage { + node_count: u64, + maximum_depth: u16, + encoded_bytes: u64, + physical_bytes: u64, +} + +impl RetentionClosureUsage { + /// Returns the unique first-scheduled record count. + #[must_use] + pub const fn node_count(self) -> u64 { + self.node_count + } + + /// Returns the maximum catalog-record edge depth reached. + #[must_use] + pub const fn maximum_depth(self) -> u16 { + self.maximum_depth + } + + /// Returns the unique structured layout payload bytes decoded. + #[must_use] + pub const fn encoded_bytes(self) -> u64 { + self.encoded_bytes + } + + /// Returns complete record bytes charged to reconstruction work. + #[must_use] + pub const fn physical_bytes(self) -> u64 { + self.physical_bytes + } + + pub(crate) const fn from_verified( + node_count: u64, + maximum_depth: u16, + encoded_bytes: u64, + physical_bytes: u64, + ) -> Self { + Self { + node_count, + maximum_depth, + encoded_bytes, + physical_bytes, + } + } +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 72567e3..a9da928 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -6,9 +6,12 @@ //! collection. mod anchor; +mod closure_counter; +mod closure_digest; mod closure_limit; mod closure_limit_error; mod closure_limits; +mod closure_usage; mod generation_expectation; mod head; mod head_error; @@ -33,9 +36,12 @@ mod root_generation; mod root_generation_error; pub use anchor::RetentionAnchor; +pub use closure_counter::RetentionClosureCounter; +pub use closure_digest::RetentionClosureDigest; pub use closure_limit::RetentionClosureLimit; pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; +pub use closure_usage::RetentionClosureUsage; pub use generation_expectation::RetentionGenerationExpectation; pub use head::RetentionHead; pub use head_error::RetentionHeadError; diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs new file mode 100644 index 0000000..4a6bf37 --- /dev/null +++ b/tests/retention_closure.rs @@ -0,0 +1,150 @@ +//! Deterministic retention-closure verification laws. + +mod support; + +use std::error::Error; + +use blake3::Hasher; +use keep::{ + AdmittedCatalog, AdmittedSegment, BlobId, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, LayoutEntryLimit, LayoutId, RegisteredRetentionProfile, + RetentionAnchor, RetentionClosureLimits, RetentionNamespace, RetentionPolicy, RetentionRoot, + RootGeneration, SegmentReadPolicy, SegmentRecordLimit, verify_retention_closure, +}; +use support::decode_hex; + +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const ONE_ZERO_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:1:", + "1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" +); +const ONE_ZERO_LAYOUT: &str = concat!( + "keep:layout:v1:flat-chunks-v1:blake3-256:220:", + "887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8" +); +const CHUNK_DIGEST_HEX: &str = "9b9c9a42912a0efdcd41e83ea024d72f10f2627d239e4eb240dd53f39ce0ff62"; +const CHUNK_RECORD_CHECKSUM_HEX: &str = + "becb46b35120723210798a47e26144b8214d5ea65d28806e0ba941d2aa66bbfa"; +const LAYOUT_RECORD_CHECKSUM_HEX: &str = + "c498a9c3cc24142926857d778fee7fd622b8b03312318a2360a68be3461168d6"; +const CLOSURE_DOMAIN: &[u8] = b"keep.retention-closure/v2\0"; + +#[test] +fn one_anchor_closure_binds_exact_evidence_and_authenticated_bytes() -> Result<(), Box> { + let segment_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let catalog_bytes = fixture(BUNDLE_CATALOG_HEX)?; + let head_bytes = fixture(BUNDLE_HEAD_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + let root = one_anchor_root()?; + + let evidence = verify_retention_closure(&root, &snapshot)?; + + assert_eq!(evidence.profile(), root.profile()); + assert_eq!(evidence.catalog_generation(), snapshot.generation()); + assert_eq!(evidence.catalog_digest(), snapshot.catalog_digest()); + assert_eq!(evidence.usage().node_count(), 2); + assert_eq!(evidence.usage().maximum_depth(), 2); + assert_eq!(evidence.usage().encoded_bytes(), 220); + assert_eq!(evidence.usage().physical_bytes(), 509); + assert_eq!(evidence.digest().as_bytes(), &expected_digest(&snapshot)?); + Ok(()) +} + +fn admitted_catalog<'catalog, 'records>( + catalog_bytes: &'catalog [u8], + segments: &'records [AdmittedSegment<'records>], +) -> Result, Box> { + ChecksummedCatalog::decode(catalog_bytes)? + .admit(segments) + .map_err(Into::into) +} + +fn one_anchor_root() -> Result> { + let blob: BlobId = ONE_ZERO_BLOB.parse()?; + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + Ok(RetentionRoot::new( + RetentionNamespace::try_from(b"contract".as_slice())?, + RootGeneration::new(1)?, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(2, 2, 220, 509)?, + ), + None, + vec![RetentionAnchor::new(blob, layout)], + )?) +} + +fn expected_digest(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Result<[u8; 32], Box> { + let profile = RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1; + let mut hasher = Hasher::new(); + hasher.update(CLOSURE_DOMAIN); + hasher.update(&profile.identity().to_be_bytes()); + hasher.update(&profile.version().to_be_bytes()); + hasher.update(profile.digest()); + hasher.update(&snapshot.generation().get().to_be_bytes()); + hasher.update(snapshot.catalog_digest().as_bytes()); + hasher.update(&2_u64.to_be_bytes()); + hasher.update(&2_u16.to_be_bytes()); + hasher.update(&[0_u8; 6]); + hasher.update(&220_u64.to_be_bytes()); + hasher.update(&509_u64.to_be_bytes()); + hasher.update(&chunk_member()?); + hasher.update(&layout_member()?); + Ok(*hasher.finalize().as_bytes()) +} + +fn chunk_member() -> Result<[u8; 96], Box> { + let mut entry = [0_u8; 96]; + *entry.first_mut().ok_or("closure member has no kind byte")? = 1; + entry + .get_mut(4..8) + .ok_or("closure member lacks chunk length")? + .copy_from_slice(&1_u32.to_be_bytes()); + entry + .get_mut(8..40) + .ok_or("closure member lacks chunk digest")? + .copy_from_slice(&digest(CHUNK_DIGEST_HEX)?); + entry + .get_mut(64..96) + .ok_or("closure member lacks checksum")? + .copy_from_slice(&digest(CHUNK_RECORD_CHECKSUM_HEX)?); + Ok(entry) +} + +fn layout_member() -> Result<[u8; 96], Box> { + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + let mut entry = [0_u8; 96]; + *entry.first_mut().ok_or("closure member has no kind byte")? = 2; + entry + .get_mut(4..64) + .ok_or("closure member lacks layout identity")? + .copy_from_slice(&layout.encode_binary()); + entry + .get_mut(64..96) + .ok_or("closure member lacks checksum")? + .copy_from_slice(&digest(LAYOUT_RECORD_CHECKSUM_HEX)?); + Ok(entry) +} + +fn digest(hex: &str) -> Result<[u8; 32], Box> { + decode_hex(hex)? + .try_into() + .map_err(|_source| "digest fixture is not 32 bytes".into()) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 96241455b0f5305b055792a3ebafa2efaa071c1b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:25:55 -0700 Subject: [PATCH 15/50] Fix: Decouple repeated closure entries from nodes --- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention/closure_error.rs | 16 +- .../retention/closure_error_display.rs | 10 - src/adapters/retention/closure_verifier.rs | 19 +- tests/retention_closure.rs | 2 + tests/retention_closure/repeated_chunk_law.rs | 180 ++++++++++++++++++ 6 files changed, 187 insertions(+), 42 deletions(-) create mode 100644 tests/retention_closure/repeated_chunk_law.rs diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index fa09482..ccd9c81 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md` and one-anchor success in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor and repeated-chunk laws in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/src/adapters/retention/closure_error.rs b/src/adapters/retention/closure_error.rs index 34a66d4..11b1fde 100644 --- a/src/adapters/retention/closure_error.rs +++ b/src/adapters/retention/closure_error.rs @@ -4,8 +4,8 @@ use std::error::Error; use std::fmt; use crate::{ - BlobHashError, BlobId, ChunkingError, LayoutDecodeError, LayoutEntryLimitError, LayoutId, - ProfileBoundary, RetentionClosureCounter, SegmentRecordIdentity, StorageProfileId, + BlobHashError, BlobId, ChunkingError, LayoutDecodeError, LayoutId, ProfileBoundary, + RetentionClosureCounter, SegmentRecordIdentity, StorageProfileId, }; /// Failure to derive and authenticate one complete retained root closure. @@ -29,16 +29,6 @@ pub enum RetentionClosureVerificationError { /// Candidate observed value. observed: u64, }, - /// The admitted node limit could not become a host-independent entry cap. - LayoutEntryLimitHostWidth { - /// Admitted node limit that did not fit the layout cap width. - observed: u64, - }, - /// The derived layout entry cap violated the layout protocol bound. - LayoutEntryLimit { - /// Exact layout-bound refusal. - source: LayoutEntryLimitError, - }, /// The pinned catalog omits a first-scheduled closure member. MissingMember { /// Exact missing logical record identity. @@ -106,13 +96,11 @@ pub enum RetentionClosureVerificationError { impl Error for RetentionClosureVerificationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::LayoutEntryLimit { source } => Some(source), Self::LayoutDecode { source, .. } => Some(source), Self::ProfileChunking { source, .. } => Some(source), Self::BlobHash { source, .. } => Some(source), Self::CounterOverflow { .. } | Self::LimitExceeded { .. } - | Self::LayoutEntryLimitHostWidth { .. } | Self::MissingMember { .. } | Self::AnchorTargetMismatch { .. } | Self::ProfileVerifierUnavailable { .. } diff --git a/src/adapters/retention/closure_error_display.rs b/src/adapters/retention/closure_error_display.rs index d36d38a..0aa8672 100644 --- a/src/adapters/retention/closure_error_display.rs +++ b/src/adapters/retention/closure_error_display.rs @@ -25,16 +25,6 @@ pub(super) fn display( formatter, "{counter} limit {maximum} was exceeded by observed value {observed}" ), - RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } => write!( - formatter, - "closure node limit {observed} does not fit the layout entry-limit width" - ), - RetentionClosureVerificationError::LayoutEntryLimit { source } => { - write!( - formatter, - "closure-derived layout entry limit is invalid: {source}" - ) - } RetentionClosureVerificationError::MissingMember { identity } => match identity { crate::SegmentRecordIdentity::Chunk(chunk) => write!( formatter, diff --git a/src/adapters/retention/closure_verifier.rs b/src/adapters/retention/closure_verifier.rs index 30d9dc1..31e93f7 100644 --- a/src/adapters/retention/closure_verifier.rs +++ b/src/adapters/retention/closure_verifier.rs @@ -31,8 +31,7 @@ pub fn verify_retention_closure( root: &RetentionRoot, catalog: &CatalogSnapshot<'_, '_, '_>, ) -> Result { - let entry_limit = layout_entry_limit(root)?; - let mut verifier = ClosureVerifier::new(root, catalog, entry_limit); + let mut verifier = ClosureVerifier::new(root, catalog); for anchor in root.anchors().iter().copied() { verifier.verify_anchor(anchor)?; } @@ -42,7 +41,6 @@ pub fn verify_retention_closure( struct ClosureVerifier<'snapshot, 'head, 'catalog, 'records> { root: &'snapshot RetentionRoot, catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, - entry_limit: LayoutEntryLimit, accounting: ClosureAccounting, records: BTreeMap>, } @@ -51,12 +49,10 @@ impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'ca const fn new( root: &'snapshot RetentionRoot, catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, - entry_limit: LayoutEntryLimit, ) -> Self { Self { root, catalog, - entry_limit, accounting: ClosureAccounting::new(root.limits()), records: BTreeMap::new(), } @@ -75,7 +71,7 @@ impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'ca self.accounting .add_encoded(record.header().payload_length().get())?; } - let policy = LayoutDecodePolicy::new(self.entry_limit).with_expected_id(layout_id); + let policy = LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM).with_expected_id(layout_id); let layout = AdmittedLayout::decode_record(record.payload(), policy).map_err(|source| { RetentionClosureVerificationError::LayoutDecode { layout: layout_id, @@ -154,17 +150,6 @@ impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'ca } } -fn layout_entry_limit( - root: &RetentionRoot, -) -> Result { - let observed = root.limits().nodes(); - let value = u32::try_from(observed).map_err(|_source| { - RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } - })?; - LayoutEntryLimit::new(value) - .map_err(|source| RetentionClosureVerificationError::LayoutEntryLimit { source }) -} - fn require_anchor_target( anchor: RetentionAnchor, layout: &AdmittedLayout, diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs index 4a6bf37..ee6dd77 100644 --- a/tests/retention_closure.rs +++ b/tests/retention_closure.rs @@ -1,5 +1,7 @@ //! Deterministic retention-closure verification laws. +#[path = "retention_closure/repeated_chunk_law.rs"] +mod repeated_chunk_law; mod support; use std::error::Error; diff --git a/tests/retention_closure/repeated_chunk_law.rs b/tests/retention_closure/repeated_chunk_law.rs new file mode 100644 index 0000000..7c0985c --- /dev/null +++ b/tests/retention_closure/repeated_chunk_law.rs @@ -0,0 +1,180 @@ +//! Repeated logical chunk accounting law. + +use std::cell::RefCell; +use std::error::Error; +use std::io::{self, Write}; +use std::rc::Rc; + +use keep::{ + AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, + CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, FastCdc, + LayoutEntryLimit, RegisteredRetentionProfile, RegisteredStorageProfile, RetentionAnchor, + RetentionClosureLimits, RetentionNamespace, RetentionPolicy, RetentionRoot, RootGeneration, + SegmentReadPolicy, SegmentRecordLimit, SegmentStage, StagedSegment, verify_retention_closure, +}; + +const REPETITIONS: usize = 3; +const RECORD_OVERHEAD: u64 = 144; + +#[test] +fn repeated_chunk_occurrences_consume_physical_bytes_not_unique_nodes() -> Result<(), Box> +{ + let repeated = repeated_source()?; + let source = repeated.bytes; + let spans = repeated.spans; + let blob = BlobId::hash_bytes(&source)?; + let layout = AdmittedLayout::from_spans( + blob, + RegisteredStorageProfile::FAST_CDC_64K_V1, + spans, + LayoutEntryLimit::MAXIMUM, + )?; + let canonical_layout = layout.encode_record()?; + let chunk_length = source + .len() + .checked_div(REPETITIONS) + .ok_or("repetition count is zero")?; + let chunk = source + .get(..chunk_length) + .ok_or("repeated source omits its first chunk")?; + let (stage, probe) = MemoryStage::new(); + let staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; + let staged = staged.append(AdmittedSegmentRecord::for_chunk(chunk)?)?; + let staged = staged.append(AdmittedSegmentRecord::for_layout(&canonical_layout)?)?; + let _sealed = staged.seal()?; + let segment_bytes = probe.bytes(); + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let canonical_catalog = + CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let canonical_head = CanonicalPublicationHead::for_catalog(canonical_catalog.checksummed()); + let catalog = canonical_catalog.checksummed().admit(&segments)?; + let head = ChecksummedPublicationHead::decode(canonical_head.encoded())?; + let snapshot = head.admit(catalog)?; + let encoded_bytes = u64::try_from(canonical_layout.bytes().len())?; + let chunk_bytes = u64::try_from(chunk.len())?; + let repetitions = u64::try_from(REPETITIONS)?; + let physical_bytes = RECORD_OVERHEAD + .checked_add(encoded_bytes) + .and_then(|layout_record| { + RECORD_OVERHEAD + .checked_add(chunk_bytes) + .and_then(|chunk_record| chunk_record.checked_mul(repetitions)) + .and_then(|chunks| layout_record.checked_add(chunks)) + }) + .ok_or("physical-byte oracle overflowed")?; + let root = RetentionRoot::new( + RetentionNamespace::try_from(b"repeated".as_slice())?, + RootGeneration::new(1)?, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(2, 2, encoded_bytes, physical_bytes)?, + ), + None, + vec![RetentionAnchor::new(blob, canonical_layout.id())], + )?; + + let evidence = verify_retention_closure(&root, &snapshot)?; + + assert_eq!(evidence.usage().node_count(), 2); + assert_eq!(evidence.usage().maximum_depth(), 2); + assert_eq!(evidence.usage().encoded_bytes(), encoded_bytes); + assert_eq!(evidence.usage().physical_bytes(), physical_bytes); + Ok(()) +} + +struct RepeatedSource { + bytes: Vec, + spans: Vec, +} + +fn repeated_source() -> Result> { + let candidate_length = usize::try_from( + RegisteredStorageProfile::FAST_CDC_64K_V1 + .maximum_chunk_length() + .get(), + )?; + let candidate = vec![0_u8; candidate_length]; + let first_pass = detect(&candidate)?; + let first = first_pass + .first() + .copied() + .ok_or("profile emitted no candidate chunk")?; + let chunk_length = usize::try_from(first.length().get())?; + let chunk = candidate + .get(..chunk_length) + .ok_or("candidate omits its first emitted chunk")?; + let source_length = chunk_length + .checked_mul(REPETITIONS) + .ok_or("repeated source length overflowed")?; + let mut source = Vec::new(); + source.try_reserve_exact(source_length)?; + for _index in 0..REPETITIONS { + source.extend_from_slice(chunk); + } + let spans = detect(&source)?; + if spans.len() != REPETITIONS || !spans.iter().all(|span| span.id() == first.id()) { + return Err("repeated source did not reproduce one exact chunk identity".into()); + } + Ok(RepeatedSource { + bytes: source, + spans, + }) +} + +fn detect(bytes: &[u8]) -> Result, Box> { + let mut spans = Vec::new(); + let mut detector = FastCdc::new(); + detector.feed(bytes, |span| spans.push(span))?; + if let Some(span) = detector.finish()? { + spans.push(span); + } + Ok(spans) +} + +struct MemoryStage { + bytes: Rc>>, +} + +struct MemoryProbe { + bytes: Rc>>, +} + +impl MemoryStage { + fn new() -> (Self, MemoryProbe) { + let bytes = Rc::new(RefCell::new(Vec::new())); + ( + Self { + bytes: Rc::clone(&bytes), + }, + MemoryProbe { bytes }, + ) + } +} + +impl Write for MemoryStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl MemoryProbe { + fn bytes(&self) -> Vec { + self.bytes.borrow().clone() + } +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} From 163cfa1a6227e1be8c494ff9b9c988f0476f8a9c Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:36:49 -0700 Subject: [PATCH 16/50] Test: Prove closure refusal precedence --- docs/formats/segment-store-v2/closure.md | 17 +++ docs/formats/segment-store-v2/requirements.md | 2 +- tests/retention_closure.rs | 8 ++ .../adversarial_catalog_laws.rs | 116 ++++++++++++++++++ .../limit_precedence_laws.rs | 64 ++++++++++ tests/retention_closure/memory_stage.rs | 49 ++++++++ tests/retention_closure/one_zero_bundle.rs | 63 ++++++++++ tests/retention_closure/repeated_chunk_law.rs | 61 ++------- 8 files changed, 326 insertions(+), 54 deletions(-) create mode 100644 tests/retention_closure/adversarial_catalog_laws.rs create mode 100644 tests/retention_closure/limit_precedence_laws.rs create mode 100644 tests/retention_closure/memory_stage.rs create mode 100644 tests/retention_closure/one_zero_bundle.rs diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 0f88d6c..599f4dc 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -169,6 +169,23 @@ The digest binds the exact profile, catalog, observed resource use, logical member set, and record checksums. The transition receipt binds it beside the root's separate anchor-set digest; neither digest substitutes for the other. +## Executable evidence + +- The [one-anchor closure law](../../../tests/retention_closure.rs) freezes the + exact counters, canonical member transcript, closure digest, and authenticated + reconstruction result. +- The + [repeated-chunk law](../../../tests/retention_closure/repeated_chunk_law.rs) + proves that logical reconstruction work and unique-node evidence remain + separate. +- The + [adversarial-catalog laws](../../../tests/retention_closure/adversarial_catalog_laws.rs) + prove exact missing-member refusal and target-mismatch precedence. +- The + [limit-precedence laws](../../../tests/retention_closure/limit_precedence_laws.rs) + prove the documented depth, node, physical-byte, and encoded-byte refusal + order. + ## Evidence and nonclaims Successful evidence records the closure digest, all four observed counters, diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index ccd9c81..708271a 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor and repeated-chunk laws in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, and adversarial-catalog laws in `tests/retention_closure.rs`; property and corruption tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs index ee6dd77..7510d11 100644 --- a/tests/retention_closure.rs +++ b/tests/retention_closure.rs @@ -1,5 +1,13 @@ //! Deterministic retention-closure verification laws. +#[path = "retention_closure/adversarial_catalog_laws.rs"] +mod adversarial_catalog_laws; +#[path = "retention_closure/limit_precedence_laws.rs"] +mod limit_precedence_laws; +#[path = "retention_closure/memory_stage.rs"] +mod memory_stage; +#[path = "retention_closure/one_zero_bundle.rs"] +mod one_zero_bundle; #[path = "retention_closure/repeated_chunk_law.rs"] mod repeated_chunk_law; mod support; diff --git a/tests/retention_closure/adversarial_catalog_laws.rs b/tests/retention_closure/adversarial_catalog_laws.rs new file mode 100644 index 0000000..9f89dc8 --- /dev/null +++ b/tests/retention_closure/adversarial_catalog_laws.rs @@ -0,0 +1,116 @@ +//! Adversarial catalog and first-refusal ordering laws. + +use std::error::Error; + +use keep::{ + AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, + CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, LayoutDecodePolicy, + LayoutEntryLimit, RetentionClosureLimits, RetentionClosureVerificationError, RetentionRoot, + SegmentRecordIdentity, VerifiedRetentionClosure, verify_retention_closure, +}; + +use super::{ + ONE_ZERO_BLOB, maximum_policy, + memory_stage::segment_bytes, + one_zero_bundle::{root_with_limits, verify_fixture}, + support::{layout_record_bytes, require_error}, +}; + +const CHUNK_CATALOG_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-catalog.hex"); +const CHUNK_HEAD_HEX: &str = include_str!("../../conformance/segment-store/v1/one-zero-head.hex"); +const CHUNK_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); + +#[test] +fn missing_layout_is_an_exact_first_scheduled_member_refusal() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 509)?, None)?; + let error = require_error( + verify_fixture(&root, CHUNK_SEGMENT_HEX, CHUNK_CATALOG_HEX, CHUNK_HEAD_HEX)?, + "chunk-only catalog unexpectedly satisfied a retained layout", + )?; + let expected = root + .anchors() + .first() + .copied() + .ok_or("retention root omits its required anchor")? + .layout_id(); + + assert!(matches!( + error, + RetentionClosureVerificationError::MissingMember { + identity: SegmentRecordIdentity::Layout(layout) + } if layout == expected + )); + Ok(()) +} + +#[test] +fn missing_chunk_is_an_exact_logical_occurrence_refusal() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 509)?, None)?; + let error = require_error( + verify_layout_only(&root)?, + "layout-only catalog unexpectedly reconstructed a retained blob", + )?; + let expected = expected_chunk_identity()?; + + assert!(matches!( + error, + RetentionClosureVerificationError::MissingMember { identity } if identity == expected + )); + Ok(()) +} + +#[test] +fn anchor_target_mismatch_precedes_chunk_traversal() -> Result<(), Box> { + let expected = BlobId::hash_bytes(b"adversarial anchor")?; + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 509)?, Some(expected))?; + let error = require_error( + verify_layout_only(&root)?, + "mismatched anchor target unexpectedly verified", + )?; + let observed: BlobId = ONE_ZERO_BLOB.parse()?; + + assert!(matches!( + error, + RetentionClosureVerificationError::AnchorTargetMismatch { + expected: actual_expected, + observed: actual_observed, + .. + } if actual_expected == expected && actual_observed == observed + )); + Ok(()) +} + +fn verify_layout_only( + root: &RetentionRoot, +) -> Result, Box> { + let layout = one_zero_layout()?; + let canonical_layout = layout.encode_record()?; + let records = [AdmittedSegmentRecord::for_layout(&canonical_layout)?]; + let segment_bytes = segment_bytes(&records)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let head = CanonicalPublicationHead::for_catalog(catalog.checksummed()); + let admitted = catalog.checksummed().admit(&segments)?; + let snapshot = ChecksummedPublicationHead::decode(head.encoded())?.admit(admitted)?; + Ok(verify_retention_closure(root, &snapshot)) +} + +fn expected_chunk_identity() -> Result> { + let layout = one_zero_layout()?; + let entry = layout + .entries() + .first() + .copied() + .ok_or("one-zero layout omits its chunk entry")?; + Ok(SegmentRecordIdentity::Chunk(entry.chunk_id())) +} + +fn one_zero_layout() -> Result> { + Ok(AdmittedLayout::decode_record( + &layout_record_bytes("one-zero")?, + LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM), + )?) +} diff --git a/tests/retention_closure/limit_precedence_laws.rs b/tests/retention_closure/limit_precedence_laws.rs new file mode 100644 index 0000000..e11ee93 --- /dev/null +++ b/tests/retention_closure/limit_precedence_laws.rs @@ -0,0 +1,64 @@ +//! Closure resource-limit precedence laws. + +use std::error::Error; + +use keep::{ + RetentionClosureCounter, RetentionClosureLimits, RetentionClosureVerificationError, + RetentionRoot, +}; + +use super::{ + one_zero_bundle::{root_with_limits, verify_bundle}, + support::require_error, +}; + +#[test] +fn depth_refusal_precedes_second_node_admission() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 1, 220, 509)?, None)?; + + assert_limit(&root, RetentionClosureCounter::Depth, 1, 2) +} + +#[test] +fn node_refusal_follows_depth_admission_and_precedes_chunk_lookup() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(1, 2, 220, 509)?, None)?; + + assert_limit(&root, RetentionClosureCounter::Nodes, 1, 2) +} + +#[test] +fn physical_byte_refusal_precedes_layout_decoding() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 363)?, None)?; + + assert_limit(&root, RetentionClosureCounter::PhysicalBytes, 363, 364) +} + +#[test] +fn encoded_byte_refusal_follows_layout_record_charge() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 219, 509)?, None)?; + + assert_limit(&root, RetentionClosureCounter::EncodedBytes, 219, 220) +} + +fn assert_limit( + root: &RetentionRoot, + counter: RetentionClosureCounter, + maximum: u64, + observed: u64, +) -> Result<(), Box> { + let error = require_error( + verify_bundle(root)?, + "resource-constrained closure unexpectedly verified", + )?; + assert!(matches!( + error, + RetentionClosureVerificationError::LimitExceeded { + counter: actual_counter, + maximum: actual_maximum, + observed: actual_observed, + } if actual_counter == counter + && actual_maximum == maximum + && actual_observed == observed + )); + Ok(()) +} diff --git a/tests/retention_closure/memory_stage.rs b/tests/retention_closure/memory_stage.rs new file mode 100644 index 0000000..e0bde89 --- /dev/null +++ b/tests/retention_closure/memory_stage.rs @@ -0,0 +1,49 @@ +//! In-memory segment-stage support for closure integration laws. +#![allow( + clippy::redundant_pub_crate, + reason = "private integration-test siblings share this segment fixture" +)] + +use std::cell::RefCell; +use std::io::{self, Write}; +use std::rc::Rc; + +use keep::{ + AdmittedSegmentRecord, SegmentRecordLimit, SegmentStage, SegmentWriteError, StagedSegment, +}; + +pub(super) fn segment_bytes( + records: &[AdmittedSegmentRecord<'_>], +) -> Result, SegmentWriteError> { + let bytes = Rc::new(RefCell::new(Vec::new())); + let stage = MemoryStage { + bytes: Rc::clone(&bytes), + }; + let mut staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; + for record in records { + staged = staged.append(*record)?; + } + let _sealed = staged.seal()?; + Ok(bytes.borrow().clone()) +} + +struct MemoryStage { + bytes: Rc>>, +} + +impl Write for MemoryStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/tests/retention_closure/one_zero_bundle.rs b/tests/retention_closure/one_zero_bundle.rs new file mode 100644 index 0000000..c6267ca --- /dev/null +++ b/tests/retention_closure/one_zero_bundle.rs @@ -0,0 +1,63 @@ +//! One-zero closure fixture construction and verification. +#![allow( + clippy::redundant_pub_crate, + reason = "private integration-test siblings share this closure fixture" +)] + +use std::error::Error; + +use keep::{ + BlobId, RetentionAnchor, RetentionClosureLimits, RetentionClosureVerificationError, + RetentionNamespace, RetentionPolicy, RetentionRoot, RootGeneration, VerifiedRetentionClosure, + verify_retention_closure, +}; + +use super::{ + BUNDLE_CATALOG_HEX, BUNDLE_HEAD_HEX, BUNDLE_SEGMENT_HEX, ONE_ZERO_BLOB, ONE_ZERO_LAYOUT, + admitted_catalog, fixture, maximum_policy, +}; + +pub(super) fn root_with_limits( + limits: RetentionClosureLimits, + target: Option, +) -> Result> { + let blob = target.map_or_else(|| ONE_ZERO_BLOB.parse(), Ok)?; + Ok(RetentionRoot::new( + RetentionNamespace::try_from(b"adversarial".as_slice())?, + RootGeneration::new(1)?, + RetentionPolicy::new( + keep::RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + limits, + ), + None, + vec![RetentionAnchor::new(blob, ONE_ZERO_LAYOUT.parse()?)], + )?) +} + +pub(super) fn verify_bundle( + root: &RetentionRoot, +) -> Result, Box> { + verify_fixture( + root, + BUNDLE_SEGMENT_HEX, + BUNDLE_CATALOG_HEX, + BUNDLE_HEAD_HEX, + ) +} + +pub(super) fn verify_fixture( + root: &RetentionRoot, + segment_hex: &str, + catalog_hex: &str, + head_hex: &str, +) -> Result, Box> { + let segment_bytes = fixture(segment_hex)?; + let catalog_bytes = fixture(catalog_hex)?; + let head_bytes = fixture(head_hex)?; + let segment = keep::AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = keep::ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(verify_retention_closure(root, &snapshot)) +} diff --git a/tests/retention_closure/repeated_chunk_law.rs b/tests/retention_closure/repeated_chunk_law.rs index 7c0985c..ba06dc1 100644 --- a/tests/retention_closure/repeated_chunk_law.rs +++ b/tests/retention_closure/repeated_chunk_law.rs @@ -1,18 +1,17 @@ //! Repeated logical chunk accounting law. -use std::cell::RefCell; use std::error::Error; -use std::io::{self, Write}; -use std::rc::Rc; use keep::{ AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, FastCdc, LayoutEntryLimit, RegisteredRetentionProfile, RegisteredStorageProfile, RetentionAnchor, RetentionClosureLimits, RetentionNamespace, RetentionPolicy, RetentionRoot, RootGeneration, - SegmentReadPolicy, SegmentRecordLimit, SegmentStage, StagedSegment, verify_retention_closure, + SegmentReadPolicy, SegmentRecordLimit, verify_retention_closure, }; +use super::memory_stage::segment_bytes; + const REPETITIONS: usize = 3; const RECORD_OVERHEAD: u64 = 144; @@ -37,12 +36,11 @@ fn repeated_chunk_occurrences_consume_physical_bytes_not_unique_nodes() -> Resul let chunk = source .get(..chunk_length) .ok_or("repeated source omits its first chunk")?; - let (stage, probe) = MemoryStage::new(); - let staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; - let staged = staged.append(AdmittedSegmentRecord::for_chunk(chunk)?)?; - let staged = staged.append(AdmittedSegmentRecord::for_layout(&canonical_layout)?)?; - let _sealed = staged.seal()?; - let segment_bytes = probe.bytes(); + let records = [ + AdmittedSegmentRecord::for_chunk(chunk)?, + AdmittedSegmentRecord::for_layout(&canonical_layout)?, + ]; + let segment_bytes = segment_bytes(&records)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; let segments = [segment]; let canonical_catalog = @@ -132,49 +130,6 @@ fn detect(bytes: &[u8]) -> Result, Box> { Ok(spans) } -struct MemoryStage { - bytes: Rc>>, -} - -struct MemoryProbe { - bytes: Rc>>, -} - -impl MemoryStage { - fn new() -> (Self, MemoryProbe) { - let bytes = Rc::new(RefCell::new(Vec::new())); - ( - Self { - bytes: Rc::clone(&bytes), - }, - MemoryProbe { bytes }, - ) - } -} - -impl Write for MemoryStage { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.bytes.borrow_mut().extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl SegmentStage for MemoryStage { - fn synchronize(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl MemoryProbe { - fn bytes(&self) -> Vec { - self.bytes.borrow().clone() - } -} - const fn maximum_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } From d9b1771849ba35ba49b2f4847fc5f02bd27abba4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:40:51 -0700 Subject: [PATCH 17/50] Test: Model closure resource boundaries --- docs/formats/segment-store-v2/closure.md | 28 ++--- docs/formats/segment-store-v2/requirements.md | 2 +- tests/retention_closure.rs | 2 + tests/retention_closure/closure_model_laws.rs | 102 ++++++++++++++++++ 4 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 tests/retention_closure/closure_model_laws.rs diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 599f4dc..b4bb152 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -171,20 +171,20 @@ root's separate anchor-set digest; neither digest substitutes for the other. ## Executable evidence -- The [one-anchor closure law](../../../tests/retention_closure.rs) freezes the - exact counters, canonical member transcript, closure digest, and authenticated - reconstruction result. -- The - [repeated-chunk law](../../../tests/retention_closure/repeated_chunk_law.rs) - proves that logical reconstruction work and unique-node evidence remain - separate. -- The - [adversarial-catalog laws](../../../tests/retention_closure/adversarial_catalog_laws.rs) - prove exact missing-member refusal and target-mismatch precedence. -- The - [limit-precedence laws](../../../tests/retention_closure/limit_precedence_laws.rs) - prove the documented depth, node, physical-byte, and encoded-byte refusal - order. +- The [one-anchor law](../../../tests/retention_closure.rs) freezes exact + counters, the member transcript, the digest, and reconstruction. +- The [repeated-chunk + law](../../../tests/retention_closure/repeated_chunk_law.rs) separates + reconstruction work from unique-node evidence. +- The [adversarial-catalog + laws](../../../tests/retention_closure/adversarial_catalog_laws.rs) prove exact + missing-member and target-mismatch precedence. +- The [limit-precedence + laws](../../../tests/retention_closure/limit_precedence_laws.rs) prove the + depth, node, physical-byte, and encoded-byte refusal order. +- The [closure model + laws](../../../tests/retention_closure/closure_model_laws.rs) compare all + `3 × 3 × 3 × 5 = 135` one-zero boundary policies with a boring model. ## Evidence and nonclaims diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 708271a..5dda56e 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, and adversarial-catalog laws in `tests/retention_closure.rs`; property and corruption tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corruption tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs index 7510d11..0ed1b3d 100644 --- a/tests/retention_closure.rs +++ b/tests/retention_closure.rs @@ -2,6 +2,8 @@ #[path = "retention_closure/adversarial_catalog_laws.rs"] mod adversarial_catalog_laws; +#[path = "retention_closure/closure_model_laws.rs"] +mod closure_model_laws; #[path = "retention_closure/limit_precedence_laws.rs"] mod limit_precedence_laws; #[path = "retention_closure/memory_stage.rs"] diff --git a/tests/retention_closure/closure_model_laws.rs b/tests/retention_closure/closure_model_laws.rs new file mode 100644 index 0000000..12fc915 --- /dev/null +++ b/tests/retention_closure/closure_model_laws.rs @@ -0,0 +1,102 @@ +//! Exhaustive closure-accounting outcomes against a boring model. + +use std::error::Error; + +use keep::{RetentionClosureCounter, RetentionClosureLimits, RetentionClosureVerificationError}; + +use super::one_zero_bundle::{root_with_limits, verify_bundle}; + +const NODE_LIMITS: [u64; 3] = [1, 2, 3]; +const DEPTH_LIMITS: [u16; 3] = [1, 2, 3]; +const ENCODED_LIMITS: [u64; 3] = [219, 220, 221]; +const PHYSICAL_LIMITS: [u64; 5] = [363, 364, 508, 509, 510]; + +#[test] +fn exhaustive_boundary_policies_agree_with_the_boring_model() -> Result<(), Box> { + for nodes in NODE_LIMITS { + for depth in DEPTH_LIMITS { + for encoded in ENCODED_LIMITS { + for physical in PHYSICAL_LIMITS { + let limits = RetentionClosureLimits::new(nodes, depth, encoded, physical)?; + let root = root_with_limits(limits, None)?; + let observed = classify(verify_bundle(&root)?)?; + let expected = model(nodes, depth, encoded, physical); + + assert_eq!( + observed, expected, + "nodes={nodes} depth={depth} encoded={encoded} physical={physical}" + ); + } + } + } + } + Ok(()) +} + +fn model(nodes: u64, depth: u16, encoded: u64, physical: u64) -> Outcome { + if physical < 364 { + return Outcome::limit(RetentionClosureCounter::PhysicalBytes, physical, 364); + } + if encoded < 220 { + return Outcome::limit(RetentionClosureCounter::EncodedBytes, encoded, 220); + } + if depth < 2 { + return Outcome::limit(RetentionClosureCounter::Depth, u64::from(depth), 2); + } + if nodes < 2 { + return Outcome::limit(RetentionClosureCounter::Nodes, nodes, 2); + } + if physical < 509 { + return Outcome::limit(RetentionClosureCounter::PhysicalBytes, physical, 509); + } + Outcome::Verified { + nodes: 2, + depth: 2, + encoded: 220, + physical: 509, + } +} + +fn classify( + result: Result, +) -> Result> { + match result { + Ok(evidence) => Ok(Outcome::Verified { + nodes: evidence.usage().node_count(), + depth: evidence.usage().maximum_depth(), + encoded: evidence.usage().encoded_bytes(), + physical: evidence.usage().physical_bytes(), + }), + Err(RetentionClosureVerificationError::LimitExceeded { + counter, + maximum, + observed, + }) => Ok(Outcome::limit(counter, maximum, observed)), + Err(error) => Err(error.into()), + } +} + +#[derive(Debug, Eq, PartialEq)] +enum Outcome { + Limit { + counter: RetentionClosureCounter, + maximum: u64, + observed: u64, + }, + Verified { + nodes: u64, + depth: u16, + encoded: u64, + physical: u64, + }, +} + +impl Outcome { + const fn limit(counter: RetentionClosureCounter, maximum: u64, observed: u64) -> Self { + Self::Limit { + counter, + maximum, + observed, + } + } +} From e44bfd78a4d1e4ce40fba063573fd0c945e11b3d Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:47:42 -0700 Subject: [PATCH 18/50] Docs: Define closure corruption boundary --- docs/formats/segment-store-v2/README.md | 2 + .../segment-store-v2/closure-corruption.md | 69 +++++++++++ docs/formats/segment-store-v2/closure.md | 8 +- docs/formats/segment-store-v2/requirements.md | 2 +- .../retention_store_v2_protocol_contract.rs | 111 ++---------------- .../closure_contract_laws.rs | 42 +++++++ .../migration_contract_laws.rs | 76 ++++++++++++ 7 files changed, 201 insertions(+), 109 deletions(-) create mode 100644 docs/formats/segment-store-v2/closure-corruption.md create mode 100644 xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs create mode 100644 xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 8ba96bf..103e379 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -42,6 +42,8 @@ The following pages form one protocol: root-generation, manifest, retention-head, and transition rules. - [Closure verification](closure.md) owns deterministic traversal, exact resource accounting, authenticated reconstruction, and closure evidence. +- [Closure corruption boundary](closure-corruption.md) owns the admitted-record + ingress proof and its exact refusal evidence. - [GC and disposition records](gc.md) owns the canonical planned intent, completion, and recovery-disposition byte grammars. - [Migration and recovery](recovery.md) owns the exact root namespace, diff --git a/docs/formats/segment-store-v2/closure-corruption.md b/docs/formats/segment-store-v2/closure-corruption.md new file mode 100644 index 0000000..8e31924 --- /dev/null +++ b/docs/formats/segment-store-v2/closure-corruption.md @@ -0,0 +1,69 @@ +# Closure Corruption Boundary + +- Status: Normative version-2 protocol; executable ingress evidence implemented +- Format coordinate: `keep.segment-store/v2` +- Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) +- Parent contract: [Closure verification](closure.md) + +This page defines where corrupt closure-member bytes refuse. Its primary job is +to keep untrusted byte admission separate from deterministic closure traversal. + +## Trust boundary + +`verify_retention_closure` accepts a validated `RetentionRoot` and an immutable +`CatalogSnapshot`. It does not accept untrusted bytes, raw segment records, +paths, readers, or caller lookup callbacks. + +A record reaches that snapshot only through this proof chain: + +1. `ChecksummedSegmentRecord::decode` admits exact framing and its checksum. +2. `ChecksummedSegmentRecord::admit` recomputes the chunk or layout identity + from the payload and returns an `AdmittedSegmentRecord`. +3. `AdmittedSegment::decode` admits every complete nested record and the + segment seal and digest. +4. `ChecksummedCatalog::admit` binds each catalog entry to the exact admitted + record identity, checksum, and top-level location. +5. `ChecksummedPublicationHead::admit` binds the admitted catalog's generation, + length, and digest into a `CatalogSnapshot`. + +Failure at any step makes the next type unconstructible through the public API. +Closure verification therefore has no corruption fallback and never +reinterprets a malformed record as a missing member. + +## Exact refusal ownership + +The inherited version-1 boundaries retain their typed errors: + +- malformed record framing or checksum returns `SegmentRecordDecodeError`; +- chunk payload identity disagreement or malformed layout payload returns + `SegmentRecordAdmissionError`; +- complete-segment corruption returns `SegmentReadError`; +- catalog location, identity, checksum, or segment disagreement returns + `CatalogAdmissionError`; and +- publication-head disagreement returns `CatalogSnapshotError`. + +`RetentionClosureVerificationError::MissingMember` means the pinned, +fully admitted catalog has no binding for the scheduled logical identity. It +does not mean bytes were present but corrupt. + +## Executable evidence + +- The [segment-record framing + laws](../../../tests/segment_record/framing_laws.rs) cover checksum and + framing corruption. +- The [segment-record admission + laws](../../../tests/segment_record/admission_laws.rs) cover content-valid + checksums whose chunk or layout payload does not match its declared identity. +- The [segment corruption-localization + laws](../../../tests/segment/identity_laws.rs) prove record refusal precedes + the outer segment digest. +- The [`segment_format` fuzz + target](../../../fuzz/fuzz_targets/segment_format.rs) reaches record decoding, + record admission, and complete-segment admission from deterministic canonical + seeds owned by the [Rust seed-corpus + task](../../../xtask/src/fuzz_seed_corpus/segment_seeds.rs). + +These proofs establish ingress safety. They do not prove that a future +retention publication adapter preserves the original source chain when it maps +these failures into an operation-level error; that obligation remains with +`KEEP-RETENTION-006`. diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index b4bb152..6e1f0e1 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -28,10 +28,10 @@ read paths, enumerate a filesystem, consult a clock, invoke a caller callback, or replace a missing witness. Version 2 selects the single record bound to each logical identity by the pinned catalog. -The catalog has already admitted each bound segment record's framing, checksum, -logical identity, and payload. Closure verification consumes those proofs, -decodes layouts again under the closure budget, and authenticates each complete -logical blob. +The [corruption boundary](closure-corruption.md) defines how each bound record +earns framing, checksum, logical-identity, and payload proofs before it enters a +`CatalogSnapshot`. Closure verification consumes those proofs, decodes layouts +under the closure budget, and authenticates each complete logical blob. ## Deterministic traversal diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 5dda56e..62419a8 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corruption tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index d413b2c..d2dc2ae 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -2,6 +2,11 @@ #![cfg(feature = "repository-tasks")] +#[path = "retention_store_v2_protocol_contract/closure_contract_laws.rs"] +mod closure_contract_laws; +#[path = "retention_store_v2_protocol_contract/migration_contract_laws.rs"] +mod migration_contract_laws; + use std::fs; use std::io; use std::path::{Path, PathBuf}; @@ -43,6 +48,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "successor to `keep.segment-store/v1`", "[Retention records and publication](retention.md)", "[Closure verification](closure.md)", + "[Closure corruption boundary](closure-corruption.md)", "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", "[Migration crash points](migration-crash.md)", @@ -94,110 +100,6 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box> { - let closure = normalized(&read(&format!("{FORMAT_ROOT}/closure.md"))?); - - for required in [ - "one pinned, completely verified catalog generation", - "first scheduled", - "anchor is not a closure node", - "unique `SegmentRecordIdentity`", - "depth `1`", - "depth `2`", - "canonical layout payload length", - "complete segment-record length", - "checked addition before", - "repeated logical occurrence", - "replay the exact registered storage profile", - "authenticate the complete `BlobId`", - "keep.retention-closure/v2\\0", - "96-byte closure-member entries", - "canonical typed-identity order", - "Missing members still consume", - ] { - assert!( - closure.contains(required), - "segment-store v2 closure contract omits `{required}`" - ); - } - Ok(()) -} - -#[test] -fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> -{ - let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); - - for required in [ - "one-way explicit migration", - "`migration.intent`", - "`migration.intent.next`", - "`migration.receipt`", - "`migration.receipt.next`", - "`FORMAT.next`", - "`migration.intent` is exactly 256 bytes", - "`migration.receipt` is exactly 256 bytes", - "catalog generation, length, and digest", - "`definition.tsv`", - "migration inventory entry is exactly 56 bytes", - "2,097,152", - "keep.store-migration-intent/v2\\0", - "keep.store-format-marker/v2\\0", - "deterministically derived store identifier", - "absence of `retention/HEAD` is the canonical empty retention state", - "pre-effect incomplete stage", - "keep.initial-retention-state/v2\\0", - "keep.initial-gc-state/v2\\0", - "keep.empty-disposition-set/v2\\0", - "root.next` is durable before a new namespace directory", - "`KEEP-CRASH-036`", - "`KEEP-CRASH-073`", - "partial migration", - "Version-1 admission refuses", - "`reader.lock`", - "`GcRetirementIntent`", - "`GcRetirementReceipt`", - "`RecoveryDispositionReceipt`", - "unknown entry", - "unrecoverable ambiguity", - "idempotent", - "process death", - ] { - assert!( - recovery.contains(required), - "segment-store v2 recovery contract omits `{required}`" - ); - } - Ok(()) -} - -#[test] -fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> -{ - let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); - - for required in [ - "never writes canonical fixed names in place", - "`migration.intent.next`", - "`FORMAT.next`", - "`migration.receipt.next`", - "linked without replacement", - "pre-effect incomplete stage", - "`KEEP-CRASH-053`", - "`KEEP-CRASH-073`", - "`0x00000000000003ff`", - "before, during, and after process-death evidence", - ] { - assert!( - migration.contains(required), - "segment-store v2 migration crash protocol omits `{required}`" - ); - } - Ok(()) -} - #[test] fn gc_records_are_bounded_before_their_implementation() -> Result<(), Box> { let gc = normalized(&read(&format!("{FORMAT_ROOT}/gc.md"))?); @@ -253,6 +155,7 @@ fn requirement_ledger_names_planned_and_executable_evidence() fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box> { for name in [ "README.md", + "closure-corruption.md", "closure.md", "gc.md", "migration-crash.md", diff --git a/xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs new file mode 100644 index 0000000..34fe451 --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs @@ -0,0 +1,42 @@ +//! Closure verification and corruption-boundary contract laws. + +use super::{FORMAT_ROOT, normalized, read}; + +#[test] +fn closure_accounting_has_exact_units_and_canonical_evidence() +-> Result<(), Box> { + let closure = format!( + "{} {}", + normalized(&read(&format!("{FORMAT_ROOT}/closure.md"))?), + normalized(&read(&format!("{FORMAT_ROOT}/closure-corruption.md"))?) + ); + + for required in [ + "one pinned, completely verified catalog generation", + "first scheduled", + "anchor is not a closure node", + "unique `SegmentRecordIdentity`", + "depth `1`", + "depth `2`", + "canonical layout payload length", + "complete segment-record length", + "checked addition before", + "repeated logical occurrence", + "replay the exact registered storage profile", + "authenticate the complete `BlobId`", + "keep.retention-closure/v2\\0", + "96-byte closure-member entries", + "canonical typed-identity order", + "Missing members still consume", + "`CatalogSnapshot`", + "does not accept untrusted bytes", + "segment-record admission laws", + "`segment_format` fuzz target", + ] { + assert!( + closure.contains(required), + "segment-store v2 closure contract omits `{required}`" + ); + } + Ok(()) +} diff --git a/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs new file mode 100644 index 0000000..1a479cf --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs @@ -0,0 +1,76 @@ +//! Migration and recovery written-contract laws. + +use super::{FORMAT_ROOT, normalized, read}; + +#[test] +fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> +{ + let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); + + for required in [ + "one-way explicit migration", + "`migration.intent`", + "`migration.intent.next`", + "`migration.receipt`", + "`migration.receipt.next`", + "`FORMAT.next`", + "`migration.intent` is exactly 256 bytes", + "`migration.receipt` is exactly 256 bytes", + "catalog generation, length, and digest", + "`definition.tsv`", + "migration inventory entry is exactly 56 bytes", + "2,097,152", + "keep.store-migration-intent/v2\\0", + "keep.store-format-marker/v2\\0", + "deterministically derived store identifier", + "absence of `retention/HEAD` is the canonical empty retention state", + "pre-effect incomplete stage", + "keep.initial-retention-state/v2\\0", + "keep.initial-gc-state/v2\\0", + "keep.empty-disposition-set/v2\\0", + "root.next` is durable before a new namespace directory", + "`KEEP-CRASH-036`", + "`KEEP-CRASH-073`", + "partial migration", + "Version-1 admission refuses", + "`reader.lock`", + "`GcRetirementIntent`", + "`GcRetirementReceipt`", + "`RecoveryDispositionReceipt`", + "unknown entry", + "unrecoverable ambiguity", + "idempotent", + "process death", + ] { + assert!( + recovery.contains(required), + "segment-store v2 recovery contract omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> +{ + let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); + + for required in [ + "never writes canonical fixed names in place", + "`migration.intent.next`", + "`FORMAT.next`", + "`migration.receipt.next`", + "linked without replacement", + "pre-effect incomplete stage", + "`KEEP-CRASH-053`", + "`KEEP-CRASH-073`", + "`0x00000000000003ff`", + "before, during, and after process-death evidence", + ] { + assert!( + migration.contains(required), + "segment-store v2 migration crash protocol omits `{required}`" + ); + } + Ok(()) +} From b82b17ca8b8cd8aab99e3ef88c897b57cdc4ac3b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:59:49 -0700 Subject: [PATCH 19/50] Add: Preflight retention transitions --- CHANGELOG.md | 3 + README.md | 9 +- docs/formats/segment-store-v2/README.md | 14 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 4 + src/adapters/retention.rs | 4 + .../retention/transition_preflight.rs | 80 ++++++++ .../retention/transition_preflight_error.rs | 39 ++++ src/lib.rs | 13 +- tests/retention_preflight.rs | 179 ++++++++++++++++++ 10 files changed, 329 insertions(+), 18 deletions(-) create mode 100644 src/adapters/retention/transition_preflight.rs create mode 100644 src/adapters/retention/transition_preflight_error.rs create mode 100644 tests/retention_preflight.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 74f5188..3ad25fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ after its public API and format compatibility policies are established. ### Changed +- Retention transition preflight now combines exact expected-generation + planning with deterministic closure verification against one pinned catalog + before any future publication storage call. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 7194bed..b1da614 100644 --- a/README.md +++ b/README.md @@ -117,10 +117,11 @@ exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state -transition planning; and deterministic bounded closure verification against a -pinned catalog are implemented. Publication, recovery, compaction, and garbage -collection remain planned. Presence in the reference CAS does not claim -retention, crash recovery, or durability. +transition planning; deterministic bounded closure verification against a +pinned catalog; and a combined transition preflight proof are implemented. +Publication, recovery, compaction, and garbage collection remain planned. +Presence in the reference CAS does not claim retention, crash recovery, or +durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 103e379..3c12a8e 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -75,10 +75,10 @@ types now admit exact namespace bytes, namespace digests, root and liveness generations, registered realization profiles, bounded closure policies, reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition -planning and deterministic bounded closure verification against one pinned -catalog are available. Production filesystem retention publication, recovery, -migration, and garbage collection do not exist yet. Requirements that remain -planned or in progress in issue #19 or issue #21 are not complete -implementation evidence. A store must refuse unsupported version-2 state until -the relevant corruption, model-based, crash-injection, recovery, and fuzz -evidence is implemented. +planning, deterministic bounded closure verification against one pinned +catalog, and their combined preflight proof are available. Production +filesystem retention publication, recovery, migration, and garbage collection +do not exist yet. Requirements that remain planned or in progress in issue #19 +or issue #21 are not complete implementation evidence. A store must refuse +unsupported version-2 state until the relevant corruption, model-based, +crash-injection, recovery, and fuzz evidence is implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 62419a8..abbeb89 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index e7e9529..f7241a9 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -246,6 +246,10 @@ authenticated reconstruction, and canonical digest defined by transition. Keep never omits one failed member and continues with a smaller live set. +`preflight_retention_transition` now combines steps 3 and 4 below without I/O. +It returns a consequential publish or already-committed proof only after exact +generation planning and complete closure verification succeed in that order. + Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the catalog `HEAD`. diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 74d5b77..7471b67 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -39,6 +39,8 @@ mod root_integrity; mod root_semantic_header; mod transition_error; mod transition_planner; +mod transition_preflight; +mod transition_preflight_error; mod transition_readiness; mod verified_closure; @@ -57,5 +59,7 @@ pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; pub use transition_planner::plan_retention_transition; +pub use transition_preflight::{RetentionTransitionPreflight, preflight_retention_transition}; +pub use transition_preflight_error::RetentionTransitionPreflightError; pub use transition_readiness::RetentionTransitionReadiness; pub use verified_closure::VerifiedRetentionClosure; diff --git a/src/adapters/retention/transition_preflight.rs b/src/adapters/retention/transition_preflight.rs new file mode 100644 index 0000000..06992b6 --- /dev/null +++ b/src/adapters/retention/transition_preflight.rs @@ -0,0 +1,80 @@ +//! This boundary module owns complete retention transition preflight. + +use super::{ + AdmittedRetentionRoot, RetentionTransitionPreflightError, RetentionTransitionReadiness, + VerifiedRetentionClosure, plan_retention_transition, verify_retention_closure, +}; +use crate::CatalogSnapshot; +use crate::retention::RetentionGenerationExpectation; + +/// Complete storage-independent proof required before retention publication. +#[must_use = "retention preflight must be consumed by publication or handled explicitly"] +#[derive(Debug)] +pub enum RetentionTransitionPreflight<'encoded> { + /// The candidate is an exact successor whose verified closure must publish. + Publish { + /// Fully admitted canonical candidate root. + candidate: AdmittedRetentionRoot<'encoded>, + /// Closure proof against the exact pinned catalog. + closure: VerifiedRetentionClosure, + }, + /// The exact candidate is current and its closure still verifies. + AlreadyCommitted { + /// Fully admitted byte-identical current root. + candidate: AdmittedRetentionRoot<'encoded>, + /// Current closure proof against the exact pinned catalog. + closure: VerifiedRetentionClosure, + }, +} + +impl<'encoded> RetentionTransitionPreflight<'encoded> { + /// Borrows the fully admitted candidate root. + pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { + match self { + Self::Publish { candidate, .. } | Self::AlreadyCommitted { candidate, .. } => candidate, + } + } + + /// Returns the complete verified closure evidence. + pub const fn closure(&self) -> VerifiedRetentionClosure { + match self { + Self::Publish { closure, .. } | Self::AlreadyCommitted { closure, .. } => *closure, + } + } +} + +/// Proves generation and closure invariants before retention storage mutation. +/// +/// Generation planning completes before closure traversal. Exact replay still +/// requires the current closure to verify against the pinned catalog. The +/// function performs no I/O and inherits closure verification's root-bounded +/// record index and per-layout entry allocation. +/// +/// # Errors +/// +/// Returns [`RetentionTransitionPreflightError::Transition`] for generation or +/// successor refusal, then [`RetentionTransitionPreflightError::Closure`] for +/// the first deterministic closure refusal. +pub fn preflight_retention_transition<'encoded>( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, + candidate: AdmittedRetentionRoot<'encoded>, + catalog: &CatalogSnapshot<'_, '_, '_>, +) -> Result, RetentionTransitionPreflightError> { + let readiness = plan_retention_transition(expected, current, candidate) + .map_err(|source| RetentionTransitionPreflightError::Transition { source })?; + let closure = + verify_retention_closure(readiness.candidate().root(), catalog).map_err(|source| { + RetentionTransitionPreflightError::Closure { + source: Box::new(source), + } + })?; + Ok(match readiness { + RetentionTransitionReadiness::Publish { candidate } => { + RetentionTransitionPreflight::Publish { candidate, closure } + } + RetentionTransitionReadiness::AlreadyCommitted { candidate } => { + RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } + } + }) +} diff --git a/src/adapters/retention/transition_preflight_error.rs b/src/adapters/retention/transition_preflight_error.rs new file mode 100644 index 0000000..61113b1 --- /dev/null +++ b/src/adapters/retention/transition_preflight_error.rs @@ -0,0 +1,39 @@ +//! This boundary module owns retention transition preflight failures. + +use std::error::Error; +use std::fmt; + +use super::{RetentionClosureVerificationError, RetentionTransitionError}; + +/// Failure before a retention transition may invoke publication storage. +#[derive(Debug)] +pub enum RetentionTransitionPreflightError { + /// Generation or exact-successor planning refused the candidate. + Transition { + /// Preserved transition-planning refusal. + source: RetentionTransitionError, + }, + /// The candidate closure failed against the pinned catalog. + Closure { + /// Preserved deterministic closure refusal. + source: Box, + }, +} + +impl fmt::Display for RetentionTransitionPreflightError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Transition { .. } => formatter.write_str("retention transition planning failed"), + Self::Closure { .. } => formatter.write_str("retention closure verification failed"), + } + } +} + +impl Error for RetentionTransitionPreflightError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Transition { source } => Some(source), + Self::Closure { source } => Some(source.as_ref()), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 12b90ed..ed115fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,10 +23,10 @@ //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots //! are validated; canonical in-memory root, manifest, and head encoding and -//! decoding, storage-independent expected-state transition planning, and -//! deterministic bounded closure verification against a pinned catalog are -//! available. Retention publication, recovery, and garbage collection remain -//! intentionally absent. +//! decoding, storage-independent expected-state transition planning, +//! deterministic bounded closure verification against a pinned catalog, and a +//! combined transition preflight proof are available. Retention publication, +//! recovery, and garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -109,8 +109,9 @@ pub use adapters::{ CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionTransitionError, RetentionTransitionReadiness, VerifiedRetentionClosure, - plan_retention_transition, verify_retention_closure, + RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, + RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, + preflight_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_preflight.rs b/tests/retention_preflight.rs new file mode 100644 index 0000000..7eee6b5 --- /dev/null +++ b/tests/retention_preflight.rs @@ -0,0 +1,179 @@ +//! Retention transition preflight laws. + +mod support; + +use std::error::Error; + +use keep::{ + AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, LayoutEntryLimit, RetentionClosureVerificationError, + RetentionGenerationExpectation, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RootGeneration, SegmentReadPolicy, SegmentRecordIdentity, + SegmentRecordLimit, preflight_retention_transition, +}; +use support::{decode_hex, require_error}; + +const ROOT_HEX: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const CHUNK_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CHUNK_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const CHUNK_HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); + +#[test] +fn publish_preflight_binds_generation_and_closure_proofs() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let preflight = with_snapshot( + BUNDLE_SEGMENT_HEX, + BUNDLE_CATALOG_HEX, + BUNDLE_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + }, + )??; + + assert!(matches!( + preflight, + RetentionTransitionPreflight::Publish { + candidate, + closure, + } if candidate.root().generation() == RootGeneration::INITIAL + && closure.usage().node_count() == 2 + )); + Ok(()) +} + +#[test] +fn stale_generation_refuses_before_missing_closure_evidence() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let result = with_snapshot( + CHUNK_SEGMENT_HEX, + CHUNK_CATALOG_HEX, + CHUNK_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Current(RootGeneration::INITIAL), + None, + candidate, + snapshot, + ) + }, + )?; + let error = require_error(result, "stale generation reached closure verification")?; + + assert!(matches!( + error, + RetentionTransitionPreflightError::Transition { + source: RetentionTransitionError::StaleGeneration { + expected: RetentionGenerationExpectation::Current(expected), + observed: None, + }, + } if expected == RootGeneration::INITIAL + )); + Ok(()) +} + +#[test] +fn valid_generation_preserves_missing_closure_member() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let result = with_snapshot( + CHUNK_SEGMENT_HEX, + CHUNK_CATALOG_HEX, + CHUNK_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + }, + )?; + let error = require_error(result, "missing closure member passed preflight")?; + let RetentionTransitionPreflightError::Closure { source } = error else { + return Err("missing closure member reached the wrong preflight boundary".into()); + }; + + assert!(matches!( + *source, + RetentionClosureVerificationError::MissingMember { + identity: SegmentRecordIdentity::Layout(_), + } + )); + Ok(()) +} + +#[test] +fn exact_retry_still_returns_current_closure_evidence() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let preflight = with_snapshot( + BUNDLE_SEGMENT_HEX, + BUNDLE_CATALOG_HEX, + BUNDLE_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + }, + )??; + + assert!(matches!( + preflight, + RetentionTransitionPreflight::AlreadyCommitted { closure, .. } + if closure.usage().physical_bytes() == 509 + )); + Ok(()) +} + +fn with_snapshot( + segment_hex: &str, + catalog_hex: &str, + head_hex: &str, + operation: impl FnOnce(&CatalogSnapshot<'_, '_, '_>) -> Result, +) -> Result, Box> { + let segment_bytes = fixture(segment_hex)?; + let catalog_bytes = fixture(catalog_hex)?; + let head_bytes = fixture(head_hex)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(operation(&snapshot)) +} + +fn admitted_catalog<'catalog, 'records>( + catalog_bytes: &'catalog [u8], + segments: &'records [AdmittedSegment<'records>], +) -> Result, Box> { + ChecksummedCatalog::decode(catalog_bytes)? + .admit(segments) + .map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 50eb23caa0bcaa970f3b758852a4ce1b9ee969bb Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 23:09:20 -0700 Subject: [PATCH 20/50] Add: Name retention publication phases --- CHANGELOG.md | 3 +- README.md | 8 +- docs/formats/segment-store-v2/README.md | 13 +-- docs/formats/segment-store-v2/recovery.md | 3 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + src/adapters/retention/publication_phase.rs | 92 +++++++++++++++++++ src/lib.rs | 13 +-- tests/retention_publication_phase.rs | 83 +++++++++++++++++ 9 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 src/adapters/retention/publication_phase.rs create mode 100644 tests/retention_publication_phase.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad25fc..7b9474b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ after its public API and format compatibility policies are established. - Retention transition preflight now combines exact expected-generation planning with deterministic closure verification against one pinned catalog - before any future publication storage call. + before any future publication storage call. A typed 17-phase vocabulary + freezes the planned durability and crash-boundary order. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index b1da614..39d7afe 100644 --- a/README.md +++ b/README.md @@ -118,10 +118,10 @@ matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a -pinned catalog; and a combined transition preflight proof are implemented. -Publication, recovery, compaction, and garbage collection remain planned. -Presence in the reference CAS does not claim retention, crash recovery, or -durability. +pinned catalog; a combined transition preflight proof; and the exact 17-phase +publication vocabulary are implemented. Publication execution, recovery, +compaction, and garbage collection remain planned. Presence in the reference +CAS does not claim retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 3c12a8e..12001af 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -76,9 +76,10 @@ generations, registered realization profiles, bounded closure policies, reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition planning, deterministic bounded closure verification against one pinned -catalog, and their combined preflight proof are available. Production -filesystem retention publication, recovery, migration, and garbage collection -do not exist yet. Requirements that remain planned or in progress in issue #19 -or issue #21 are not complete implementation evidence. A store must refuse -unsupported version-2 state until the relevant corruption, model-based, -crash-injection, recovery, and fuzz evidence is implemented. +catalog, their combined preflight proof, and the exact 17-phase publication +vocabulary are available. Production filesystem retention publication, +recovery, migration, and garbage collection do not exist yet. Requirements +that remain planned or in progress in issue #19 or issue #21 are not complete +implementation evidence. A store must refuse unsupported version-2 state until +the relevant corruption, model-based, crash-injection, recovery, and fuzz +evidence is implemented. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index dfd29dc..604d0e3 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -277,6 +277,9 @@ The retention crash points are: | `KEEP-CRASH-051` | retained manifest-stage removal | | `KEEP-CRASH-052` | retention cleanup synchronization | +`RetentionPublicationPhase::ALL` freezes this exact order as a typed public +vocabulary. Storage execution and process-death evidence remain unimplemented. + Each point requires before, during, and after process-death evidence. Restart must establish exact catalog visibility, retention head, namespace generation, orphan classification, stage disposition, and recovery report. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index abbeb89..fa450bd 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -14,7 +14,7 @@ case is not evidence. | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs`; storage execution, operation-order, and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 7471b67..cc00ceb 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -27,6 +27,7 @@ mod manifest_field_decoder; mod manifest_header_decoder; mod manifest_integrity; mod manifest_semantic_header; +mod publication_phase; mod root_anchor_decoder; mod root_decode_error; mod root_decode_error_display; @@ -55,6 +56,7 @@ pub use closure_verifier::verify_retention_closure; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; +pub use publication_phase::RetentionPublicationPhase; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; diff --git a/src/adapters/retention/publication_phase.rs b/src/adapters/retention/publication_phase.rs new file mode 100644 index 0000000..130bc77 --- /dev/null +++ b/src/adapters/retention/publication_phase.rs @@ -0,0 +1,92 @@ +//! This boundary module owns exact retention publication durability phases. + +use std::fmt; + +/// Storage transition attempted by retention namespace publication. +/// +/// [`Self::ALL`] corresponds in order to `KEEP-CRASH-036` through +/// `KEEP-CRASH-052`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionPublicationPhase { + /// Write the complete canonical `root.next`. + WriteRootStage, + /// Synchronize `root.next`. + SynchronizeRootStage, + /// Create or exactly admit the digest-named root namespace. + AdmitRootNamespace, + /// Synchronize `retention/roots` after namespace admission. + SynchronizeRootsAfterNamespace, + /// Link the synchronized root stage into its immutable namespace. + LinkRoot, + /// Synchronize the digest-named root namespace after linking. + SynchronizeRootNamespace, + /// Write the complete canonical `manifest.next`. + WriteManifestStage, + /// Synchronize `manifest.next`. + SynchronizeManifestStage, + /// Link the synchronized manifest into its immutable pool. + LinkManifest, + /// Synchronize the immutable manifest pool. + SynchronizeManifestPool, + /// Write the complete canonical retention `head.next`. + WriteHeadStage, + /// Synchronize the retention `head.next`. + SynchronizeHeadStage, + /// Atomically replace the retention `HEAD`. + ReplaceHead, + /// Synchronize `retention` after head replacement. + SynchronizeRetentionNamespace, + /// Remove the retained `root.next`. + RemoveRootStage, + /// Remove the retained `manifest.next`. + RemoveManifestStage, + /// Synchronize `retention` after stage cleanup. + SynchronizeCleanup, +} + +impl RetentionPublicationPhase { + /// Every publication phase in normative crash-boundary order. + pub const ALL: [Self; 17] = [ + Self::WriteRootStage, + Self::SynchronizeRootStage, + Self::AdmitRootNamespace, + Self::SynchronizeRootsAfterNamespace, + Self::LinkRoot, + Self::SynchronizeRootNamespace, + Self::WriteManifestStage, + Self::SynchronizeManifestStage, + Self::LinkManifest, + Self::SynchronizeManifestPool, + Self::WriteHeadStage, + Self::SynchronizeHeadStage, + Self::ReplaceHead, + Self::SynchronizeRetentionNamespace, + Self::RemoveRootStage, + Self::RemoveManifestStage, + Self::SynchronizeCleanup, + ]; +} + +impl fmt::Display for RetentionPublicationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::WriteRootStage => "root-stage write", + Self::SynchronizeRootStage => "root-stage synchronization", + Self::AdmitRootNamespace => "root-namespace admission", + Self::SynchronizeRootsAfterNamespace => "post-namespace roots synchronization", + Self::LinkRoot => "immutable root link", + Self::SynchronizeRootNamespace => "root-namespace synchronization", + Self::WriteManifestStage => "manifest-stage write", + Self::SynchronizeManifestStage => "manifest-stage synchronization", + Self::LinkManifest => "immutable manifest link", + Self::SynchronizeManifestPool => "manifest-pool synchronization", + Self::WriteHeadStage => "retention-head-stage write", + Self::SynchronizeHeadStage => "retention-head-stage synchronization", + Self::ReplaceHead => "retention-head replacement", + Self::SynchronizeRetentionNamespace => "retention-namespace synchronization", + Self::RemoveRootStage => "retained root-stage removal", + Self::RemoveManifestStage => "retained manifest-stage removal", + Self::SynchronizeCleanup => "retention cleanup synchronization", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index ed115fb..52eabd1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,8 +25,9 @@ //! are validated; canonical in-memory root, manifest, and head encoding and //! decoding, storage-independent expected-state transition planning, //! deterministic bounded closure verification against a pinned catalog, and a -//! combined transition preflight proof are available. Retention publication, -//! recovery, and garbage collection remain intentionally absent. +//! combined transition preflight proof and exact publication phase vocabulary +//! are available. Retention publication execution, recovery, and garbage +//! collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -108,10 +109,10 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, - RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, - preflight_retention_transition, verify_retention_closure, + RetentionManifestEncodeError, RetentionPublicationPhase, RetentionRootDecodeError, + RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, + plan_retention_transition, preflight_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_publication_phase.rs b/tests/retention_publication_phase.rs new file mode 100644 index 0000000..6fb343e --- /dev/null +++ b/tests/retention_publication_phase.rs @@ -0,0 +1,83 @@ +//! Retention publication phase vocabulary laws. + +use keep::RetentionPublicationPhase; + +#[test] +fn publication_phases_are_complete_ordered_and_stably_named() { + let expected = [ + ( + RetentionPublicationPhase::WriteRootStage, + "root-stage write", + ), + ( + RetentionPublicationPhase::SynchronizeRootStage, + "root-stage synchronization", + ), + ( + RetentionPublicationPhase::AdmitRootNamespace, + "root-namespace admission", + ), + ( + RetentionPublicationPhase::SynchronizeRootsAfterNamespace, + "post-namespace roots synchronization", + ), + (RetentionPublicationPhase::LinkRoot, "immutable root link"), + ( + RetentionPublicationPhase::SynchronizeRootNamespace, + "root-namespace synchronization", + ), + ( + RetentionPublicationPhase::WriteManifestStage, + "manifest-stage write", + ), + ( + RetentionPublicationPhase::SynchronizeManifestStage, + "manifest-stage synchronization", + ), + ( + RetentionPublicationPhase::LinkManifest, + "immutable manifest link", + ), + ( + RetentionPublicationPhase::SynchronizeManifestPool, + "manifest-pool synchronization", + ), + ( + RetentionPublicationPhase::WriteHeadStage, + "retention-head-stage write", + ), + ( + RetentionPublicationPhase::SynchronizeHeadStage, + "retention-head-stage synchronization", + ), + ( + RetentionPublicationPhase::ReplaceHead, + "retention-head replacement", + ), + ( + RetentionPublicationPhase::SynchronizeRetentionNamespace, + "retention-namespace synchronization", + ), + ( + RetentionPublicationPhase::RemoveRootStage, + "retained root-stage removal", + ), + ( + RetentionPublicationPhase::RemoveManifestStage, + "retained manifest-stage removal", + ), + ( + RetentionPublicationPhase::SynchronizeCleanup, + "retention cleanup synchronization", + ), + ]; + + assert_eq!( + RetentionPublicationPhase::ALL, + expected.map(|entry| entry.0) + ); + assert_eq!( + RetentionPublicationPhase::ALL.map(|phase| phase.to_string()), + expected.map(|entry| entry.1.to_owned()) + ); +} From 261a8a65395d0860dcb15f944fc62615ae224d48 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 23:28:06 -0700 Subject: [PATCH 21/50] Add: Define retention publication storage --- CHANGELOG.md | 2 +- README.md | 3 +- docs/formats/segment-store-v2/README.md | 13 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 8 +- src/adapters/retention.rs | 4 + src/adapters/retention/namespace_admission.rs | 11 ++ src/adapters/retention/publication_storage.rs | 139 ++++++++++++++++++ src/lib.rs | 12 +- tests/retention_publication_storage.rs | 60 ++++++++ .../recording_storage.rs | 122 +++++++++++++++ 11 files changed, 359 insertions(+), 17 deletions(-) create mode 100644 src/adapters/retention/namespace_admission.rs create mode 100644 src/adapters/retention/publication_storage.rs create mode 100644 tests/retention_publication_storage.rs create mode 100644 tests/retention_publication_storage/recording_storage.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9474b..5a25d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ after its public API and format compatibility policies are established. - Retention transition preflight now combines exact expected-generation planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary - freezes the planned durability and crash-boundary order. + and blocking storage port freeze the durability and crash-boundary contract. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 39d7afe..c10200e 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ power loss. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase -publication vocabulary are implemented. Publication execution, recovery, +publication vocabulary with a blocking storage capability port are +implemented. Publication orchestration, filesystem execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 12001af..f1c627a 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -77,9 +77,10 @@ reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition planning, deterministic bounded closure verification against one pinned catalog, their combined preflight proof, and the exact 17-phase publication -vocabulary are available. Production filesystem retention publication, -recovery, migration, and garbage collection do not exist yet. Requirements -that remain planned or in progress in issue #19 or issue #21 are not complete -implementation evidence. A store must refuse unsupported version-2 state until -the relevant corruption, model-based, crash-injection, recovery, and fuzz -evidence is implemented. +vocabulary with a blocking storage capability port are available. Publication +orchestration and production filesystem retention publication, recovery, +migration, and garbage collection do not exist yet. Requirements that remain +planned or in progress in issue #19 or issue #21 are not complete implementation +evidence. A store must refuse unsupported version-2 state until the relevant +corruption, model-based, crash-injection, recovery, and fuzz evidence is +implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index fa450bd..298d6f9 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -14,7 +14,7 @@ case is not evidence. | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs`; storage execution, operation-order, and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index f7241a9..c55f051 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,11 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root, manifest, and head codecs plus -storage-independent expected-state transition planning. Closure verification, -filesystem publication, recovery, and garbage collection remain absent. +implements validated in-memory root, manifest, and head codecs, +storage-independent expected-state transition planning, deterministic closure +verification, and a blocking publication storage capability port. Publication +orchestration, filesystem execution, recovery, and garbage collection remain +absent. ## Global retention manifest diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index cc00ceb..73a63f5 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -27,7 +27,9 @@ mod manifest_field_decoder; mod manifest_header_decoder; mod manifest_integrity; mod manifest_semantic_header; +mod namespace_admission; mod publication_phase; +mod publication_storage; mod root_anchor_decoder; mod root_decode_error; mod root_decode_error_display; @@ -56,7 +58,9 @@ pub use closure_verifier::verify_retention_closure; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; +pub use namespace_admission::RetentionNamespaceAdmission; pub use publication_phase::RetentionPublicationPhase; +pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; diff --git a/src/adapters/retention/namespace_admission.rs b/src/adapters/retention/namespace_admission.rs new file mode 100644 index 0000000..4d9b1f7 --- /dev/null +++ b/src/adapters/retention/namespace_admission.rs @@ -0,0 +1,11 @@ +//! This boundary module owns digest-named retention namespace admission outcomes. + +/// Result of exact retention root-namespace admission. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionNamespaceAdmission { + /// The exact digest-named directory already existed and was admitted. + Existing, + /// The exact digest-named directory was created and admitted. + Created, +} diff --git a/src/adapters/retention/publication_storage.rs b/src/adapters/retention/publication_storage.rs new file mode 100644 index 0000000..cbef5d8 --- /dev/null +++ b/src/adapters/retention/publication_storage.rs @@ -0,0 +1,139 @@ +//! This boundary module owns blocking retention publication durability capabilities. + +use std::io; + +use super::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + RetentionNamespaceAdmission, +}; + +/// Blocking storage capabilities for one writer-locked retention publication. +/// +/// An implementation must retain exclusive writer authority and one pinned +/// store root for the complete operation. Each method corresponds to one +/// [`RetentionPublicationPhase`](super::RetentionPublicationPhase) and must not +/// report success before the named durability and verification obligations are +/// complete. +pub trait RetentionPublicationStorage { + /// Exclusively creates and completely writes the canonical root stage. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()>; + + /// Synchronizes the complete root stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_root_stage(&mut self) -> io::Result<()>; + + /// Creates or exactly admits the candidate's digest-named root namespace. + /// + /// # Errors + /// + /// Returns the exact namespace creation or admission failure. + fn admit_root_namespace( + &mut self, + root: &AdmittedRetentionRoot<'_>, + ) -> io::Result; + + /// Synchronizes `retention/roots` after namespace creation. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_roots_after_namespace(&mut self) -> io::Result<()>; + + /// Links and completely verifies the immutable root-pool entry. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_root(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()>; + + /// Synchronizes the candidate's digest-named root namespace. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_namespace(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()>; + + /// Exclusively creates and completely writes the canonical manifest stage. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_manifest_stage(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()>; + + /// Synchronizes the complete manifest stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_manifest_stage(&mut self) -> io::Result<()>; + + /// Links and completely verifies the immutable manifest-pool entry. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_manifest(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()>; + + /// Synchronizes the immutable manifest pool. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_manifest_pool(&mut self) -> io::Result<()>; + + /// Exclusively creates and completely writes the canonical head stage. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_head_stage(&mut self, head: &CanonicalRetentionHead) -> io::Result<()>; + + /// Synchronizes the complete head stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_head_stage(&mut self) -> io::Result<()>; + + /// Atomically replaces `retention/HEAD` with the synchronized head stage. + /// + /// # Errors + /// + /// Returns the exact replacement failure. + fn replace_head(&mut self) -> io::Result<()>; + + /// Synchronizes `retention` after head replacement. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_retention_namespace(&mut self) -> io::Result<()>; + + /// Removes only the retained root stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_root_stage(&mut self) -> io::Result<()>; + + /// Removes only the retained manifest stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_manifest_stage(&mut self) -> io::Result<()>; + + /// Synchronizes `retention` after both stage removals. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_cleanup(&mut self) -> io::Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index 52eabd1..bda7161 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,8 @@ //! decoding, storage-independent expected-state transition planning, //! deterministic bounded closure verification against a pinned catalog, and a //! combined transition preflight proof and exact publication phase vocabulary -//! are available. Retention publication execution, recovery, and garbage +//! with a blocking storage capability port are available. Retention +//! publication orchestration, filesystem execution, recovery, and garbage //! collection remain intentionally absent. #[cfg(test)] @@ -109,10 +110,11 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionPublicationPhase, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, - RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, - plan_retention_transition, preflight_retention_transition, verify_retention_closure, + RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationPhase, + RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, + RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, + preflight_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_publication_storage.rs b/tests/retention_publication_storage.rs new file mode 100644 index 0000000..900a85b --- /dev/null +++ b/tests/retention_publication_storage.rs @@ -0,0 +1,60 @@ +//! Retention publication storage-port laws. + +#[path = "retention_publication_storage/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, ChecksummedRetentionHead, RetentionNamespaceAdmission, + RetentionPublicationPhase, RetentionPublicationStorage, +}; +use recording_storage::RecordingStorage; +use support::decode_hex; + +const ROOT_HEX: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const MANIFEST_HEX: &str = include_str!("../conformance/segment-store/v2/one-root-manifest.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v2/one-root-head.hex"); + +#[test] +fn storage_port_names_one_capability_per_publication_phase() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let manifest_bytes = fixture(MANIFEST_HEX)?; + let head_bytes = fixture(HEAD_HEX)?; + let root = AdmittedRetentionRoot::decode(&root_bytes)?; + let admitted_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let manifest = CanonicalRetentionManifest::from_manifest(admitted_manifest.manifest())?; + let checksummed_head = ChecksummedRetentionHead::decode(&head_bytes)?; + let head = CanonicalRetentionHead::from_head(checksummed_head.head()); + let mut storage = RecordingStorage::new(); + + storage.write_root_stage(&root)?; + storage.synchronize_root_stage()?; + assert_eq!( + storage.admit_root_namespace(&root)?, + RetentionNamespaceAdmission::Created + ); + storage.synchronize_roots_after_namespace()?; + storage.link_root(&root)?; + storage.synchronize_root_namespace(&root)?; + storage.write_manifest_stage(&manifest)?; + storage.synchronize_manifest_stage()?; + storage.link_manifest(&manifest)?; + storage.synchronize_manifest_pool()?; + storage.write_head_stage(&head)?; + storage.synchronize_head_stage()?; + storage.replace_head()?; + storage.synchronize_retention_namespace()?; + storage.remove_root_stage()?; + storage.remove_manifest_stage()?; + storage.synchronize_cleanup()?; + + assert_eq!(storage.observed(), RetentionPublicationPhase::ALL); + Ok(()) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/retention_publication_storage/recording_storage.rs b/tests/retention_publication_storage/recording_storage.rs new file mode 100644 index 0000000..51047dc --- /dev/null +++ b/tests/retention_publication_storage/recording_storage.rs @@ -0,0 +1,122 @@ +//! Deterministic retention publication storage recorder. + +use std::io; + +use keep::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationStorage, +}; + +/// Storage port that records every attempted publication phase. +#[derive(Default)] +pub struct RecordingStorage { + observed: Vec, +} + +impl RecordingStorage { + /// Creates an empty recorder. + pub const fn new() -> Self { + Self { + observed: Vec::new(), + } + } + + /// Returns every recorded phase in call order. + pub fn observed(&self) -> &[RetentionPublicationPhase] { + &self.observed + } + + fn record(&mut self, phase: RetentionPublicationPhase) { + self.observed.push(phase); + } +} + +impl RetentionPublicationStorage for RecordingStorage { + fn write_root_stage(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.record(RetentionPublicationPhase::WriteRootStage); + Ok(()) + } + + fn synchronize_root_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRootStage); + Ok(()) + } + + fn admit_root_namespace( + &mut self, + _root: &AdmittedRetentionRoot<'_>, + ) -> io::Result { + self.record(RetentionPublicationPhase::AdmitRootNamespace); + Ok(RetentionNamespaceAdmission::Created) + } + + fn synchronize_roots_after_namespace(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRootsAfterNamespace); + Ok(()) + } + + fn link_root(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.record(RetentionPublicationPhase::LinkRoot); + Ok(()) + } + + fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRootNamespace); + Ok(()) + } + + fn write_manifest_stage(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.record(RetentionPublicationPhase::WriteManifestStage); + Ok(()) + } + + fn synchronize_manifest_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeManifestStage); + Ok(()) + } + + fn link_manifest(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.record(RetentionPublicationPhase::LinkManifest); + Ok(()) + } + + fn synchronize_manifest_pool(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeManifestPool); + Ok(()) + } + + fn write_head_stage(&mut self, _head: &CanonicalRetentionHead) -> io::Result<()> { + self.record(RetentionPublicationPhase::WriteHeadStage); + Ok(()) + } + + fn synchronize_head_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeHeadStage); + Ok(()) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::ReplaceHead); + Ok(()) + } + + fn synchronize_retention_namespace(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRetentionNamespace); + Ok(()) + } + + fn remove_root_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::RemoveRootStage); + Ok(()) + } + + fn remove_manifest_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::RemoveManifestStage); + Ok(()) + } + + fn synchronize_cleanup(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeCleanup); + Ok(()) + } +} From b8affa75aa99e9f512c3035908627186f63a9767 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 23:57:00 -0700 Subject: [PATCH 22/50] Add: Prepare retention publication artifacts --- CHANGELOG.md | 3 +- README.md | 8 +- docs/formats/segment-store-v2/README.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 8 +- src/adapters/retention.rs | 8 + .../retention/manifest_entry_update.rs | 85 +++++++++ .../retention/prepared_publication.rs | 94 +++++++++ .../retention/publication_preparation.rs | 67 +++++++ .../publication_preparation_error.rs | 179 ++++++++++++++++++ src/adapters/retention/successor_manifest.rs | 146 ++++++++++++++ src/lib.rs | 21 +- src/retention/liveness_generation.rs | 3 + tests/retention_publication_preparation.rs | 11 ++ .../fixture.rs | 121 ++++++++++++ .../initial_laws.rs | 35 ++++ .../refusal_laws.rs | 78 ++++++++ .../successor_laws.rs | 117 ++++++++++++ 18 files changed, 973 insertions(+), 20 deletions(-) create mode 100644 src/adapters/retention/manifest_entry_update.rs create mode 100644 src/adapters/retention/prepared_publication.rs create mode 100644 src/adapters/retention/publication_preparation.rs create mode 100644 src/adapters/retention/publication_preparation_error.rs create mode 100644 src/adapters/retention/successor_manifest.rs create mode 100644 tests/retention_publication_preparation.rs create mode 100644 tests/retention_publication_preparation/fixture.rs create mode 100644 tests/retention_publication_preparation/initial_laws.rs create mode 100644 tests/retention_publication_preparation/refusal_laws.rs create mode 100644 tests/retention_publication_preparation/successor_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a25d50..766e479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ after its public API and format compatibility policies are established. - Retention transition preflight now combines exact expected-generation planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary - and blocking storage port freeze the durability and crash-boundary contract. + and blocking storage port freeze the durability and crash-boundary contract; + preparation binds preflight to exact canonical manifest and head successors. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index c10200e..38623fb 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,11 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Publication orchestration, filesystem execution, recovery, -compaction, and garbage collection remain planned. Presence in the reference -CAS does not claim retention, crash recovery, or durability. +implemented. Storage-independent preparation also binds preflight to exact +canonical manifest and head successors. Publication orchestration, filesystem +execution, recovery, compaction, and garbage collection remain planned. +Presence in the reference CAS does not claim durable retention or crash +recovery. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index f1c627a..54c7c1d 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -77,10 +77,11 @@ reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition planning, deterministic bounded closure verification against one pinned catalog, their combined preflight proof, and the exact 17-phase publication -vocabulary with a blocking storage capability port are available. Publication +vocabulary with a blocking storage capability port are available. +Storage-independent preparation derives exact canonical manifest and head +successors from coherent preflight and current-manifest evidence. Publication orchestration and production filesystem retention publication, recovery, migration, and garbage collection do not exist yet. Requirements that remain planned or in progress in issue #19 or issue #21 are not complete implementation evidence. A store must refuse unsupported version-2 state until the relevant -corruption, model-based, crash-injection, recovery, and fuzz evidence is -implemented. +corruption, model-based, crash-injection, recovery, and fuzz evidence exists. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 298d6f9..58b56b1 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index c55f051..cb9cc03 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -248,9 +248,11 @@ authenticated reconstruction, and canonical digest defined by transition. Keep never omits one failed member and continues with a smaller live set. -`preflight_retention_transition` now combines steps 3 and 4 below without I/O. -It returns a consequential publish or already-committed proof only after exact -generation planning and complete closure verification succeed in that order. +`preflight_retention_transition` combines steps 3 and 4 without I/O, returning +publish or already-committed only after generation and closure verification. +`prepare_retention_publication` binds that proof to the current manifest, +refuses incoherent root coordinates, and derives exact canonical successors; +exact retry produces no new global artifacts. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 73a63f5..addcd7c 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -23,12 +23,16 @@ mod manifest_decoder; mod manifest_encode_error; mod manifest_encoder; mod manifest_entry_decoder; +mod manifest_entry_update; mod manifest_field_decoder; mod manifest_header_decoder; mod manifest_integrity; mod manifest_semantic_header; mod namespace_admission; +mod prepared_publication; mod publication_phase; +mod publication_preparation; +mod publication_preparation_error; mod publication_storage; mod root_anchor_decoder; mod root_decode_error; @@ -40,6 +44,7 @@ mod root_field_decoder; mod root_header_decoder; mod root_integrity; mod root_semantic_header; +mod successor_manifest; mod transition_error; mod transition_planner; mod transition_preflight; @@ -59,7 +64,10 @@ pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use namespace_admission::RetentionNamespaceAdmission; +pub use prepared_publication::{PreparedRetentionPublication, RetentionPublicationPreparation}; pub use publication_phase::RetentionPublicationPhase; +pub use publication_preparation::prepare_retention_publication; +pub use publication_preparation_error::RetentionPublicationPreparationError; pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/manifest_entry_update.rs b/src/adapters/retention/manifest_entry_update.rs new file mode 100644 index 0000000..3047439 --- /dev/null +++ b/src/adapters/retention/manifest_entry_update.rs @@ -0,0 +1,85 @@ +//! This boundary module owns bounded retention manifest entry updates. + +use super::RetentionPublicationPreparationError; +use crate::{RetentionManifest, RetentionManifestEntry, RetentionManifestError}; + +#[derive(Clone, Copy)] +pub(super) enum ManifestEntryUpdate { + Insert { + index: usize, + entry: RetentionManifestEntry, + }, + Replace { + index: usize, + entry: RetentionManifestEntry, + }, +} + +pub(super) fn apply( + current: &[RetentionManifestEntry], + update: ManifestEntryUpdate, +) -> Result, RetentionPublicationPreparationError> { + let inserts = usize::from(matches!(update, ManifestEntryUpdate::Insert { .. })); + let observed = current.len().checked_add(inserts).ok_or( + RetentionPublicationPreparationError::Manifest { + source: RetentionManifestError::EntryCountExceeded { + maximum: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed: usize::MAX, + }, + }, + )?; + require_admitted_count(observed)?; + let mut entries = Vec::new(); + entries + .try_reserve_exact(observed) + .map_err(|source| RetentionPublicationPreparationError::EntryAllocation { source })?; + let (index, entry, skip) = match update { + ManifestEntryUpdate::Insert { index, entry } => (index, entry, 0), + ManifestEntryUpdate::Replace { index, entry } => (index, entry, 1), + }; + let (before, remaining) = split(current, index)?; + let after = + remaining + .get(skip..) + .ok_or(RetentionPublicationPreparationError::ManifestEntryIndex { + index, + entry_count: current.len(), + })?; + entries.extend_from_slice(before); + entries.push(entry); + entries.extend_from_slice(after); + Ok(entries) +} + +fn require_admitted_count(observed: usize) -> Result<(), RetentionPublicationPreparationError> { + let admitted = u32::try_from(observed).map_err(|_| entry_count_error(observed))?; + if admitted > RetentionManifest::MAXIMUM_ENTRY_COUNT { + Err(entry_count_error(observed)) + } else { + Ok(()) + } +} + +fn split( + current: &[RetentionManifestEntry], + index: usize, +) -> Result< + (&[RetentionManifestEntry], &[RetentionManifestEntry]), + RetentionPublicationPreparationError, +> { + current.split_at_checked(index).ok_or( + RetentionPublicationPreparationError::ManifestEntryIndex { + index, + entry_count: current.len(), + }, + ) +} + +const fn entry_count_error(observed: usize) -> RetentionPublicationPreparationError { + RetentionPublicationPreparationError::Manifest { + source: RetentionManifestError::EntryCountExceeded { + maximum: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed, + }, + } +} diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs new file mode 100644 index 0000000..6eca896 --- /dev/null +++ b/src/adapters/retention/prepared_publication.rs @@ -0,0 +1,94 @@ +//! This boundary module owns storage-ready retention publication artifacts. + +use super::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + VerifiedRetentionClosure, +}; +use crate::LivenessGeneration; + +/// Canonical global artifacts ready for ordered storage execution. +#[must_use = "prepared retention publication must be executed or handled explicitly"] +#[derive(Debug)] +pub struct PreparedRetentionPublication { + manifest: CanonicalRetentionManifest, + head: CanonicalRetentionHead, + liveness_generation: LivenessGeneration, +} + +impl PreparedRetentionPublication { + /// Returns the complete canonical successor manifest. + pub const fn manifest(&self) -> &CanonicalRetentionManifest { + &self.manifest + } + + /// Returns the complete canonical successor head. + pub const fn head(&self) -> &CanonicalRetentionHead { + &self.head + } + + /// Returns the exact successor global liveness generation. + pub const fn liveness_generation(&self) -> LivenessGeneration { + self.liveness_generation + } + + pub(super) const fn new( + manifest: CanonicalRetentionManifest, + head: CanonicalRetentionHead, + liveness_generation: LivenessGeneration, + ) -> Self { + Self { + manifest, + head, + liveness_generation, + } + } +} + +/// Result of binding one preflight proof to the current global manifest. +#[must_use = "retention publication preparation must be handled explicitly"] +#[derive(Debug)] +pub struct RetentionPublicationPreparation<'encoded> { + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, + publication: Option, +} + +impl<'encoded> RetentionPublicationPreparation<'encoded> { + /// Borrows the admitted candidate root. + pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { + &self.candidate + } + + /// Returns the revalidated candidate closure. + pub const fn closure(&self) -> VerifiedRetentionClosure { + self.closure + } + + /// Returns new global artifacts, or normal absence for an exact retry. + pub const fn publication(&self) -> Option<&PreparedRetentionPublication> { + self.publication.as_ref() + } + + pub(super) const fn publish( + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, + publication: PreparedRetentionPublication, + ) -> Self { + Self { + candidate, + closure, + publication: Some(publication), + } + } + + pub(super) const fn already_committed( + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, + ) -> Self { + Self { + candidate, + closure, + publication: None, + } + } +} diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs new file mode 100644 index 0000000..da1b6ee --- /dev/null +++ b/src/adapters/retention/publication_preparation.rs @@ -0,0 +1,67 @@ +//! This boundary module owns storage-independent retention publication preparation. + +use super::{ + AdmittedRetentionManifest, CanonicalRetentionHead, CanonicalRetentionManifest, + PreparedRetentionPublication, RetentionPublicationPreparation, + RetentionPublicationPreparationError, RetentionTransitionPreflight, successor_manifest, +}; +use crate::{RetentionHead, RetentionManifestLength}; + +/// Binds preflight evidence to one current manifest and canonical successors. +/// +/// A publish result owns the complete manifest and head bytes required by the +/// storage protocol. An exact retry returns no new global artifacts. This +/// function performs no I/O. +/// +/// # Errors +/// +/// Returns [`RetentionPublicationPreparationError`] when the current manifest +/// disagrees with the preflight candidate, checked generation arithmetic or +/// bounded allocation fails, or canonical successor construction refuses. +pub fn prepare_retention_publication<'encoded>( + preflight: RetentionTransitionPreflight<'encoded>, + current_manifest: Option<&AdmittedRetentionManifest<'_>>, +) -> Result, RetentionPublicationPreparationError> { + match preflight { + RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } => { + successor_manifest::require_current_selection(&candidate, current_manifest)?; + Ok(RetentionPublicationPreparation::already_committed( + candidate, closure, + )) + } + RetentionTransitionPreflight::Publish { candidate, closure } => { + let semantic_manifest = successor_manifest::build(&candidate, current_manifest)?; + let liveness_generation = semantic_manifest.generation(); + let predecessor = semantic_manifest.predecessor(); + let manifest = CanonicalRetentionManifest::from_manifest(&semantic_manifest).map_err( + |source| RetentionPublicationPreparationError::ManifestEncoding { source }, + )?; + let manifest_length = manifest_length(&manifest)?; + let semantic_head = RetentionHead::new( + liveness_generation, + manifest_length, + manifest.digest(), + predecessor, + ) + .map_err(|source| RetentionPublicationPreparationError::Head { source })?; + let head = CanonicalRetentionHead::from_head(&semantic_head); + let publication = + PreparedRetentionPublication::new(manifest, head, liveness_generation); + Ok(RetentionPublicationPreparation::publish( + candidate, + closure, + publication, + )) + } + } +} + +fn manifest_length( + manifest: &CanonicalRetentionManifest, +) -> Result { + let observed = manifest.encoded().len(); + let value = u64::try_from(observed) + .map_err(|_| RetentionPublicationPreparationError::ManifestLengthOverflow { observed })?; + RetentionManifestLength::new(value) + .map_err(|source| RetentionPublicationPreparationError::ManifestLength { source }) +} diff --git a/src/adapters/retention/publication_preparation_error.rs b/src/adapters/retention/publication_preparation_error.rs new file mode 100644 index 0000000..9a17d8c --- /dev/null +++ b/src/adapters/retention/publication_preparation_error.rs @@ -0,0 +1,179 @@ +//! This boundary module owns retention publication preparation failures. + +use std::collections::TryReserveError; +use std::error::Error; +use std::fmt; + +use super::RetentionManifestEncodeError; +use crate::{ + LivenessGenerationError, RetentionHeadError, RetentionManifestError, + RetentionManifestLengthError, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, + RootGenerationError, +}; + +/// Failure to bind preflight evidence into canonical global artifacts. +#[derive(Debug)] +pub enum RetentionPublicationPreparationError { + /// Exact retry had no current global manifest to select its root. + CurrentManifestRequired { + /// Candidate namespace that must be selected. + namespace: RetentionNamespaceDigest, + }, + /// Exact retry was absent from the current global manifest. + CurrentManifestEntryMissing { + /// Candidate namespace absent from the manifest. + namespace: RetentionNamespaceDigest, + /// Candidate root generation. + generation: RootGeneration, + /// Candidate root digest. + digest: RetentionRootDigest, + }, + /// Current manifest entry and candidate did not form a successor. + ManifestSuccessorMismatch { + /// Candidate namespace. + namespace: RetentionNamespaceDigest, + /// Generation selected by the current manifest. + current_generation: RootGeneration, + /// Root digest selected by the current manifest. + current_digest: RetentionRootDigest, + /// Candidate root generation. + candidate_generation: RootGeneration, + /// Candidate-declared predecessor. + candidate_predecessor: Option, + }, + /// Exact retry disagreed with the current manifest selection. + CurrentManifestEntryMismatch { + /// Candidate namespace. + namespace: RetentionNamespaceDigest, + /// Generation selected by the current manifest. + current_generation: RootGeneration, + /// Root digest selected by the current manifest. + current_digest: RetentionRootDigest, + /// Candidate root generation. + candidate_generation: RootGeneration, + /// Candidate root digest. + candidate_digest: RetentionRootDigest, + }, + /// A namespace absent from the manifest carried a noninitial candidate. + UnexpectedNamespaceSuccessor { + /// Candidate namespace. + namespace: RetentionNamespaceDigest, + /// Noninitial candidate generation. + generation: RootGeneration, + /// Candidate-declared predecessor. + predecessor: Option, + }, + /// Internal manifest update index escaped the admitted entry range. + ManifestEntryIndex { + /// Attempted insertion or replacement index. + index: usize, + /// Current manifest entry count. + entry_count: usize, + }, + /// Global liveness generation could not advance. + LivenessGeneration { + /// Preserved checked-generation refusal. + source: LivenessGenerationError, + }, + /// A manifest-selected root generation could not advance. + RootGeneration { + /// Preserved checked-generation refusal. + source: RootGenerationError, + }, + /// Bounded successor-entry allocation was refused. + EntryAllocation { + /// Preserved allocation refusal. + source: TryReserveError, + }, + /// Successor manifest semantics were refused. + Manifest { + /// Preserved semantic refusal. + source: RetentionManifestError, + }, + /// Canonical successor manifest encoding failed. + ManifestEncoding { + /// Preserved encoding refusal. + source: RetentionManifestEncodeError, + }, + /// Host length could not fit the protocol length domain. + ManifestLengthOverflow { + /// Observed canonical byte length. + observed: usize, + }, + /// Canonical successor manifest length was refused. + ManifestLength { + /// Preserved typed-length refusal. + source: RetentionManifestLengthError, + }, + /// Successor head semantics were refused. + Head { + /// Preserved semantic refusal. + source: RetentionHeadError, + }, +} + +impl fmt::Display for RetentionPublicationPreparationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentManifestRequired { .. } => { + formatter.write_str("retention retry requires a current global manifest") + } + Self::CurrentManifestEntryMissing { .. } => { + formatter.write_str("retention retry root is absent from the current manifest") + } + Self::ManifestSuccessorMismatch { .. } => { + formatter.write_str("retention manifest entry disagrees with the candidate root") + } + Self::CurrentManifestEntryMismatch { .. } => { + formatter.write_str("retention retry disagrees with the current manifest entry") + } + Self::UnexpectedNamespaceSuccessor { .. } => { + formatter.write_str("new retention namespace candidate is not generation one") + } + Self::ManifestEntryIndex { index, entry_count } => write!( + formatter, + "retention manifest update index {index} exceeds {entry_count} entries" + ), + Self::LivenessGeneration { source } => write!(formatter, "{source}"), + Self::RootGeneration { source } => write!(formatter, "{source}"), + Self::EntryAllocation { .. } => { + formatter.write_str("retention successor entry allocation failed") + } + Self::Manifest { .. } => { + formatter.write_str("retention successor manifest admission failed") + } + Self::ManifestEncoding { .. } => { + formatter.write_str("retention successor manifest encoding failed") + } + Self::ManifestLengthOverflow { observed } => write!( + formatter, + "retention manifest byte length {observed} exceeds the protocol integer domain" + ), + Self::ManifestLength { .. } => { + formatter.write_str("retention successor manifest length admission failed") + } + Self::Head { .. } => formatter.write_str("retention successor head admission failed"), + } + } +} + +impl Error for RetentionPublicationPreparationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LivenessGeneration { source } => Some(source), + Self::RootGeneration { source } => Some(source), + Self::EntryAllocation { source } => Some(source), + Self::Manifest { source } => Some(source), + Self::ManifestEncoding { source } => Some(source), + Self::ManifestLength { source } => Some(source), + Self::Head { source } => Some(source), + Self::CurrentManifestRequired { .. } + | Self::CurrentManifestEntryMissing { .. } + | Self::ManifestSuccessorMismatch { .. } + | Self::CurrentManifestEntryMismatch { .. } + | Self::UnexpectedNamespaceSuccessor { .. } + | Self::ManifestEntryIndex { .. } + | Self::ManifestLengthOverflow { .. } => None, + } + } +} diff --git a/src/adapters/retention/successor_manifest.rs b/src/adapters/retention/successor_manifest.rs new file mode 100644 index 0000000..d87b0d6 --- /dev/null +++ b/src/adapters/retention/successor_manifest.rs @@ -0,0 +1,146 @@ +//! This boundary module owns candidate-to-manifest successor binding. + +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionPublicationPreparationError, + manifest_entry_update::{self, ManifestEntryUpdate}, +}; +use crate::{LivenessGeneration, RetentionManifest, RetentionManifestEntry, RootGeneration}; + +pub(super) fn build( + candidate: &AdmittedRetentionRoot<'_>, + current: Option<&AdmittedRetentionManifest<'_>>, +) -> Result { + let entry = candidate_entry(candidate); + let Some(current) = current else { + require_initial(candidate)?; + let entries = + manifest_entry_update::apply(&[], ManifestEntryUpdate::Insert { index: 0, entry })?; + return RetentionManifest::new(LivenessGeneration::INITIAL, None, entries) + .map_err(|source| RetentionPublicationPreparationError::Manifest { source }); + }; + let generation = current + .manifest() + .generation() + .successor() + .map_err(|source| RetentionPublicationPreparationError::LivenessGeneration { source })?; + let namespace = entry.namespace(); + let entries = current.manifest().entries(); + let update = match entries.binary_search_by_key(&namespace, |item| item.namespace()) { + Ok(index) => { + let selected = entries.get(index).copied().ok_or( + RetentionPublicationPreparationError::ManifestEntryIndex { + index, + entry_count: entries.len(), + }, + )?; + require_successor(selected, candidate)?; + ManifestEntryUpdate::Replace { index, entry } + } + Err(index) => { + require_initial(candidate)?; + ManifestEntryUpdate::Insert { index, entry } + } + }; + let entries = manifest_entry_update::apply(entries, update)?; + RetentionManifest::new(generation, Some(current.digest()), entries) + .map_err(|source| RetentionPublicationPreparationError::Manifest { source }) +} + +pub(super) fn require_current_selection( + candidate: &AdmittedRetentionRoot<'_>, + current: Option<&AdmittedRetentionManifest<'_>>, +) -> Result<(), RetentionPublicationPreparationError> { + let namespace = candidate.root().namespace().digest(); + let current = current + .ok_or(RetentionPublicationPreparationError::CurrentManifestRequired { namespace })?; + let entry = current + .manifest() + .entries() + .binary_search_by_key(&namespace, |item| item.namespace()) + .ok() + .and_then(|index| current.manifest().entries().get(index)) + .copied() + .ok_or_else( + || RetentionPublicationPreparationError::CurrentManifestEntryMissing { + namespace, + generation: candidate.root().generation(), + digest: candidate.digest(), + }, + )?; + if entry.root_generation() == candidate.root().generation() + && entry.root_digest() == candidate.digest() + { + Ok(()) + } else { + Err(current_mismatch(entry, candidate)) + } +} + +fn require_initial( + candidate: &AdmittedRetentionRoot<'_>, +) -> Result<(), RetentionPublicationPreparationError> { + if candidate.root().generation() == RootGeneration::INITIAL + && candidate.root().predecessor().is_none() + { + Ok(()) + } else { + Err( + RetentionPublicationPreparationError::UnexpectedNamespaceSuccessor { + namespace: candidate.root().namespace().digest(), + generation: candidate.root().generation(), + predecessor: candidate.root().predecessor(), + }, + ) + } +} + +fn require_successor( + current: RetentionManifestEntry, + candidate: &AdmittedRetentionRoot<'_>, +) -> Result<(), RetentionPublicationPreparationError> { + let expected_generation = current + .root_generation() + .successor() + .map_err(|source| RetentionPublicationPreparationError::RootGeneration { source })?; + if candidate.root().generation() == expected_generation + && candidate.root().predecessor() == Some(current.root_digest()) + { + Ok(()) + } else { + Err(successor_mismatch(current, candidate)) + } +} + +fn candidate_entry(candidate: &AdmittedRetentionRoot<'_>) -> RetentionManifestEntry { + RetentionManifestEntry::new( + candidate.root().namespace().digest(), + candidate.root().generation(), + candidate.digest(), + ) +} + +const fn successor_mismatch( + current: RetentionManifestEntry, + candidate: &AdmittedRetentionRoot<'_>, +) -> RetentionPublicationPreparationError { + RetentionPublicationPreparationError::ManifestSuccessorMismatch { + namespace: current.namespace(), + current_generation: current.root_generation(), + current_digest: current.root_digest(), + candidate_generation: candidate.root().generation(), + candidate_predecessor: candidate.root().predecessor(), + } +} + +const fn current_mismatch( + current: RetentionManifestEntry, + candidate: &AdmittedRetentionRoot<'_>, +) -> RetentionPublicationPreparationError { + RetentionPublicationPreparationError::CurrentManifestEntryMismatch { + namespace: current.namespace(), + current_generation: current.root_generation(), + current_digest: current.root_digest(), + candidate_generation: candidate.root().generation(), + candidate_digest: candidate.digest(), + } +} diff --git a/src/lib.rs b/src/lib.rs index bda7161..624ef17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,9 +26,10 @@ //! decoding, storage-independent expected-state transition planning, //! deterministic bounded closure verification against a pinned catalog, and a //! combined transition preflight proof and exact publication phase vocabulary -//! with a blocking storage capability port are available. Retention -//! publication orchestration, filesystem execution, recovery, and garbage -//! collection remain intentionally absent. +//! with a blocking storage capability port are available. Storage-independent +//! preparation binds preflight to exact canonical manifest and head successors. +//! Retention publication orchestration, filesystem execution, recovery, and +//! garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -109,12 +110,14 @@ pub use adapters::{ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationPhase, - RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, - RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, - preflight_retention_transition, verify_retention_closure, + PreparedRetentionPublication, RetentionClosureVerificationError, RetentionHeadDecodeError, + RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, + RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationPreparationError, RetentionPublicationStorage, RetentionRootDecodeError, + RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, + plan_retention_transition, preflight_retention_transition, prepare_retention_publication, + verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/src/retention/liveness_generation.rs b/src/retention/liveness_generation.rs index 3936c5d..0f67ec9 100644 --- a/src/retention/liveness_generation.rs +++ b/src/retention/liveness_generation.rs @@ -13,6 +13,9 @@ use super::LivenessGenerationError; pub struct LivenessGeneration(NonZeroU64); impl LivenessGeneration { + /// First global retention liveness generation. + pub const INITIAL: Self = Self(NonZeroU64::MIN); + /// Admits one positive liveness generation. /// /// # Errors diff --git a/tests/retention_publication_preparation.rs b/tests/retention_publication_preparation.rs new file mode 100644 index 0000000..d18b19b --- /dev/null +++ b/tests/retention_publication_preparation.rs @@ -0,0 +1,11 @@ +//! Retention publication preparation laws. + +#[path = "retention_publication_preparation/fixture.rs"] +pub mod fixture; +#[path = "retention_publication_preparation/initial_laws.rs"] +mod initial_laws; +#[path = "retention_publication_preparation/refusal_laws.rs"] +mod refusal_laws; +#[path = "retention_publication_preparation/successor_laws.rs"] +mod successor_laws; +mod support; diff --git a/tests/retention_publication_preparation/fixture.rs b/tests/retention_publication_preparation/fixture.rs new file mode 100644 index 0000000..f171d23 --- /dev/null +++ b/tests/retention_publication_preparation/fixture.rs @@ -0,0 +1,121 @@ +//! Shared admitted retention publication preparation fixtures. + +use std::error::Error; + +use keep::{ + AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, CanonicalRetentionRoot, + CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, + RetentionNamespace, RetentionPolicy, RetentionRoot, SegmentReadPolicy, SegmentRecordLimit, +}; + +use crate::support::decode_hex; + +/// Frozen canonical generation-one root. +pub const ROOT_HEX: &str = include_str!("../../conformance/segment-store/v2/one-anchor-root.hex"); +/// Frozen canonical generation-one manifest. +pub const MANIFEST_HEX: &str = + include_str!("../../conformance/segment-store/v2/one-root-manifest.hex"); +/// Frozen canonical generation-one retention head. +pub const HEAD_HEX: &str = include_str!("../../conformance/segment-store/v2/one-root-head.hex"); +const SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const CATALOG_HEAD_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-head.hex"); + +/// Decodes the frozen root transport. +/// +/// # Errors +/// +/// Returns the exact fixture transport refusal. +pub fn root_bytes() -> Result, Box> { + fixture(ROOT_HEX) +} + +/// Decodes the frozen manifest transport. +/// +/// # Errors +/// +/// Returns the exact fixture transport refusal. +pub fn manifest_bytes() -> Result, Box> { + fixture(MANIFEST_HEX) +} + +/// Builds the exact semantic successor of one admitted root. +/// +/// # Errors +/// +/// Returns the exact generation, semantic-root, or encoding refusal. +pub fn successor_root( + current: &AdmittedRetentionRoot<'_>, +) -> Result> { + let root = RetentionRoot::new( + current.root().namespace().clone(), + current.root().generation().successor()?, + RetentionPolicy::new(current.root().profile(), current.root().limits()), + Some(current.digest()), + current.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +/// Builds a generation-one root for another namespace. +/// +/// # Errors +/// +/// Returns the exact namespace, semantic-root, or encoding refusal. +pub fn initial_root( + namespace: &[u8], + template: &AdmittedRetentionRoot<'_>, +) -> Result> { + let root = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + keep::RootGeneration::INITIAL, + RetentionPolicy::new(template.root().profile(), template.root().limits()), + None, + template.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +/// Runs one operation against the frozen one-zero catalog snapshot. +/// +/// # Errors +/// +/// Returns the exact fixture, segment, catalog, head, or snapshot refusal. +pub fn with_snapshot( + operation: impl FnOnce(&CatalogSnapshot<'_, '_, '_>) -> T, +) -> Result> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog_bytes = fixture(CATALOG_HEX)?; + let head_bytes = fixture(CATALOG_HEAD_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(operation(&snapshot)) +} + +/// Decodes one LF-terminated lowercase hexadecimal fixture. +/// +/// # Errors +/// +/// Returns a framing or hexadecimal transport refusal. +pub fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} + +fn admitted_catalog<'catalog, 'records>( + catalog_bytes: &'catalog [u8], + segments: &'records [AdmittedSegment<'records>], +) -> Result, Box> { + ChecksummedCatalog::decode(catalog_bytes)? + .admit(segments) + .map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/retention_publication_preparation/initial_laws.rs b/tests/retention_publication_preparation/initial_laws.rs new file mode 100644 index 0000000..93fbea4 --- /dev/null +++ b/tests/retention_publication_preparation/initial_laws.rs @@ -0,0 +1,35 @@ +//! Initial retention publication preparation laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionRoot, RetentionGenerationExpectation, preflight_retention_transition, + prepare_retention_publication, +}; + +use super::fixture::{HEAD_HEX, MANIFEST_HEX, fixture, root_bytes, with_snapshot}; + +#[test] +fn initial_preparation_reproduces_frozen_manifest_and_head() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, None)?; + let publication = preparation + .publication() + .ok_or("initial transition did not prepare publication")?; + + assert_eq!(preparation.candidate().encoded(), root_bytes); + assert_eq!(publication.manifest().encoded(), fixture(MANIFEST_HEX)?); + assert_eq!(publication.head().encoded().as_slice(), fixture(HEAD_HEX)?); + assert_eq!(preparation.closure().usage().node_count(), 2); + Ok(()) +} diff --git a/tests/retention_publication_preparation/refusal_laws.rs b/tests/retention_publication_preparation/refusal_laws.rs new file mode 100644 index 0000000..d1b4018 --- /dev/null +++ b/tests/retention_publication_preparation/refusal_laws.rs @@ -0,0 +1,78 @@ +//! Retention publication preparation refusal laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, LivenessGeneration, LivenessGenerationError, + RetentionGenerationExpectation, RetentionManifest, RetentionPublicationPreparationError, + preflight_retention_transition, prepare_retention_publication, +}; + +use super::fixture::{initial_root, manifest_bytes, root_bytes, with_snapshot}; +use crate::support::require_error; + +#[test] +fn manifest_disagreement_refuses_before_global_artifact_construction() -> Result<(), Box> +{ + let root_bytes = root_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let error = require_error( + prepare_retention_publication(preflight, Some(¤t_manifest)), + "manifest disagreement prepared a publication", + )?; + + assert!(matches!( + error, + RetentionPublicationPreparationError::ManifestSuccessorMismatch { .. } + )); + Ok(()) +} + +#[test] +fn exhausted_liveness_generation_refuses_before_entry_replacement() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let template = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let maximum_manifest = RetentionManifest::new( + LivenessGeneration::new(u64::MAX)?, + Some(current_manifest.digest()), + current_manifest.manifest().entries().to_vec(), + )?; + let maximum_bytes = keep::CanonicalRetentionManifest::from_manifest(&maximum_manifest)?; + let maximum = AdmittedRetentionManifest::decode(maximum_bytes.encoded())?; + let candidate_bytes = initial_root(b"exhaustion-candidate", &template)?; + let candidate = AdmittedRetentionRoot::decode(candidate_bytes.encoded())?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let error = require_error( + prepare_retention_publication(preflight, Some(&maximum)), + "exhausted liveness generation prepared a publication", + )?; + + assert!(matches!( + error, + RetentionPublicationPreparationError::LivenessGeneration { + source: LivenessGenerationError::Exhausted { current: u64::MAX } + } + )); + Ok(()) +} diff --git a/tests/retention_publication_preparation/successor_laws.rs b/tests/retention_publication_preparation/successor_laws.rs new file mode 100644 index 0000000..35172c4 --- /dev/null +++ b/tests/retention_publication_preparation/successor_laws.rs @@ -0,0 +1,117 @@ +//! Successor retention publication preparation laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, + RetentionGenerationExpectation, RootGeneration, preflight_retention_transition, + prepare_retention_publication, +}; + +use super::fixture::{initial_root, manifest_bytes, root_bytes, successor_root, with_snapshot}; + +#[test] +fn successor_replaces_only_the_selected_manifest_entry() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate_bytes = successor_root(¤t)?; + let candidate = AdmittedRetentionRoot::decode(candidate_bytes.encoded())?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let candidate_digest = candidate.digest(); + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Current(RootGeneration::INITIAL), + Some(¤t), + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let publication = preparation + .publication() + .ok_or("successor transition did not prepare publication")?; + let manifest = AdmittedRetentionManifest::decode(publication.manifest().encoded())?; + let head = ChecksummedRetentionHead::decode(publication.head().encoded())?; + let entry = manifest + .manifest() + .entries() + .first() + .ok_or("successor manifest omitted the namespace")?; + + assert_eq!(manifest.manifest().generation().get(), 2); + assert_eq!( + manifest.manifest().predecessor(), + Some(current_manifest.digest()) + ); + assert_eq!(entry.root_generation().get(), 2); + assert_eq!(entry.root_digest(), candidate_digest); + assert_eq!(head.head().manifest_digest(), manifest.digest()); + Ok(()) +} + +#[test] +fn new_namespace_is_inserted_without_changing_existing_entry() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let template = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate_bytes = initial_root(b"second-namespace", &template)?; + let candidate = AdmittedRetentionRoot::decode(candidate_bytes.encoded())?; + let candidate_namespace = candidate.root().namespace().digest(); + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let existing = *current_manifest + .manifest() + .entries() + .first() + .ok_or("fixture manifest omitted its root")?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let publication = preparation + .publication() + .ok_or("new namespace did not prepare publication")?; + let manifest = AdmittedRetentionManifest::decode(publication.manifest().encoded())?; + + assert_eq!(manifest.manifest().entry_count(), 2); + assert!(manifest.manifest().entries().contains(&existing)); + assert!( + manifest + .manifest() + .entries() + .iter() + .any(|entry| entry.namespace() == candidate_namespace) + ); + Ok(()) +} + +#[test] +fn exact_retry_prepares_no_new_global_artifacts() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + + assert!(preparation.publication().is_none()); + assert_eq!(preparation.candidate().digest(), current.digest()); + assert_eq!(preparation.closure().usage().node_count(), 2); + Ok(()) +} From ef6668478ea181d9a99e21916a162068926bf57b Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 00:09:58 -0700 Subject: [PATCH 23/50] Fix: Preserve retention transition coordinates --- CHANGELOG.md | 3 +- README.md | 7 ++-- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 4 +- .../retention/prepared_publication.rs | 22 ++++++++++- .../retention/publication_preparation.rs | 18 +++++++-- .../retention/transition_preflight.rs | 39 +++++++++++++++++-- tests/retention_preflight.rs | 5 +++ .../successor_laws.rs | 15 +++++++ 9 files changed, 101 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 766e479..91cc9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - preparation binds preflight to exact canonical manifest and head successors. + preparation preserves expected and observed namespace generations while + binding preflight to exact canonical manifest and head successors. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 38623fb..82d0d2c 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Storage-independent preparation also binds preflight to exact -canonical manifest and head successors. Publication orchestration, filesystem -execution, recovery, compaction, and garbage collection remain planned. +implemented. Storage-independent preparation preserves expected and observed +namespace generations while binding preflight to canonical manifest and head +successors. Publication orchestration, filesystem execution, recovery, +compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 58b56b1..a747567 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning, generation-before-closure preflight, and preserved coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index cb9cc03..1f37cec 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -251,8 +251,8 @@ live set. `preflight_retention_transition` combines steps 3 and 4 without I/O, returning publish or already-committed only after generation and closure verification. `prepare_retention_publication` binds that proof to the current manifest, -refuses incoherent root coordinates, and derives exact canonical successors; -exact retry produces no new global artifacts. +preserves expected and observed generations, refuses incoherent coordinates, +and derives exact canonical successors; exact retry creates no global artifacts. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs index 6eca896..a2a7abb 100644 --- a/src/adapters/retention/prepared_publication.rs +++ b/src/adapters/retention/prepared_publication.rs @@ -4,7 +4,7 @@ use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, VerifiedRetentionClosure, }; -use crate::LivenessGeneration; +use crate::{LivenessGeneration, RetentionGenerationExpectation, RootGeneration}; /// Canonical global artifacts ready for ordered storage execution. #[must_use = "prepared retention publication must be executed or handled explicitly"] @@ -48,12 +48,24 @@ impl PreparedRetentionPublication { #[must_use = "retention publication preparation must be handled explicitly"] #[derive(Debug)] pub struct RetentionPublicationPreparation<'encoded> { + expected: RetentionGenerationExpectation, + observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, publication: Option, } impl<'encoded> RetentionPublicationPreparation<'encoded> { + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Returns the namespace generation observed during transition planning. + pub const fn observed(&self) -> Option { + self.observed + } + /// Borrows the admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { &self.candidate @@ -70,11 +82,15 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { } pub(super) const fn publish( + expected: RetentionGenerationExpectation, + observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, publication: PreparedRetentionPublication, ) -> Self { Self { + expected, + observed, candidate, closure, publication: Some(publication), @@ -82,10 +98,14 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { } pub(super) const fn already_committed( + expected: RetentionGenerationExpectation, + observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, ) -> Self { Self { + expected, + observed, candidate, closure, publication: None, diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs index da1b6ee..aeae9d6 100644 --- a/src/adapters/retention/publication_preparation.rs +++ b/src/adapters/retention/publication_preparation.rs @@ -23,13 +23,23 @@ pub fn prepare_retention_publication<'encoded>( current_manifest: Option<&AdmittedRetentionManifest<'_>>, ) -> Result, RetentionPublicationPreparationError> { match preflight { - RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } => { + RetentionTransitionPreflight::AlreadyCommitted { + expected, + observed, + candidate, + closure, + } => { successor_manifest::require_current_selection(&candidate, current_manifest)?; Ok(RetentionPublicationPreparation::already_committed( - candidate, closure, + expected, observed, candidate, closure, )) } - RetentionTransitionPreflight::Publish { candidate, closure } => { + RetentionTransitionPreflight::Publish { + expected, + observed, + candidate, + closure, + } => { let semantic_manifest = successor_manifest::build(&candidate, current_manifest)?; let liveness_generation = semantic_manifest.generation(); let predecessor = semantic_manifest.predecessor(); @@ -48,6 +58,8 @@ pub fn prepare_retention_publication<'encoded>( let publication = PreparedRetentionPublication::new(manifest, head, liveness_generation); Ok(RetentionPublicationPreparation::publish( + expected, + observed, candidate, closure, publication, diff --git a/src/adapters/retention/transition_preflight.rs b/src/adapters/retention/transition_preflight.rs index 06992b6..ae9c0e7 100644 --- a/src/adapters/retention/transition_preflight.rs +++ b/src/adapters/retention/transition_preflight.rs @@ -4,8 +4,8 @@ use super::{ AdmittedRetentionRoot, RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, verify_retention_closure, }; -use crate::CatalogSnapshot; use crate::retention::RetentionGenerationExpectation; +use crate::{CatalogSnapshot, RootGeneration}; /// Complete storage-independent proof required before retention publication. #[must_use = "retention preflight must be consumed by publication or handled explicitly"] @@ -13,6 +13,10 @@ use crate::retention::RetentionGenerationExpectation; pub enum RetentionTransitionPreflight<'encoded> { /// The candidate is an exact successor whose verified closure must publish. Publish { + /// Caller-supplied expected namespace generation. + expected: RetentionGenerationExpectation, + /// Namespace generation observed during transition planning. + observed: Option, /// Fully admitted canonical candidate root. candidate: AdmittedRetentionRoot<'encoded>, /// Closure proof against the exact pinned catalog. @@ -20,6 +24,10 @@ pub enum RetentionTransitionPreflight<'encoded> { }, /// The exact candidate is current and its closure still verifies. AlreadyCommitted { + /// Caller-supplied expected namespace generation. + expected: RetentionGenerationExpectation, + /// Namespace generation observed during transition planning. + observed: Option, /// Fully admitted byte-identical current root. candidate: AdmittedRetentionRoot<'encoded>, /// Current closure proof against the exact pinned catalog. @@ -28,6 +36,20 @@ pub enum RetentionTransitionPreflight<'encoded> { } impl<'encoded> RetentionTransitionPreflight<'encoded> { + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + match self { + Self::Publish { expected, .. } | Self::AlreadyCommitted { expected, .. } => *expected, + } + } + + /// Returns the namespace generation observed during transition planning. + pub const fn observed(&self) -> Option { + match self { + Self::Publish { observed, .. } | Self::AlreadyCommitted { observed, .. } => *observed, + } + } + /// Borrows the fully admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { match self { @@ -61,6 +83,7 @@ pub fn preflight_retention_transition<'encoded>( candidate: AdmittedRetentionRoot<'encoded>, catalog: &CatalogSnapshot<'_, '_, '_>, ) -> Result, RetentionTransitionPreflightError> { + let observed = current.map(|root| root.root().generation()); let readiness = plan_retention_transition(expected, current, candidate) .map_err(|source| RetentionTransitionPreflightError::Transition { source })?; let closure = @@ -71,10 +94,20 @@ pub fn preflight_retention_transition<'encoded>( })?; Ok(match readiness { RetentionTransitionReadiness::Publish { candidate } => { - RetentionTransitionPreflight::Publish { candidate, closure } + RetentionTransitionPreflight::Publish { + expected, + observed, + candidate, + closure, + } } RetentionTransitionReadiness::AlreadyCommitted { candidate } => { - RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } + RetentionTransitionPreflight::AlreadyCommitted { + expected, + observed, + candidate, + closure, + } } }) } diff --git a/tests/retention_preflight.rs b/tests/retention_preflight.rs index 7eee6b5..774e098 100644 --- a/tests/retention_preflight.rs +++ b/tests/retention_preflight.rs @@ -44,11 +44,14 @@ fn publish_preflight_binds_generation_and_closure_proofs() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box })??; let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + assert_eq!( + preparation.expected(), + RetentionGenerationExpectation::Absent + ); + assert_eq!(preparation.observed(), None); let publication = preparation .publication() .ok_or("new namespace did not prepare publication")?; @@ -110,6 +120,11 @@ fn exact_retry_prepares_no_new_global_artifacts() -> Result<(), Box> let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + assert_eq!( + preparation.expected(), + RetentionGenerationExpectation::Absent + ); + assert_eq!(preparation.observed(), Some(RootGeneration::INITIAL)); assert!(preparation.publication().is_none()); assert_eq!(preparation.candidate().digest(), current.digest()); assert_eq!(preparation.closure().usage().node_count(), 2); From 78b0c54a047de562ef4fa914a0bc89db860b37f7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 00:24:29 -0700 Subject: [PATCH 24/50] Fix: Seal retention transition proofs --- CHANGELOG.md | 4 +- README.md | 8 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 2 +- src/adapters/retention.rs | 2 + .../retention/prepared_publication.rs | 10 +- .../retention/publication_preparation.rs | 20 ++-- .../retention/transition_disposition.rs | 11 ++ src/adapters/retention/transition_planner.rs | 9 +- .../retention/transition_preflight.rs | 100 ++++++++---------- .../retention/transition_readiness.rs | 84 +++++++++++---- src/lib.rs | 8 +- tests/retention_preflight.rs | 30 +++--- .../successor_laws.rs | 16 ++- tests/retention_transition.rs | 44 +++++--- 15 files changed, 213 insertions(+), 137 deletions(-) create mode 100644 src/adapters/retention/transition_disposition.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 91cc9aa..c0aedd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - preparation preserves expected and observed namespace generations while - binding preflight to exact canonical manifest and head successors. + unforgeable proof values preserve typed disposition plus expected and + observed generations through exact manifest and head preparation. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 82d0d2c..66bfd14 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Storage-independent preparation preserves expected and observed -namespace generations while binding preflight to canonical manifest and head -successors. Publication orchestration, filesystem execution, recovery, -compaction, and garbage collection remain planned. +implemented. Private-field proofs preserve typed disposition plus expected and +observed namespace generations through canonical manifest and head preparation. +Publication orchestration, filesystem execution, recovery, compaction, and +garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index a747567..7fe3852 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning, generation-before-closure preflight, and preserved coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable storage-independent readiness and preflight proofs with preserved disposition and coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 1f37cec..b04b150 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -249,7 +249,7 @@ transition. Keep never omits one failed member and continues with a smaller live set. `preflight_retention_transition` combines steps 3 and 4 without I/O, returning -publish or already-committed only after generation and closure verification. +an unforgeable typed disposition only after generation and closure verification. `prepare_retention_publication` binds that proof to the current manifest, preserves expected and observed generations, refuses incoherent coordinates, and derives exact canonical successors; exact retry creates no global artifacts. diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index addcd7c..4978032 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -45,6 +45,7 @@ mod root_header_decoder; mod root_integrity; mod root_semantic_header; mod successor_manifest; +mod transition_disposition; mod transition_error; mod transition_planner; mod transition_preflight; @@ -71,6 +72,7 @@ pub use publication_preparation_error::RetentionPublicationPreparationError; pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; +pub use transition_disposition::RetentionTransitionDisposition; pub use transition_error::RetentionTransitionError; pub use transition_planner::plan_retention_transition; pub use transition_preflight::{RetentionTransitionPreflight, preflight_retention_transition}; diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs index a2a7abb..d534fa6 100644 --- a/src/adapters/retention/prepared_publication.rs +++ b/src/adapters/retention/prepared_publication.rs @@ -2,7 +2,7 @@ use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - VerifiedRetentionClosure, + RetentionTransitionDisposition, VerifiedRetentionClosure, }; use crate::{LivenessGeneration, RetentionGenerationExpectation, RootGeneration}; @@ -48,6 +48,7 @@ impl PreparedRetentionPublication { #[must_use = "retention publication preparation must be handled explicitly"] #[derive(Debug)] pub struct RetentionPublicationPreparation<'encoded> { + disposition: RetentionTransitionDisposition, expected: RetentionGenerationExpectation, observed: Option, candidate: AdmittedRetentionRoot<'encoded>, @@ -56,6 +57,11 @@ pub struct RetentionPublicationPreparation<'encoded> { } impl<'encoded> RetentionPublicationPreparation<'encoded> { + /// Returns whether the candidate requires publication or is current. + pub const fn disposition(&self) -> RetentionTransitionDisposition { + self.disposition + } + /// Returns the caller-supplied expected namespace generation. pub const fn expected(&self) -> RetentionGenerationExpectation { self.expected @@ -89,6 +95,7 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { publication: PreparedRetentionPublication, ) -> Self { Self { + disposition: RetentionTransitionDisposition::Publish, expected, observed, candidate, @@ -104,6 +111,7 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { closure: VerifiedRetentionClosure, ) -> Self { Self { + disposition: RetentionTransitionDisposition::AlreadyCommitted, expected, observed, candidate, diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs index aeae9d6..01680f1 100644 --- a/src/adapters/retention/publication_preparation.rs +++ b/src/adapters/retention/publication_preparation.rs @@ -3,7 +3,8 @@ use super::{ AdmittedRetentionManifest, CanonicalRetentionHead, CanonicalRetentionManifest, PreparedRetentionPublication, RetentionPublicationPreparation, - RetentionPublicationPreparationError, RetentionTransitionPreflight, successor_manifest, + RetentionPublicationPreparationError, RetentionTransitionDisposition, + RetentionTransitionPreflight, successor_manifest, }; use crate::{RetentionHead, RetentionManifestLength}; @@ -22,24 +23,15 @@ pub fn prepare_retention_publication<'encoded>( preflight: RetentionTransitionPreflight<'encoded>, current_manifest: Option<&AdmittedRetentionManifest<'_>>, ) -> Result, RetentionPublicationPreparationError> { - match preflight { - RetentionTransitionPreflight::AlreadyCommitted { - expected, - observed, - candidate, - closure, - } => { + let (disposition, expected, observed, candidate, closure) = preflight.into_parts(); + match disposition { + RetentionTransitionDisposition::AlreadyCommitted => { successor_manifest::require_current_selection(&candidate, current_manifest)?; Ok(RetentionPublicationPreparation::already_committed( expected, observed, candidate, closure, )) } - RetentionTransitionPreflight::Publish { - expected, - observed, - candidate, - closure, - } => { + RetentionTransitionDisposition::Publish => { let semantic_manifest = successor_manifest::build(&candidate, current_manifest)?; let liveness_generation = semantic_manifest.generation(); let predecessor = semantic_manifest.predecessor(); diff --git a/src/adapters/retention/transition_disposition.rs b/src/adapters/retention/transition_disposition.rs new file mode 100644 index 0000000..257a36b --- /dev/null +++ b/src/adapters/retention/transition_disposition.rs @@ -0,0 +1,11 @@ +//! This boundary module owns retention transition disposition vocabulary. + +/// Storage-independent result of one admitted retention transition comparison. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionTransitionDisposition { + /// The candidate is an exact successor that still requires publication. + Publish, + /// The byte-identical candidate is already the selected current root. + AlreadyCommitted, +} diff --git a/src/adapters/retention/transition_planner.rs b/src/adapters/retention/transition_planner.rs index 894e0a4..d25e83e 100644 --- a/src/adapters/retention/transition_planner.rs +++ b/src/adapters/retention/transition_planner.rs @@ -20,15 +20,20 @@ pub fn plan_retention_transition<'encoded>( current: Option<&AdmittedRetentionRoot<'_>>, candidate: AdmittedRetentionRoot<'encoded>, ) -> Result, RetentionTransitionError> { + let observed = current.map(|root| root.root().generation()); if is_exact_replay(expected, current, &candidate)? { - return Ok(RetentionTransitionReadiness::AlreadyCommitted { candidate }); + return Ok(RetentionTransitionReadiness::already_committed( + expected, observed, candidate, + )); } require_expected_state(expected, current)?; match current { Some(current) => validate_successor(current, &candidate)?, None => validate_initial(&candidate)?, } - Ok(RetentionTransitionReadiness::Publish { candidate }) + Ok(RetentionTransitionReadiness::publish( + expected, observed, candidate, + )) } fn is_exact_replay( diff --git a/src/adapters/retention/transition_preflight.rs b/src/adapters/retention/transition_preflight.rs index ae9c0e7..de36960 100644 --- a/src/adapters/retention/transition_preflight.rs +++ b/src/adapters/retention/transition_preflight.rs @@ -1,67 +1,64 @@ //! This boundary module owns complete retention transition preflight. use super::{ - AdmittedRetentionRoot, RetentionTransitionPreflightError, RetentionTransitionReadiness, + AdmittedRetentionRoot, RetentionTransitionDisposition, RetentionTransitionPreflightError, VerifiedRetentionClosure, plan_retention_transition, verify_retention_closure, }; -use crate::retention::RetentionGenerationExpectation; -use crate::{CatalogSnapshot, RootGeneration}; +use crate::{CatalogSnapshot, RetentionGenerationExpectation, RootGeneration}; -/// Complete storage-independent proof required before retention publication. +/// Unforgeable storage-independent proof required before publication. #[must_use = "retention preflight must be consumed by publication or handled explicitly"] #[derive(Debug)] -pub enum RetentionTransitionPreflight<'encoded> { - /// The candidate is an exact successor whose verified closure must publish. - Publish { - /// Caller-supplied expected namespace generation. - expected: RetentionGenerationExpectation, - /// Namespace generation observed during transition planning. - observed: Option, - /// Fully admitted canonical candidate root. - candidate: AdmittedRetentionRoot<'encoded>, - /// Closure proof against the exact pinned catalog. - closure: VerifiedRetentionClosure, - }, - /// The exact candidate is current and its closure still verifies. - AlreadyCommitted { - /// Caller-supplied expected namespace generation. - expected: RetentionGenerationExpectation, - /// Namespace generation observed during transition planning. - observed: Option, - /// Fully admitted byte-identical current root. - candidate: AdmittedRetentionRoot<'encoded>, - /// Current closure proof against the exact pinned catalog. - closure: VerifiedRetentionClosure, - }, +pub struct RetentionTransitionPreflight<'encoded> { + disposition: RetentionTransitionDisposition, + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, } impl<'encoded> RetentionTransitionPreflight<'encoded> { + /// Returns whether the candidate requires publication or is current. + pub const fn disposition(&self) -> RetentionTransitionDisposition { + self.disposition + } + /// Returns the caller-supplied expected namespace generation. pub const fn expected(&self) -> RetentionGenerationExpectation { - match self { - Self::Publish { expected, .. } | Self::AlreadyCommitted { expected, .. } => *expected, - } + self.expected } /// Returns the namespace generation observed during transition planning. pub const fn observed(&self) -> Option { - match self { - Self::Publish { observed, .. } | Self::AlreadyCommitted { observed, .. } => *observed, - } + self.observed } /// Borrows the fully admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { - match self { - Self::Publish { candidate, .. } | Self::AlreadyCommitted { candidate, .. } => candidate, - } + &self.candidate } /// Returns the complete verified closure evidence. pub const fn closure(&self) -> VerifiedRetentionClosure { - match self { - Self::Publish { closure, .. } | Self::AlreadyCommitted { closure, .. } => *closure, - } + self.closure + } + + pub(super) fn into_parts( + self, + ) -> ( + RetentionTransitionDisposition, + RetentionGenerationExpectation, + Option, + AdmittedRetentionRoot<'encoded>, + VerifiedRetentionClosure, + ) { + ( + self.disposition, + self.expected, + self.observed, + self.candidate, + self.closure, + ) } } @@ -83,7 +80,6 @@ pub fn preflight_retention_transition<'encoded>( candidate: AdmittedRetentionRoot<'encoded>, catalog: &CatalogSnapshot<'_, '_, '_>, ) -> Result, RetentionTransitionPreflightError> { - let observed = current.map(|root| root.root().generation()); let readiness = plan_retention_transition(expected, current, candidate) .map_err(|source| RetentionTransitionPreflightError::Transition { source })?; let closure = @@ -92,22 +88,12 @@ pub fn preflight_retention_transition<'encoded>( source: Box::new(source), } })?; - Ok(match readiness { - RetentionTransitionReadiness::Publish { candidate } => { - RetentionTransitionPreflight::Publish { - expected, - observed, - candidate, - closure, - } - } - RetentionTransitionReadiness::AlreadyCommitted { candidate } => { - RetentionTransitionPreflight::AlreadyCommitted { - expected, - observed, - candidate, - closure, - } - } + let (disposition, expected, observed, candidate) = readiness.into_parts(); + Ok(RetentionTransitionPreflight { + disposition, + expected, + observed, + candidate, + closure, }) } diff --git a/src/adapters/retention/transition_readiness.rs b/src/adapters/retention/transition_readiness.rs index 2231f15..0fa9461 100644 --- a/src/adapters/retention/transition_readiness.rs +++ b/src/adapters/retention/transition_readiness.rs @@ -1,35 +1,83 @@ //! This boundary module owns admitted retention transition readiness. -use super::AdmittedRetentionRoot; +use super::{AdmittedRetentionRoot, RetentionTransitionDisposition}; +use crate::{RetentionGenerationExpectation, RootGeneration}; -/// Result of comparing one expected, observed, and candidate root. +/// Unforgeable result of comparing expected, observed, and candidate state. #[must_use] #[derive(Debug, Eq, PartialEq)] -pub enum RetentionTransitionReadiness<'encoded> { - /// The candidate is the exact next root and still requires publication. - Publish { - /// Fully admitted candidate root. - candidate: AdmittedRetentionRoot<'encoded>, - }, - /// The exact candidate bytes are already the current published root. - AlreadyCommitted { - /// Fully admitted byte-identical replay candidate. - candidate: AdmittedRetentionRoot<'encoded>, - }, +pub struct RetentionTransitionReadiness<'encoded> { + disposition: RetentionTransitionDisposition, + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, } impl<'encoded> RetentionTransitionReadiness<'encoded> { + /// Returns whether the candidate requires publication or is current. + pub const fn disposition(&self) -> RetentionTransitionDisposition { + self.disposition + } + + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Returns the namespace generation observed during transition planning. + pub const fn observed(&self) -> Option { + self.observed + } + /// Borrows the fully admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { - match self { - Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, - } + &self.candidate } /// Consumes the readiness proof and returns the admitted candidate root. pub fn into_candidate(self) -> AdmittedRetentionRoot<'encoded> { - match self { - Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, + self.candidate + } + + pub(super) const fn publish( + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, + ) -> Self { + Self { + disposition: RetentionTransitionDisposition::Publish, + expected, + observed, + candidate, } } + + pub(super) const fn already_committed( + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, + ) -> Self { + Self { + disposition: RetentionTransitionDisposition::AlreadyCommitted, + expected, + observed, + candidate, + } + } + + pub(super) fn into_parts( + self, + ) -> ( + RetentionTransitionDisposition, + RetentionGenerationExpectation, + Option, + AdmittedRetentionRoot<'encoded>, + ) { + ( + self.disposition, + self.expected, + self.observed, + self.candidate, + ) + } } diff --git a/src/lib.rs b/src/lib.rs index 624ef17..adbbffd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,10 +114,10 @@ pub use adapters::{ RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationPreparationError, RetentionPublicationStorage, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, - RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, - plan_retention_transition, preflight_retention_transition, prepare_retention_publication, - verify_retention_closure, + RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, + RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, + VerifiedRetentionClosure, plan_retention_transition, preflight_retention_transition, + prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_preflight.rs b/tests/retention_preflight.rs index 774e098..510a5b8 100644 --- a/tests/retention_preflight.rs +++ b/tests/retention_preflight.rs @@ -7,7 +7,7 @@ use std::error::Error; use keep::{ AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, RetentionClosureVerificationError, - RetentionGenerationExpectation, RetentionTransitionError, RetentionTransitionPreflight, + RetentionGenerationExpectation, RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflightError, RootGeneration, SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, preflight_retention_transition, }; @@ -46,15 +46,15 @@ fn publish_preflight_binds_generation_and_closure_proofs() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box RetentionGenerationExpectation::Absent ); assert_eq!(preparation.observed(), None); + assert_eq!( + preparation.disposition(), + RetentionTransitionDisposition::Publish + ); let publication = preparation .publication() .ok_or("new namespace did not prepare publication")?; @@ -125,6 +133,10 @@ fn exact_retry_prepares_no_new_global_artifacts() -> Result<(), Box> RetentionGenerationExpectation::Absent ); assert_eq!(preparation.observed(), Some(RootGeneration::INITIAL)); + assert_eq!( + preparation.disposition(), + RetentionTransitionDisposition::AlreadyCommitted + ); assert!(preparation.publication().is_none()); assert_eq!(preparation.candidate().digest(), current.digest()); assert_eq!(preparation.closure().usage().node_count(), 2); diff --git a/tests/retention_transition.rs b/tests/retention_transition.rs index a673e45..fc8064e 100644 --- a/tests/retention_transition.rs +++ b/tests/retention_transition.rs @@ -8,7 +8,7 @@ use std::io; use keep::{ AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionGenerationExpectation, - RetentionNamespace, RetentionRoot, RetentionRootDigest, RetentionTransitionReadiness, + RetentionNamespace, RetentionRoot, RetentionRootDigest, RetentionTransitionDisposition, RootGeneration, plan_retention_transition, }; @@ -20,11 +20,13 @@ fn absent_namespace_admits_only_the_initial_candidate() -> Result<(), Box Date: Thu, 30 Jul 2026 00:34:55 -0700 Subject: [PATCH 25/50] Add: Expose verified retention anchor digest --- CHANGELOG.md | 4 ++-- README.md | 8 ++++---- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 6 +++--- src/adapters/retention/admitted_root.rs | 10 +++++++++- src/adapters/retention/root_decoder.rs | 7 ++++++- src/adapters/retention/root_integrity.rs | 5 +++-- src/lib.rs | 8 ++++---- src/retention/anchor_set_digest.rs | 18 ++++++++++++++++++ src/retention/mod.rs | 2 ++ tests/retention_root_decoding.rs | 7 +++++++ 11 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 src/retention/anchor_set_digest.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c0aedd9..e45a7c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - unforgeable proof values preserve typed disposition plus expected and - observed generations through exact manifest and head preparation. + unforgeable proofs preserve disposition, expected and observed generations, + and the verified anchor-set digest through manifest and head preparation. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 66bfd14..8cf827f 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Private-field proofs preserve typed disposition plus expected and -observed namespace generations through canonical manifest and head preparation. -Publication orchestration, filesystem execution, recovery, compaction, and -garbage collection remain planned. +implemented. Private-field proofs preserve typed disposition, expected and +observed namespace generations, and the verified anchor-set digest through +canonical manifest and head preparation. Publication orchestration, filesystem +execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 7fe3852..41ba7b1 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -9,7 +9,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable storage-independent readiness and preflight proofs with preserved disposition and coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index b04b150..dddb002 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root, manifest, and head codecs, -storage-independent expected-state transition planning, deterministic closure -verification, and a blocking publication storage capability port. Publication +implements root, manifest, and head codecs with a typed verified anchor-set +digest, expected-state transition planning, deterministic closure verification, +and a blocking publication storage capability port. Publication orchestration, filesystem execution, recovery, and garbage collection remain absent. diff --git a/src/adapters/retention/admitted_root.rs b/src/adapters/retention/admitted_root.rs index ac0f33f..952b690 100644 --- a/src/adapters/retention/admitted_root.rs +++ b/src/adapters/retention/admitted_root.rs @@ -1,7 +1,7 @@ //! This boundary module owns one decoded and admitted retention root. use super::{RetentionRootDecodeError, root_decoder}; -use crate::{RetentionRoot, RetentionRootDigest}; +use crate::{RetentionAnchorSetDigest, RetentionRoot, RetentionRootDigest}; /// Borrowed canonical bytes paired with their admitted semantic root. /// @@ -14,6 +14,7 @@ use crate::{RetentionRoot, RetentionRootDigest}; pub struct AdmittedRetentionRoot<'encoded> { encoded: &'encoded [u8], root: RetentionRoot, + anchor_set_digest: RetentionAnchorSetDigest, digest: RetentionRootDigest, } @@ -39,6 +40,11 @@ impl<'encoded> AdmittedRetentionRoot<'encoded> { &self.root } + /// Returns the verified canonical anchor-set digest. + pub const fn anchor_set_digest(&self) -> RetentionAnchorSetDigest { + self.anchor_set_digest + } + /// Returns the verified canonical root digest. pub const fn digest(&self) -> RetentionRootDigest { self.digest @@ -47,11 +53,13 @@ impl<'encoded> AdmittedRetentionRoot<'encoded> { pub(super) const fn admitted( encoded: &'encoded [u8], root: RetentionRoot, + anchor_set_digest: RetentionAnchorSetDigest, digest: RetentionRootDigest, ) -> Self { Self { encoded, root, + anchor_set_digest, digest, } } diff --git a/src/adapters/retention/root_decoder.rs b/src/adapters/retention/root_decoder.rs index cf4a7b1..454f51b 100644 --- a/src/adapters/retention/root_decoder.rs +++ b/src/adapters/retention/root_decoder.rs @@ -28,7 +28,11 @@ pub(super) fn decode( observed: encoded.len(), }, )?; - root_integrity::verify_anchor_set(header.anchor_count, anchor_bytes, header.anchor_set_digest)?; + let anchor_set_digest = root_integrity::verify_anchor_set( + header.anchor_count, + anchor_bytes, + header.anchor_set_digest, + )?; let admitted_header = root_semantic_header::admit(&header)?; let namespace = RetentionNamespace::try_from(namespace_bytes) .map_err(|source| RetentionRootDecodeError::Namespace { source })?; @@ -45,6 +49,7 @@ pub(super) fn decode( Ok(AdmittedRetentionRoot::admitted( encoded, root, + anchor_set_digest, RetentionRootDigest::from_hash(digest), )) } diff --git a/src/adapters/retention/root_integrity.rs b/src/adapters/retention/root_integrity.rs index 71f842d..d753751 100644 --- a/src/adapters/retention/root_integrity.rs +++ b/src/adapters/retention/root_integrity.rs @@ -1,6 +1,7 @@ //! This boundary module owns retention root digest and checksum verification. use super::RetentionRootDecodeError; +use crate::RetentionAnchorSetDigest; pub(super) fn verify( encoded: &[u8], @@ -45,14 +46,14 @@ pub(super) fn verify_anchor_set( anchor_count: u32, anchors: &[u8], observed: [u8; 32], -) -> Result<(), RetentionRootDecodeError> { +) -> Result { let mut hasher = blake3::Hasher::new(); hasher.update(b"keep.retention-anchor-set/v2\0"); hasher.update(&anchor_count.to_be_bytes()); hasher.update(anchors); let expected = *hasher.finalize().as_bytes(); if observed == expected { - Ok(()) + Ok(RetentionAnchorSetDigest::from_verified(expected)) } else { Err(RetentionRootDecodeError::AnchorSetDigestMismatch { expected, observed }) } diff --git a/src/lib.rs b/src/lib.rs index adbbffd..13ad241 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -141,10 +141,10 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureCounter, RetentionClosureDigest, RetentionClosureLimit, - RetentionClosureLimitError, RetentionClosureLimits, RetentionClosureUsage, - RetentionGenerationExpectation, RetentionHead, RetentionHeadError, RetentionManifest, - RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, + RetentionAnchorSetDigest, RetentionClosureCounter, RetentionClosureDigest, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, + RetentionClosureUsage, RetentionGenerationExpectation, RetentionHead, RetentionHeadError, + RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, diff --git a/src/retention/anchor_set_digest.rs b/src/retention/anchor_set_digest.rs new file mode 100644 index 0000000..4f31742 --- /dev/null +++ b/src/retention/anchor_set_digest.rs @@ -0,0 +1,18 @@ +//! This module owns one verified version-2 retention anchor-set digest. + +/// BLAKE3-256 digest of one canonical retention anchor set. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionAnchorSetDigest([u8; 32]); + +impl RetentionAnchorSetDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(crate) const fn from_verified(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index a9da928..481b794 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -6,6 +6,7 @@ //! collection. mod anchor; +mod anchor_set_digest; mod closure_counter; mod closure_digest; mod closure_limit; @@ -36,6 +37,7 @@ mod root_generation; mod root_generation_error; pub use anchor::RetentionAnchor; +pub use anchor_set_digest::RetentionAnchorSetDigest; pub use closure_counter::RetentionClosureCounter; pub use closure_digest::RetentionClosureDigest; pub use closure_limit::RetentionClosureLimit; diff --git a/tests/retention_root_decoding.rs b/tests/retention_root_decoding.rs index 9405541..be1a4d3 100644 --- a/tests/retention_root_decoding.rs +++ b/tests/retention_root_decoding.rs @@ -8,6 +8,7 @@ use keep::{AdmittedRetentionRoot, RetentionRootDecodeError}; const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); const ANCHOR_SET_DIGEST_OFFSET: usize = 148; +const ANCHOR_SET_DIGEST_END: usize = 180; const ANCHOR_BODY_OFFSET: usize = 195; const ROOT_DIGEST_OFFSET: usize = 314; const CHECKSUM_OFFSET: usize = 346; @@ -21,6 +22,12 @@ fn frozen_root_decodes_to_one_complete_semantic_generation() assert_eq!(admitted.root().namespace().as_bytes(), &[0x00, 0x2f, 0xff]); assert_eq!(admitted.root().generation().get(), 1); assert_eq!(admitted.root().anchor_count(), 1); + assert_eq!( + admitted.anchor_set_digest().as_bytes(), + bytes + .get(ANCHOR_SET_DIGEST_OFFSET..ANCHOR_SET_DIGEST_END) + .ok_or_else(|| io::Error::other("frozen root lacks its anchor-set digest"))? + ); assert_eq!( admitted.digest().as_bytes(), bytes.get(314..346).ok_or_else(|| { From 1635bc8eac4c429f1dd450d32c3881a0cc41b6eb Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 00:58:35 -0700 Subject: [PATCH 26/50] Add: Execute retention publication --- CHANGELOG.md | 4 +- README.md | 8 +- docs/formats/segment-store-v2/README.md | 11 +- docs/formats/segment-store-v2/requirements.md | 6 +- docs/formats/segment-store-v2/retention.md | 10 +- src/adapters/retention.rs | 8 + .../retention/prepared_publication.rs | 24 ++- src/adapters/retention/publication_error.rs | 61 +++++++ .../retention/publication_execution.rs | 153 ++++++++++++++++++ src/adapters/retention/publication_outcome.rs | 11 ++ .../retention/publication_preparation.rs | 5 +- src/adapters/retention/publication_receipt.rs | 127 +++++++++++++++ src/adapters/retention/publication_storage.rs | 25 ++- src/adapters/retention/successor_manifest.rs | 8 +- src/lib.rs | 14 +- tests/retention_publication_execution.rs | 144 +++++++++++++++++ .../refusal_laws.rs | 94 +++++++++++ .../recording_storage.rs | 146 ++++++++++++----- 18 files changed, 784 insertions(+), 75 deletions(-) create mode 100644 src/adapters/retention/publication_error.rs create mode 100644 src/adapters/retention/publication_execution.rs create mode 100644 src/adapters/retention/publication_outcome.rs create mode 100644 src/adapters/retention/publication_receipt.rs create mode 100644 tests/retention_publication_execution.rs create mode 100644 tests/retention_publication_execution/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e45a7c4..7122bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - unforgeable proofs preserve disposition, expected and observed generations, - and the verified anchor-set digest through manifest and head preparation. + authority-revalidated orchestration executes every phase and returns an + unforgeable complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 8cf827f..331ab28 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Private-field proofs preserve typed disposition, expected and -observed namespace generations, and the verified anchor-set digest through -canonical manifest and head preparation. Publication orchestration, filesystem -execution, recovery, compaction, and garbage collection remain planned. +implemented. Private-field proofs retain every receipt coordinate. Ordered +storage-port orchestration revalidates current authority, executes all 17 +durability phases, and returns a consequential complete-coordinate receipt. +Filesystem execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 54c7c1d..150022b 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -79,9 +79,10 @@ planning, deterministic bounded closure verification against one pinned catalog, their combined preflight proof, and the exact 17-phase publication vocabulary with a blocking storage capability port are available. Storage-independent preparation derives exact canonical manifest and head -successors from coherent preflight and current-manifest evidence. Publication -orchestration and production filesystem retention publication, recovery, -migration, and garbage collection do not exist yet. Requirements that remain -planned or in progress in issue #19 or issue #21 are not complete implementation -evidence. A store must refuse unsupported version-2 state until the relevant +successors from coherent preflight and current-manifest evidence. Ordered +storage-port orchestration revalidates authority and returns a complete receipt. +Production filesystem retention publication, recovery, migration, and garbage +collection do not exist yet. Requirements still in progress in issue #19 or +issue #21 are not complete evidence. A store must refuse version-2 state until +the relevant corruption, model-based, crash-injection, recovery, and fuzz evidence exists. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 41ba7b1..53b12d3 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,12 +12,12 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable storage-independent readiness and preflight proofs with preserved disposition and coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; filesystem evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; filesystem retry remains | In progress in #19 | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index dddb002..c82960d 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -248,11 +248,11 @@ authenticated reconstruction, and canonical digest defined by transition. Keep never omits one failed member and continues with a smaller live set. -`preflight_retention_transition` combines steps 3 and 4 without I/O, returning -an unforgeable typed disposition only after generation and closure verification. -`prepare_retention_publication` binds that proof to the current manifest, -preserves expected and observed generations, refuses incoherent coordinates, -and derives exact canonical successors; exact retry creates no global artifacts. +Preflight verifies steps 3 and 4 without I/O; preparation binds that proof to +the current manifest and derives exact canonical successors. +`execute_retention_publication` revalidates current authority, executes all 17 +ordered durability phases, and returns the complete receipt only after cleanup; +exact already-committed retry revalidates authority and performs no mutation. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 4978032..10943c0 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -30,9 +30,13 @@ mod manifest_integrity; mod manifest_semantic_header; mod namespace_admission; mod prepared_publication; +mod publication_error; +mod publication_execution; +mod publication_outcome; mod publication_phase; mod publication_preparation; mod publication_preparation_error; +mod publication_receipt; mod publication_storage; mod root_anchor_decoder; mod root_decode_error; @@ -66,9 +70,13 @@ pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use namespace_admission::RetentionNamespaceAdmission; pub use prepared_publication::{PreparedRetentionPublication, RetentionPublicationPreparation}; +pub use publication_error::RetentionPublicationError; +pub use publication_execution::execute_retention_publication; +pub use publication_outcome::RetentionPublicationOutcome; pub use publication_phase::RetentionPublicationPhase; pub use publication_preparation::prepare_retention_publication; pub use publication_preparation_error::RetentionPublicationPreparationError; +pub use publication_receipt::RetentionPublicationReceipt; pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs index d534fa6..fdcf95b 100644 --- a/src/adapters/retention/prepared_publication.rs +++ b/src/adapters/retention/prepared_publication.rs @@ -1,9 +1,10 @@ //! This boundary module owns storage-ready retention publication artifacts. use super::{ - AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionTransitionDisposition, VerifiedRetentionClosure, + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, RetentionTransitionDisposition, VerifiedRetentionClosure, }; +use crate::RetentionManifestDigest; use crate::{LivenessGeneration, RetentionGenerationExpectation, RootGeneration}; /// Canonical global artifacts ready for ordered storage execution. @@ -53,6 +54,8 @@ pub struct RetentionPublicationPreparation<'encoded> { observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, + liveness_generation: LivenessGeneration, + manifest_digest: RetentionManifestDigest, publication: Option, } @@ -82,6 +85,16 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { self.closure } + /// Returns the exact selected global liveness generation. + pub const fn liveness_generation(&self) -> LivenessGeneration { + self.liveness_generation + } + + /// Returns the exact selected global manifest digest. + pub const fn manifest_digest(&self) -> RetentionManifestDigest { + self.manifest_digest + } + /// Returns new global artifacts, or normal absence for an exact retry. pub const fn publication(&self) -> Option<&PreparedRetentionPublication> { self.publication.as_ref() @@ -94,12 +107,16 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { closure: VerifiedRetentionClosure, publication: PreparedRetentionPublication, ) -> Self { + let liveness_generation = publication.liveness_generation(); + let manifest_digest = publication.manifest().digest(); Self { disposition: RetentionTransitionDisposition::Publish, expected, observed, candidate, closure, + liveness_generation, + manifest_digest, publication: Some(publication), } } @@ -109,6 +126,7 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, + current_manifest: &AdmittedRetentionManifest<'_>, ) -> Self { Self { disposition: RetentionTransitionDisposition::AlreadyCommitted, @@ -116,6 +134,8 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { observed, candidate, closure, + liveness_generation: current_manifest.manifest().generation(), + manifest_digest: current_manifest.digest(), publication: None, } } diff --git a/src/adapters/retention/publication_error.rs b/src/adapters/retention/publication_error.rs new file mode 100644 index 0000000..f002c72 --- /dev/null +++ b/src/adapters/retention/publication_error.rs @@ -0,0 +1,61 @@ +//! This boundary module owns retention publication execution failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RetentionPublicationPhase, RetentionTransitionDisposition}; + +/// Failure before or during ordered retention publication. +#[derive(Debug)] +pub enum RetentionPublicationError { + /// Current authority could not be revalidated before mutation. + CurrentVerification { + /// Preserved storage refusal. + source: io::Error, + }, + /// Storage requested publication from an already-committed preparation. + DispositionMismatch { + /// Disposition proven during storage-independent preparation. + prepared: RetentionTransitionDisposition, + /// Disposition observed under current writer authority. + observed: RetentionTransitionDisposition, + }, + /// A publish disposition lacked its private canonical artifacts. + MissingPublicationArtifacts, + /// One exact durability phase failed. + Storage { + /// Phase attempted when storage refused. + phase: RetentionPublicationPhase, + /// Preserved storage refusal. + source: io::Error, + }, +} + +impl fmt::Display for RetentionPublicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentVerification { .. } => { + formatter.write_str("retention publication authority verification failed") + } + Self::DispositionMismatch { .. } => { + formatter.write_str("retention publication disposition changed inconsistently") + } + Self::MissingPublicationArtifacts => { + formatter.write_str("retention publication artifacts are missing") + } + Self::Storage { phase, .. } => { + write!(formatter, "retention publication phase {phase} failed") + } + } + } +} + +impl Error for RetentionPublicationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::CurrentVerification { source } | Self::Storage { source, .. } => Some(source), + Self::DispositionMismatch { .. } | Self::MissingPublicationArtifacts => None, + } + } +} diff --git a/src/adapters/retention/publication_execution.rs b/src/adapters/retention/publication_execution.rs new file mode 100644 index 0000000..e7a7d30 --- /dev/null +++ b/src/adapters/retention/publication_execution.rs @@ -0,0 +1,153 @@ +//! This boundary module owns ordered retention publication execution. + +use std::io; + +use super::{ + PreparedRetentionPublication, RetentionNamespaceAdmission, RetentionPublicationError, + RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionTransitionDisposition, +}; + +/// Executes one prepared retention transition under revalidated authority. +/// +/// Exact already-committed state performs no publication mutation. A new +/// publication returns only after head visibility and cleanup are synchronized. +/// +/// # Errors +/// +/// Returns [`RetentionPublicationError`] for current-state revalidation, +/// disposition disagreement, missing private artifacts, or the exact failed +/// durability phase. Failure returns no receipt. +pub fn execute_retention_publication( + storage: &mut impl RetentionPublicationStorage, + preparation: &RetentionPublicationPreparation<'_>, +) -> Result { + let observed = storage + .verify_current(preparation) + .map_err(|source| RetentionPublicationError::CurrentVerification { source })?; + if observed == RetentionTransitionDisposition::AlreadyCommitted { + return Ok(RetentionPublicationReceipt::new( + RetentionPublicationOutcome::AlreadyCommitted, + None, + preparation, + )); + } + if preparation.disposition() != RetentionTransitionDisposition::Publish { + return Err(RetentionPublicationError::DispositionMismatch { + prepared: preparation.disposition(), + observed, + }); + } + let publication = preparation + .publication() + .ok_or(RetentionPublicationError::MissingPublicationArtifacts)?; + let namespace_admission = execute_root(storage, preparation)?; + execute_manifest(storage, publication)?; + execute_head(storage, publication)?; + execute_cleanup(storage)?; + Ok(RetentionPublicationReceipt::new( + RetentionPublicationOutcome::Published, + Some(namespace_admission), + preparation, + )) +} + +fn execute_root( + storage: &mut impl RetentionPublicationStorage, + preparation: &RetentionPublicationPreparation<'_>, +) -> Result { + let root = preparation.candidate(); + require( + storage.write_root_stage(root), + RetentionPublicationPhase::WriteRootStage, + )?; + require( + storage.synchronize_root_stage(), + RetentionPublicationPhase::SynchronizeRootStage, + )?; + let admission = require( + storage.admit_root_namespace(root), + RetentionPublicationPhase::AdmitRootNamespace, + )?; + if admission == RetentionNamespaceAdmission::Created { + require( + storage.synchronize_roots_after_namespace(), + RetentionPublicationPhase::SynchronizeRootsAfterNamespace, + )?; + } + require(storage.link_root(root), RetentionPublicationPhase::LinkRoot)?; + require( + storage.synchronize_root_namespace(root), + RetentionPublicationPhase::SynchronizeRootNamespace, + )?; + Ok(admission) +} + +fn execute_manifest( + storage: &mut impl RetentionPublicationStorage, + publication: &PreparedRetentionPublication, +) -> Result<(), RetentionPublicationError> { + let manifest = publication.manifest(); + require( + storage.write_manifest_stage(manifest), + RetentionPublicationPhase::WriteManifestStage, + )?; + require( + storage.synchronize_manifest_stage(), + RetentionPublicationPhase::SynchronizeManifestStage, + )?; + require( + storage.link_manifest(manifest), + RetentionPublicationPhase::LinkManifest, + )?; + require( + storage.synchronize_manifest_pool(), + RetentionPublicationPhase::SynchronizeManifestPool, + ) +} + +fn execute_head( + storage: &mut impl RetentionPublicationStorage, + publication: &PreparedRetentionPublication, +) -> Result<(), RetentionPublicationError> { + require( + storage.write_head_stage(publication.head()), + RetentionPublicationPhase::WriteHeadStage, + )?; + require( + storage.synchronize_head_stage(), + RetentionPublicationPhase::SynchronizeHeadStage, + )?; + require( + storage.replace_head(), + RetentionPublicationPhase::ReplaceHead, + )?; + require( + storage.synchronize_retention_namespace(), + RetentionPublicationPhase::SynchronizeRetentionNamespace, + ) +} + +fn execute_cleanup( + storage: &mut impl RetentionPublicationStorage, +) -> Result<(), RetentionPublicationError> { + require( + storage.remove_root_stage(), + RetentionPublicationPhase::RemoveRootStage, + )?; + require( + storage.remove_manifest_stage(), + RetentionPublicationPhase::RemoveManifestStage, + )?; + require( + storage.synchronize_cleanup(), + RetentionPublicationPhase::SynchronizeCleanup, + ) +} + +fn require( + result: io::Result, + phase: RetentionPublicationPhase, +) -> Result { + result.map_err(|source| RetentionPublicationError::Storage { phase, source }) +} diff --git a/src/adapters/retention/publication_outcome.rs b/src/adapters/retention/publication_outcome.rs new file mode 100644 index 0000000..6f88a3f --- /dev/null +++ b/src/adapters/retention/publication_outcome.rs @@ -0,0 +1,11 @@ +//! This boundary module owns retention publication outcome vocabulary. + +/// Durable outcome of one authority-revalidated retention publication. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionPublicationOutcome { + /// The complete successor became durable and visible. + Published, + /// The exact candidate and global manifest were already current. + AlreadyCommitted, +} diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs index 01680f1..600306f 100644 --- a/src/adapters/retention/publication_preparation.rs +++ b/src/adapters/retention/publication_preparation.rs @@ -26,9 +26,10 @@ pub fn prepare_retention_publication<'encoded>( let (disposition, expected, observed, candidate, closure) = preflight.into_parts(); match disposition { RetentionTransitionDisposition::AlreadyCommitted => { - successor_manifest::require_current_selection(&candidate, current_manifest)?; + let current = + successor_manifest::require_current_selection(&candidate, current_manifest)?; Ok(RetentionPublicationPreparation::already_committed( - expected, observed, candidate, closure, + expected, observed, candidate, closure, current, )) } RetentionTransitionDisposition::Publish => { diff --git a/src/adapters/retention/publication_receipt.rs b/src/adapters/retention/publication_receipt.rs new file mode 100644 index 0000000..cd8238e --- /dev/null +++ b/src/adapters/retention/publication_receipt.rs @@ -0,0 +1,127 @@ +//! This boundary module owns consequential retention publication receipts. + +use super::{ + RetentionNamespaceAdmission, RetentionPublicationOutcome, RetentionPublicationPreparation, +}; +use crate::{ + CatalogDigest, CatalogGeneration, LivenessGeneration, RegisteredRetentionProfile, + RetentionAnchorSetDigest, RetentionClosureDigest, RetentionGenerationExpectation, + RetentionManifestDigest, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, +}; + +/// Complete durable coordinates returned after retention publication. +#[must_use = "retention publication receipts bind the durable outcome"] +#[derive(Debug, Eq, PartialEq)] +pub struct RetentionPublicationReceipt { + outcome: RetentionPublicationOutcome, + namespace_admission: Option, + namespace: RetentionNamespaceDigest, + expected: RetentionGenerationExpectation, + observed: Option, + root_generation: RootGeneration, + root_digest: RetentionRootDigest, + liveness_generation: LivenessGeneration, + manifest_digest: RetentionManifestDigest, + profile: RegisteredRetentionProfile, + anchor_set_digest: RetentionAnchorSetDigest, + closure_digest: RetentionClosureDigest, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, +} + +impl RetentionPublicationReceipt { + /// Returns the durable publication outcome. + pub const fn outcome(&self) -> RetentionPublicationOutcome { + self.outcome + } + + /// Returns namespace creation or admission for a new publication. + pub const fn namespace_admission(&self) -> Option { + self.namespace_admission + } + + /// Returns the selected retention namespace digest. + pub const fn namespace(&self) -> RetentionNamespaceDigest { + self.namespace + } + + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Returns the namespace generation observed before publication. + pub const fn observed(&self) -> Option { + self.observed + } + + /// Returns the committed namespace root generation. + pub const fn root_generation(&self) -> RootGeneration { + self.root_generation + } + + /// Returns the committed canonical root digest. + pub const fn root_digest(&self) -> RetentionRootDigest { + self.root_digest + } + + /// Returns the selected global liveness generation. + pub const fn liveness_generation(&self) -> LivenessGeneration { + self.liveness_generation + } + + /// Returns the selected global manifest digest. + pub const fn manifest_digest(&self) -> RetentionManifestDigest { + self.manifest_digest + } + + /// Returns the registered realization profile. + pub const fn profile(&self) -> RegisteredRetentionProfile { + self.profile + } + + /// Returns the verified anchor-set digest. + pub const fn anchor_set_digest(&self) -> RetentionAnchorSetDigest { + self.anchor_set_digest + } + + /// Returns the verified closure transcript digest. + pub const fn closure_digest(&self) -> RetentionClosureDigest { + self.closure_digest + } + + /// Returns the pinned catalog generation. + pub const fn catalog_generation(&self) -> CatalogGeneration { + self.catalog_generation + } + + /// Returns the pinned catalog digest. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.catalog_digest + } + + pub(super) fn new( + outcome: RetentionPublicationOutcome, + namespace_admission: Option, + preparation: &RetentionPublicationPreparation<'_>, + ) -> Self { + let candidate = preparation.candidate(); + let closure = preparation.closure(); + Self { + outcome, + namespace_admission, + namespace: candidate.root().namespace().digest(), + expected: preparation.expected(), + observed: preparation.observed(), + root_generation: candidate.root().generation(), + root_digest: candidate.digest(), + liveness_generation: preparation.liveness_generation(), + manifest_digest: preparation.manifest_digest(), + profile: candidate.root().profile(), + anchor_set_digest: candidate.anchor_set_digest(), + closure_digest: closure.digest(), + catalog_generation: closure.catalog_generation(), + catalog_digest: closure.catalog_digest(), + } + } +} diff --git a/src/adapters/retention/publication_storage.rs b/src/adapters/retention/publication_storage.rs index cbef5d8..7a28232 100644 --- a/src/adapters/retention/publication_storage.rs +++ b/src/adapters/retention/publication_storage.rs @@ -4,17 +4,32 @@ use std::io; use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionNamespaceAdmission, + RetentionNamespaceAdmission, RetentionPublicationPreparation, RetentionTransitionDisposition, }; /// Blocking storage capabilities for one writer-locked retention publication. /// /// An implementation must retain exclusive writer authority and one pinned -/// store root for the complete operation. Each method corresponds to one -/// [`RetentionPublicationPhase`](super::RetentionPublicationPhase) and must not -/// report success before the named durability and verification obligations are -/// complete. +/// store root for the complete operation. After `verify_current`, each method +/// corresponds to one [`RetentionPublicationPhase`](super::RetentionPublicationPhase) +/// and must not report success before the named durability and verification +/// obligations are complete. pub trait RetentionPublicationStorage { + /// Reopens and verifies current authority against the complete preparation. + /// + /// `Publish` requires the expected predecessor and global manifest + /// coordinates to remain current. `AlreadyCommitted` requires the exact + /// candidate root and selected global manifest coordinates to be current. + /// Fixed-stage recovery state must refuse before either disposition. + /// + /// # Errors + /// + /// Returns the exact current-state or recovery-required refusal. + fn verify_current( + &mut self, + preparation: &RetentionPublicationPreparation<'_>, + ) -> io::Result; + /// Exclusively creates and completely writes the canonical root stage. /// /// # Errors diff --git a/src/adapters/retention/successor_manifest.rs b/src/adapters/retention/successor_manifest.rs index d87b0d6..3859c16 100644 --- a/src/adapters/retention/successor_manifest.rs +++ b/src/adapters/retention/successor_manifest.rs @@ -46,10 +46,10 @@ pub(super) fn build( .map_err(|source| RetentionPublicationPreparationError::Manifest { source }) } -pub(super) fn require_current_selection( +pub(super) fn require_current_selection<'borrow, 'encoded>( candidate: &AdmittedRetentionRoot<'_>, - current: Option<&AdmittedRetentionManifest<'_>>, -) -> Result<(), RetentionPublicationPreparationError> { + current: Option<&'borrow AdmittedRetentionManifest<'encoded>>, +) -> Result<&'borrow AdmittedRetentionManifest<'encoded>, RetentionPublicationPreparationError> { let namespace = candidate.root().namespace().digest(); let current = current .ok_or(RetentionPublicationPreparationError::CurrentManifestRequired { namespace })?; @@ -70,7 +70,7 @@ pub(super) fn require_current_selection( if entry.root_generation() == candidate.root().generation() && entry.root_digest() == candidate.digest() { - Ok(()) + Ok(current) } else { Err(current_mismatch(entry, candidate)) } diff --git a/src/lib.rs b/src/lib.rs index 13ad241..91cd10a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,8 +28,9 @@ //! combined transition preflight proof and exact publication phase vocabulary //! with a blocking storage capability port are available. Storage-independent //! preparation binds preflight to exact canonical manifest and head successors. -//! Retention publication orchestration, filesystem execution, recovery, and -//! garbage collection remain intentionally absent. +//! Ordered publication revalidates authority, executes all durability phases, +//! and returns a complete receipt. Filesystem execution, recovery, and garbage +//! collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -112,12 +113,13 @@ pub use adapters::{ CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, PreparedRetentionPublication, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, - RetentionPublicationPhase, RetentionPublicationPreparation, - RetentionPublicationPreparationError, RetentionPublicationStorage, RetentionRootDecodeError, + RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, + RetentionPublicationPreparation, RetentionPublicationPreparationError, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, - VerifiedRetentionClosure, plan_retention_transition, preflight_retention_transition, - prepare_retention_publication, verify_retention_closure, + VerifiedRetentionClosure, execute_retention_publication, plan_retention_transition, + preflight_retention_transition, prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_publication_execution.rs b/tests/retention_publication_execution.rs new file mode 100644 index 0000000..59813ef --- /dev/null +++ b/tests/retention_publication_execution.rs @@ -0,0 +1,144 @@ +//! Ordered retention publication and consequential receipt laws. + +#[path = "retention_publication_preparation/fixture.rs"] +pub mod fixture; +#[path = "retention_publication_storage/recording_storage.rs"] +pub mod recording_storage; +#[path = "retention_publication_execution/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionGenerationExpectation, + RetentionNamespaceAdmission, RetentionPublicationOutcome, RetentionPublicationPhase, + RetentionPublicationPreparation, RootGeneration, execute_retention_publication, + preflight_retention_transition, prepare_retention_publication, +}; + +use fixture::{manifest_bytes, root_bytes, with_snapshot}; +use recording_storage::RecordingStorage; + +#[test] +fn publication_executes_every_phase_and_returns_complete_coordinates() -> Result<(), Box> +{ + let root_bytes = root_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let namespace = candidate.root().namespace().digest(); + let root_generation = candidate.root().generation(); + let root_digest = candidate.digest(); + let profile = candidate.root().profile(); + let anchor_set_digest = candidate.anchor_set_digest(); + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + let closure = preflight.closure(); + let preparation = prepare_retention_publication(preflight, None)?; + let publication = preparation + .publication() + .ok_or("initial transition omitted publication artifacts")?; + let liveness_generation = publication.liveness_generation(); + let manifest_digest = publication.manifest().digest(); + let mut storage = RecordingStorage::new(); + + let receipt = execute_retention_publication(&mut storage, &preparation)?; + + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), RetentionPublicationPhase::ALL); + assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); + assert_eq!( + receipt.namespace_admission(), + Some(RetentionNamespaceAdmission::Created) + ); + assert_eq!(receipt.namespace(), namespace); + assert_eq!(receipt.expected(), RetentionGenerationExpectation::Absent); + assert_eq!(receipt.observed(), None); + assert_eq!(receipt.root_generation(), root_generation); + assert_eq!(receipt.root_digest(), root_digest); + assert_eq!(receipt.liveness_generation(), liveness_generation); + assert_eq!(receipt.manifest_digest(), manifest_digest); + assert_eq!(receipt.profile(), profile); + assert_eq!(receipt.anchor_set_digest(), anchor_set_digest); + assert_eq!(receipt.closure_digest(), closure.digest()); + assert_eq!(receipt.catalog_generation(), closure.catalog_generation()); + assert_eq!(receipt.catalog_digest(), closure.catalog_digest()); + Ok(()) +} + +#[test] +fn exact_retry_revalidates_authority_without_publication_mutation() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + })??; + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let mut storage = RecordingStorage::already_committed(); + + let receipt = execute_retention_publication(&mut storage, &preparation)?; + + assert_eq!(storage.verification_count(), 1); + assert!(storage.observed().is_empty()); + assert_eq!( + receipt.outcome(), + RetentionPublicationOutcome::AlreadyCommitted + ); + assert_eq!(receipt.namespace_admission(), None); + assert_eq!(receipt.observed(), Some(RootGeneration::INITIAL)); + assert_eq!( + receipt.liveness_generation(), + current_manifest.manifest().generation() + ); + assert_eq!(receipt.manifest_digest(), current_manifest.digest()); + Ok(()) +} + +#[test] +fn existing_namespace_skips_only_the_parent_directory_synchronization() -> Result<(), Box> +{ + let root_bytes = root_bytes()?; + let preparation = initial_preparation(&root_bytes)?; + let mut storage = RecordingStorage::existing_namespace(); + let expected = RetentionPublicationPhase::ALL + .into_iter() + .filter(|phase| *phase != RetentionPublicationPhase::SynchronizeRootsAfterNamespace) + .collect::>(); + + let receipt = execute_retention_publication(&mut storage, &preparation)?; + + assert_eq!(storage.observed(), expected); + assert_eq!( + receipt.namespace_admission(), + Some(RetentionNamespaceAdmission::Existing) + ); + Ok(()) +} + +pub(crate) fn initial_preparation( + root_bytes: &[u8], +) -> Result, Box> { + let candidate = AdmittedRetentionRoot::decode(root_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + prepare_retention_publication(preflight, None).map_err(Into::into) +} diff --git a/tests/retention_publication_execution/refusal_laws.rs b/tests/retention_publication_execution/refusal_laws.rs new file mode 100644 index 0000000..f76a081 --- /dev/null +++ b/tests/retention_publication_execution/refusal_laws.rs @@ -0,0 +1,94 @@ +//! Retention publication execution refusal laws. + +use std::error::Error; +use std::io; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionGenerationExpectation, + RetentionPublicationError, RetentionPublicationPhase, RetentionTransitionDisposition, + execute_retention_publication, preflight_retention_transition, prepare_retention_publication, +}; + +use crate::fixture::{manifest_bytes, root_bytes, with_snapshot}; +use crate::recording_storage::RecordingStorage; +use crate::support::require_error; + +#[test] +fn current_authority_refusal_precedes_every_publication_phase() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let preparation = crate::initial_preparation(&root_bytes)?; + let mut storage = RecordingStorage::verification_failure(); + + let error = require_error( + execute_retention_publication(&mut storage, &preparation), + "authority refusal returned a receipt", + )?; + + assert!(matches!( + error, + RetentionPublicationError::CurrentVerification { source } + if source.kind() == io::ErrorKind::PermissionDenied + )); + assert_eq!(storage.verification_count(), 1); + assert!(storage.observed().is_empty()); + Ok(()) +} + +#[test] +fn every_phase_failure_stops_before_all_later_mutation() -> Result<(), Box> { + let mut expected = Vec::new(); + for failing_phase in RetentionPublicationPhase::ALL { + expected.push(failing_phase); + let root_bytes = root_bytes()?; + let preparation = crate::initial_preparation(&root_bytes)?; + let mut storage = RecordingStorage::failing_at(failing_phase); + + let error = require_error( + execute_retention_publication(&mut storage, &preparation), + "phase failure returned a receipt", + )?; + + assert!(matches!( + error, + RetentionPublicationError::Storage { phase, source } + if phase == failing_phase && source.kind() == io::ErrorKind::Other + )); + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), expected); + } + Ok(()) +} + +#[test] +fn changed_disposition_refuses_before_publication_mutation() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + })??; + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let mut storage = RecordingStorage::new(); + + let error = require_error( + execute_retention_publication(&mut storage, &preparation), + "changed disposition returned a receipt", + )?; + + assert!(matches!( + error, + RetentionPublicationError::DispositionMismatch { + prepared: RetentionTransitionDisposition::AlreadyCommitted, + observed: RetentionTransitionDisposition::Publish, + } + )); + assert!(storage.observed().is_empty()); + Ok(()) +} diff --git a/tests/retention_publication_storage/recording_storage.rs b/tests/retention_publication_storage/recording_storage.rs index 51047dc..fb7a07d 100644 --- a/tests/retention_publication_storage/recording_storage.rs +++ b/tests/retention_publication_storage/recording_storage.rs @@ -4,13 +4,24 @@ use std::io; use keep::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationStorage, + RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationStorage, RetentionTransitionDisposition, }; /// Storage port that records every attempted publication phase. -#[derive(Default)] pub struct RecordingStorage { observed: Vec, + verification_count: usize, + disposition: RetentionTransitionDisposition, + namespace_admission: RetentionNamespaceAdmission, + fail_at: Option, + verification_failure: Option, +} + +impl Default for RecordingStorage { + fn default() -> Self { + Self::new() + } } impl RecordingStorage { @@ -18,6 +29,59 @@ impl RecordingStorage { pub const fn new() -> Self { Self { observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: None, + verification_failure: None, + } + } + + /// Creates a recorder that observes the candidate as already committed. + pub const fn already_committed() -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::AlreadyCommitted, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: None, + verification_failure: None, + } + } + + /// Creates a publisher that admits an existing root namespace. + pub const fn existing_namespace() -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Existing, + fail_at: None, + verification_failure: None, + } + } + + /// Creates a publisher that fails at one exact durability phase. + pub const fn failing_at(phase: RetentionPublicationPhase) -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: Some(phase), + verification_failure: None, + } + } + + /// Creates a publisher that refuses current-authority verification. + pub const fn verification_failure() -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: None, + verification_failure: Some(io::ErrorKind::PermissionDenied), } } @@ -26,97 +90,105 @@ impl RecordingStorage { &self.observed } - fn record(&mut self, phase: RetentionPublicationPhase) { + /// Returns the number of authority-verification calls. + pub const fn verification_count(&self) -> usize { + self.verification_count + } + + fn record(&mut self, phase: RetentionPublicationPhase) -> io::Result<()> { self.observed.push(phase); + if self.fail_at == Some(phase) { + Err(io::Error::other("injected retention publication failure")) + } else { + Ok(()) + } } } impl RetentionPublicationStorage for RecordingStorage { + fn verify_current( + &mut self, + _preparation: &RetentionPublicationPreparation<'_>, + ) -> io::Result { + self.verification_count = self + .verification_count + .checked_add(1) + .ok_or_else(|| io::Error::other("verification count overflow"))?; + match self.verification_failure { + Some(kind) => Err(io::Error::new(kind, "injected authority failure")), + None => Ok(self.disposition), + } + } + fn write_root_stage(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.record(RetentionPublicationPhase::WriteRootStage); - Ok(()) + self.record(RetentionPublicationPhase::WriteRootStage) } fn synchronize_root_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRootStage); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRootStage) } fn admit_root_namespace( &mut self, _root: &AdmittedRetentionRoot<'_>, ) -> io::Result { - self.record(RetentionPublicationPhase::AdmitRootNamespace); - Ok(RetentionNamespaceAdmission::Created) + self.record(RetentionPublicationPhase::AdmitRootNamespace)?; + Ok(self.namespace_admission) } fn synchronize_roots_after_namespace(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRootsAfterNamespace); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRootsAfterNamespace) } fn link_root(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.record(RetentionPublicationPhase::LinkRoot); - Ok(()) + self.record(RetentionPublicationPhase::LinkRoot) } fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRootNamespace); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRootNamespace) } fn write_manifest_stage(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { - self.record(RetentionPublicationPhase::WriteManifestStage); - Ok(()) + self.record(RetentionPublicationPhase::WriteManifestStage) } fn synchronize_manifest_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeManifestStage); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeManifestStage) } fn link_manifest(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { - self.record(RetentionPublicationPhase::LinkManifest); - Ok(()) + self.record(RetentionPublicationPhase::LinkManifest) } fn synchronize_manifest_pool(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeManifestPool); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeManifestPool) } fn write_head_stage(&mut self, _head: &CanonicalRetentionHead) -> io::Result<()> { - self.record(RetentionPublicationPhase::WriteHeadStage); - Ok(()) + self.record(RetentionPublicationPhase::WriteHeadStage) } fn synchronize_head_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeHeadStage); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeHeadStage) } fn replace_head(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::ReplaceHead); - Ok(()) + self.record(RetentionPublicationPhase::ReplaceHead) } fn synchronize_retention_namespace(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRetentionNamespace); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRetentionNamespace) } fn remove_root_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::RemoveRootStage); - Ok(()) + self.record(RetentionPublicationPhase::RemoveRootStage) } fn remove_manifest_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::RemoveManifestStage); - Ok(()) + self.record(RetentionPublicationPhase::RemoveManifestStage) } fn synchronize_cleanup(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeCleanup); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeCleanup) } } From c0fb6496698fe83dc2f29649bf5c221e6c65ea13 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:03:54 -0700 Subject: [PATCH 27/50] Fix: Seal retention receipt outcomes --- .../retention/publication_execution.rs | 15 +++---- src/adapters/retention/publication_receipt.rs | 43 +++++++++++++++---- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/adapters/retention/publication_execution.rs b/src/adapters/retention/publication_execution.rs index e7a7d30..cb4aa09 100644 --- a/src/adapters/retention/publication_execution.rs +++ b/src/adapters/retention/publication_execution.rs @@ -4,8 +4,8 @@ use std::io; use super::{ PreparedRetentionPublication, RetentionNamespaceAdmission, RetentionPublicationError, - RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, - RetentionPublicationReceipt, RetentionPublicationStorage, RetentionTransitionDisposition, + RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationReceipt, + RetentionPublicationStorage, RetentionTransitionDisposition, }; /// Executes one prepared retention transition under revalidated authority. @@ -26,11 +26,7 @@ pub fn execute_retention_publication( .verify_current(preparation) .map_err(|source| RetentionPublicationError::CurrentVerification { source })?; if observed == RetentionTransitionDisposition::AlreadyCommitted { - return Ok(RetentionPublicationReceipt::new( - RetentionPublicationOutcome::AlreadyCommitted, - None, - preparation, - )); + return Ok(RetentionPublicationReceipt::already_committed(preparation)); } if preparation.disposition() != RetentionTransitionDisposition::Publish { return Err(RetentionPublicationError::DispositionMismatch { @@ -45,9 +41,8 @@ pub fn execute_retention_publication( execute_manifest(storage, publication)?; execute_head(storage, publication)?; execute_cleanup(storage)?; - Ok(RetentionPublicationReceipt::new( - RetentionPublicationOutcome::Published, - Some(namespace_admission), + Ok(RetentionPublicationReceipt::published( + namespace_admission, preparation, )) } diff --git a/src/adapters/retention/publication_receipt.rs b/src/adapters/retention/publication_receipt.rs index cd8238e..f3405af 100644 --- a/src/adapters/retention/publication_receipt.rs +++ b/src/adapters/retention/publication_receipt.rs @@ -13,8 +13,7 @@ use crate::{ #[must_use = "retention publication receipts bind the durable outcome"] #[derive(Debug, Eq, PartialEq)] pub struct RetentionPublicationReceipt { - outcome: RetentionPublicationOutcome, - namespace_admission: Option, + effect: RetentionPublicationEffect, namespace: RetentionNamespaceDigest, expected: RetentionGenerationExpectation, observed: Option, @@ -32,12 +31,20 @@ pub struct RetentionPublicationReceipt { impl RetentionPublicationReceipt { /// Returns the durable publication outcome. pub const fn outcome(&self) -> RetentionPublicationOutcome { - self.outcome + match self.effect { + RetentionPublicationEffect::Published(_) => RetentionPublicationOutcome::Published, + RetentionPublicationEffect::AlreadyCommitted => { + RetentionPublicationOutcome::AlreadyCommitted + } + } } /// Returns namespace creation or admission for a new publication. pub const fn namespace_admission(&self) -> Option { - self.namespace_admission + match self.effect { + RetentionPublicationEffect::Published(admission) => Some(admission), + RetentionPublicationEffect::AlreadyCommitted => None, + } } /// Returns the selected retention namespace digest. @@ -100,16 +107,28 @@ impl RetentionPublicationReceipt { self.catalog_digest } - pub(super) fn new( - outcome: RetentionPublicationOutcome, - namespace_admission: Option, + pub(super) fn published( + namespace_admission: RetentionNamespaceAdmission, + preparation: &RetentionPublicationPreparation<'_>, + ) -> Self { + Self::new( + RetentionPublicationEffect::Published(namespace_admission), + preparation, + ) + } + + pub(super) fn already_committed(preparation: &RetentionPublicationPreparation<'_>) -> Self { + Self::new(RetentionPublicationEffect::AlreadyCommitted, preparation) + } + + fn new( + effect: RetentionPublicationEffect, preparation: &RetentionPublicationPreparation<'_>, ) -> Self { let candidate = preparation.candidate(); let closure = preparation.closure(); Self { - outcome, - namespace_admission, + effect, namespace: candidate.root().namespace().digest(), expected: preparation.expected(), observed: preparation.observed(), @@ -125,3 +144,9 @@ impl RetentionPublicationReceipt { } } } + +#[derive(Debug, Eq, PartialEq)] +enum RetentionPublicationEffect { + Published(RetentionNamespaceAdmission), + AlreadyCommitted, +} From 13988860f50f6de70e3795fb66f71ef9bf7a0728 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:05:58 -0700 Subject: [PATCH 28/50] Docs: Correct retention execution boundary --- docs/formats/segment-store-v2/retention.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index c82960d..e1d192c 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -162,9 +162,9 @@ retention/roots// Names with alternate width, case, suffix, generation, or digest refuse. Keep implements root, manifest, and head codecs with a typed verified anchor-set digest, expected-state transition planning, deterministic closure verification, -and a blocking publication storage capability port. Publication -orchestration, filesystem execution, recovery, and garbage collection remain -absent. +a blocking publication storage capability port, and ordered storage-port +orchestration. Production filesystem execution, recovery, and garbage +collection remain absent. ## Global retention manifest From eee636b7b90a127fba8364e5a283251b696f221a Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:18:48 -0700 Subject: [PATCH 29/50] Test: Fuzz retention record parsers --- docs/formats/segment-store-v2/requirements.md | 2 +- fuzz/Cargo.toml | 7 +++ fuzz/README.md | 5 ++ fuzz/fuzz_targets/retention_format.rs | 35 ++++++++++++++ xtask/src/fuzz_campaign/target/tests.rs | 1 + xtask/src/fuzz_seed_corpus.rs | 2 + xtask/src/fuzz_seed_corpus/retention_seeds.rs | 46 +++++++++++++++++++ .../fuzz_seed_corpus/tests/materialization.rs | 30 +++++++++++- .../retention_store_v2_protocol_contract.rs | 2 + .../parser_fuzz_laws.rs | 31 +++++++++++++ 10 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 fuzz/fuzz_targets/retention_format.rs create mode 100644 xtask/src/fuzz_seed_corpus/retention_seeds.rs create mode 100644 xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 53b12d3..456f139 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -11,7 +11,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | -| `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | +| `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; filesystem evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; crash injection remains | In progress in #19 | diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 5ab41b7..b041049 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -124,6 +124,13 @@ test = false doc = false bench = false +[[bin]] +name = "retention_format" +path = "fuzz_targets/retention_format.rs" +test = false +doc = false +bench = false + [[bin]] name = "segment_format" path = "fuzz_targets/segment_format.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 0bcff69..8b6af19 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -65,6 +65,11 @@ decoders. Canonical generation-1, generation-2, and two-record bundle artifacts keep mutations inside framing, ordering, coordinate, checksum, and digest validation; every admitted value must retain its exact input bytes. +The `retention_format` seeds select the public retention-root, +retention-manifest, and retention-head decoders. The canonical one-root +generation keeps mutations inside framing, semantic, ordering, checksum, and +digest validation; every admitted value must retain its exact input bytes. + The `segment_format` seeds select the public segment-header, record-header, complete-record, seal, and complete-segment boundaries. Canonical empty, one-record, and bundled segments keep mutations inside the nested parsers; diff --git a/fuzz/fuzz_targets/retention_format.rs b/fuzz/fuzz_targets/retention_format.rs new file mode 100644 index 0000000..b35808c --- /dev/null +++ b/fuzz/fuzz_targets/retention_format.rs @@ -0,0 +1,35 @@ +#![no_main] + +//! This target owns canonical retention-record parser fuzzing. + +use keep::{AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|bytes: &[u8]| { + let Some((&selector, input)) = bytes.split_first() else { + return; + }; + match selector { + 0 => root(input), + 1 => manifest(input), + _ => head(input), + } +}); + +fn root(input: &[u8]) { + if let Ok(root) = AdmittedRetentionRoot::decode(input) { + assert_eq!(root.encoded(), input); + } +} + +fn manifest(input: &[u8]) { + if let Ok(manifest) = AdmittedRetentionManifest::decode(input) { + assert_eq!(manifest.encoded(), input); + } +} + +fn head(input: &[u8]) { + if let Ok(head) = ChecksummedRetentionHead::decode(input) { + assert_eq!(head.encoded(), input); + } +} diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index 82e88b3..7b72f99 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -31,6 +31,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "golden_protocol", "layout_record", "repository_json", + "retention_format", "segment_format", ] ); diff --git a/xtask/src/fuzz_seed_corpus.rs b/xtask/src/fuzz_seed_corpus.rs index 61c76c0..1455e04 100644 --- a/xtask/src/fuzz_seed_corpus.rs +++ b/xtask/src/fuzz_seed_corpus.rs @@ -5,6 +5,7 @@ mod cdc_seeds; mod filesystem; mod identity_seeds; mod layout_seeds; +mod retention_seeds; mod segment_seeds; use std::error::Error; @@ -69,6 +70,7 @@ pub(super) fn prepare(repository_root: &Path) -> Result<(), FuzzSeedError> { seeds.extend(cdc_seeds::seeds()?); seeds.extend(golden_protocol_seeds_from(&files)?); seeds.extend(layout_seeds::seeds(&files)?); + seeds.extend(retention_seeds::seeds(&files)?); seeds.extend(segment_seeds::seeds(&files)?); files.write_seeds(&seeds) } diff --git a/xtask/src/fuzz_seed_corpus/retention_seeds.rs b/xtask/src/fuzz_seed_corpus/retention_seeds.rs new file mode 100644 index 0000000..152dc16 --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/retention_seeds.rs @@ -0,0 +1,46 @@ +//! This module owns canonical retention-record fuzz seeds. + +use std::path::Path; + +use super::filesystem::RepositoryFiles; +use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; +use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; + +const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v2"; + +pub(super) const FIXTURES: [(u8, &str); 3] = [ + (0, "one-anchor-root.hex"), + (1, "one-root-manifest.hex"), + (2, "one-root-head.hex"), +]; + +pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { + let mut seeds = Vec::new(); + for (selector, fixture) in FIXTURES { + let name = fixture + .strip_suffix(".hex") + .ok_or_else(|| FuzzSeedError::violation("retention fixture lacks .hex suffix"))?; + let encoded = fixture_bytes(files, fixture)?; + seeds.push(Seed::new( + "retention_format", + name, + prefixed(selector, &encoded)?, + )?); + } + Ok(seeds) +} + +fn fixture_bytes(files: &RepositoryFiles, fixture: &'static str) -> Result, FuzzSeedError> { + let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); + let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; + let lines = framed_lines(&transport, MAX_SEED_BYTES) + .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; + let [encoded] = lines.as_slice() else { + return Err(FuzzSeedError::violation(format!( + "{fixture} must contain exactly one hexadecimal line" + ))); + }; + decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { + FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) + }) +} diff --git a/xtask/src/fuzz_seed_corpus/tests/materialization.rs b/xtask/src/fuzz_seed_corpus/tests/materialization.rs index 856ccf5..5ed813f 100644 --- a/xtask/src/fuzz_seed_corpus/tests/materialization.rs +++ b/xtask/src/fuzz_seed_corpus/tests/materialization.rs @@ -3,7 +3,9 @@ use std::collections::BTreeMap; use std::path::Path; -use super::super::{FuzzSeedError, catalog_seeds, layout_seeds, prepare, segment_seeds}; +use super::super::{ + FuzzSeedError, catalog_seeds, layout_seeds, prepare, retention_seeds, segment_seeds, +}; use crate::test_directory::TestDirectory; const TABLES: [&str; 5] = [ @@ -39,14 +41,16 @@ fn seed_preparation_materializes_the_complete_deterministic_set() copy_layout_fixtures(source_root, root)?; copy_segment_fixtures(source_root, root)?; copy_catalog_fixtures(source_root, root)?; + copy_retention_fixtures(source_root, root)?; prepare(root)?; let corpus = root.join("fuzz/corpus"); let first = seed_contents(&corpus)?; - assert_eq!(first.len(), 40); + assert_eq!(first.len(), 43); assert_eq!(target_seed_count(&first, "catalog_format/"), 6); assert_eq!(target_seed_count(&first, "golden_protocol/"), 9); assert_eq!(target_seed_count(&first, "layout_record/"), 4); + assert_eq!(target_seed_count(&first, "retention_format/"), 3); assert_eq!(target_seed_count(&first, "segment_format/"), 8); prepare(root)?; assert_eq!(seed_contents(&corpus)?, first); @@ -112,6 +116,28 @@ fn copy_catalog_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeed Ok(()) } +fn copy_retention_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { + use std::fs; + + let retention_directory = root.join("conformance/segment-store/v2"); + fs::create_dir_all(&retention_directory).map_err(|source| { + FuzzSeedError::io( + "create test retention conformance root", + &retention_directory, + source, + ) + })?; + for (_selector, fixture) in retention_seeds::FIXTURES { + let source_path = source_root + .join("conformance/segment-store/v2") + .join(fixture); + let destination = retention_directory.join(fixture); + fs::copy(&source_path, &destination) + .map_err(|source| FuzzSeedError::io("copy test retention", &destination, source))?; + } + Ok(()) +} + fn target_seed_count(contents: &BTreeMap>, prefix: &str) -> usize { contents .keys() diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index d2dc2ae..09f2316 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -6,6 +6,8 @@ mod closure_contract_laws; #[path = "retention_store_v2_protocol_contract/migration_contract_laws.rs"] mod migration_contract_laws; +#[path = "retention_store_v2_protocol_contract/parser_fuzz_laws.rs"] +mod parser_fuzz_laws; use std::fs; use std::io; diff --git a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs new file mode 100644 index 0000000..bc2919f --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs @@ -0,0 +1,31 @@ +//! Fuzz-evidence laws for durable retention parser boundaries. + +use std::error::Error; +use std::path::Path; + +const FUZZ_MANIFEST: &str = include_str!("../../../fuzz/Cargo.toml"); +const FUZZ_GUIDE: &str = include_str!("../../../fuzz/README.md"); +const REQUIREMENTS: &str = include_str!("../../../docs/formats/segment-store-v2/requirements.md"); + +#[test] +fn retention_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box> { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest must have a repository parent")?; + + assert!( + repository_root + .join("fuzz/fuzz_targets/retention_format.rs") + .is_file() + ); + assert!( + repository_root + .join("xtask/src/fuzz_seed_corpus/retention_seeds.rs") + .is_file() + ); + assert!(FUZZ_MANIFEST.contains("name = \"retention_format\"")); + assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/retention_format.rs\"")); + assert!(FUZZ_GUIDE.contains("The `retention_format` seeds")); + assert!(REQUIREMENTS.contains("`retention_format`")); + Ok(()) +} From aabfb895a24569fc30f236e1f24aa09c99eeb304 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:40:17 -0700 Subject: [PATCH 30/50] Add: Admit version two format markers --- CHANGELOG.md | 12 +- README.md | 6 +- docs/formats/segment-store-v2/recovery.md | 5 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/mod.rs | 2 + src/adapters/store_migration.rs | 16 ++ .../store_migration/admitted_format_marker.rs | 58 ++++++ .../canonical_format_marker.rs | 36 ++++ .../format_definition_digest.rs | 25 +++ .../format_marker_decode_error.rs | 63 +++++++ .../format_marker_decode_error_display.rs | 50 ++++++ .../store_migration/format_marker_decoder.rs | 137 ++++++++++++++ .../store_migration/format_marker_digest.rs | 18 ++ .../store_migration/format_marker_encoder.rs | 27 +++ src/lib.rs | 77 ++++---- tests/store_format_marker.rs | 168 ++++++++++++++++++ 16 files changed, 656 insertions(+), 46 deletions(-) create mode 100644 src/adapters/store_migration.rs create mode 100644 src/adapters/store_migration/admitted_format_marker.rs create mode 100644 src/adapters/store_migration/canonical_format_marker.rs create mode 100644 src/adapters/store_migration/format_definition_digest.rs create mode 100644 src/adapters/store_migration/format_marker_decode_error.rs create mode 100644 src/adapters/store_migration/format_marker_decode_error_display.rs create mode 100644 src/adapters/store_migration/format_marker_decoder.rs create mode 100644 src/adapters/store_migration/format_marker_digest.rs create mode 100644 src/adapters/store_migration/format_marker_encoder.rs create mode 100644 tests/store_format_marker.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7122bc3..2afa12d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- Retention transition preflight now combines exact expected-generation - planning with deterministic closure verification against one pinned catalog - before any future publication storage call. A typed 17-phase vocabulary - and blocking storage port freeze the durability and crash-boundary contract; - authority-revalidated orchestration executes every phase and returns an - unforgeable complete-coordinate receipt after durable cleanup. +- The version-2 store-format marker now has exact canonical encoding, + registered-definition admission, checksum verification, and domain-separated + identity. Retention transition preflight combines expected-generation + planning with deterministic closure verification against one pinned catalog; + authority-revalidated 17-phase orchestration returns an unforgeable + complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 331ab28..00f27e8 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,8 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Version-2 retention values; canonical in-memory root, global +power loss. Canonical version-2 store-format marker encoding and admission are +implemented. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase @@ -123,7 +124,8 @@ publication vocabulary with a blocking storage capability port are implemented. Private-field proofs retain every receipt coordinate. Ordered storage-port orchestration revalidates current authority, executes all 17 durability phases, and returns a consequential complete-coordinate receipt. -Filesystem execution, recovery, compaction, and garbage collection remain planned. +Filesystem migration, retention execution, recovery, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 604d0e3..ee9bcb3 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -60,6 +60,11 @@ The format-definition digest is BLAKE3-256 of its domain followed by the exact corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. +`CanonicalStoreFormatMarker` produces the one registered marker, and +`AdmittedStoreFormatMarker` admits exact canonical bytes only after framing, +checksum, definition, and namespace-bound validation. Store detection and +filesystem migration remain absent. + ## Reader fence `reader.lock` is a persistent regular zero-length file. Its contents and diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 456f139..c84c9cf 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Intent and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | golden-format fixtures | Planned in #19 | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact format-marker codec and `tests/store_format_marker.rs`; intent and receipt admission remain | In progress in #19 | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index d55d271..20952e2 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -299,6 +299,7 @@ mod store_initialization_error; mod store_initialization_phase; mod store_initialization_receipt; mod store_initialization_storage; +mod store_migration; mod sync_capable_directory; #[cfg(test)] #[path = "../../tests/support/mod.rs"] @@ -481,6 +482,7 @@ pub use store_initialization_error::StoreInitializationError; pub use store_initialization_phase::StoreInitializationPhase; pub use store_initialization_receipt::StoreInitializationReceipt; pub use store_initialization_storage::StoreInitializationStorage; +pub use store_migration::*; pub use writer_lock_acquire_error::WriterLockAcquireError; pub use writer_lock_acquire_phase::WriterLockAcquirePhase; diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs new file mode 100644 index 0000000..87d54bc --- /dev/null +++ b/src/adapters/store_migration.rs @@ -0,0 +1,16 @@ +//! Canonical version-2 store migration record adapters. + +mod admitted_format_marker; +mod canonical_format_marker; +mod format_definition_digest; +mod format_marker_decode_error; +mod format_marker_decode_error_display; +mod format_marker_decoder; +mod format_marker_digest; +mod format_marker_encoder; + +pub use admitted_format_marker::AdmittedStoreFormatMarker; +pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use format_definition_digest::StoreFormatDefinitionDigest; +pub use format_marker_decode_error::StoreFormatMarkerDecodeError; +pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/admitted_format_marker.rs b/src/adapters/store_migration/admitted_format_marker.rs new file mode 100644 index 0000000..5a66024 --- /dev/null +++ b/src/adapters/store_migration/admitted_format_marker.rs @@ -0,0 +1,58 @@ +//! This boundary module owns admitted version-2 store-format markers. + +use super::{ + StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, + format_marker_decoder, +}; + +/// Borrowed canonical marker bytes with verified version-2 format identity. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AdmittedStoreFormatMarker<'encoded> { + encoded: &'encoded [u8], + definition_digest: StoreFormatDefinitionDigest, + digest: StoreFormatMarkerDigest, +} + +impl<'encoded> AdmittedStoreFormatMarker<'encoded> { + /// Decodes and verifies one exact version-2 store-format marker. + /// + /// This operation performs no allocation or I/O. + /// + /// # Errors + /// + /// Returns [`StoreFormatMarkerDecodeError`] for wrong framing, + /// unsupported fields, checksum disagreement, or an unregistered + /// definition or namespace bound. + pub fn decode(encoded: &'encoded [u8]) -> Result { + format_marker_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the registered format-definition digest. + pub const fn definition_digest(&self) -> StoreFormatDefinitionDigest { + self.definition_digest + } + + /// Returns the identity of all marker bytes. + pub const fn digest(&self) -> StoreFormatMarkerDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + definition_digest: StoreFormatDefinitionDigest, + digest: StoreFormatMarkerDigest, + ) -> Self { + Self { + encoded, + definition_digest, + digest, + } + } +} diff --git a/src/adapters/store_migration/canonical_format_marker.rs b/src/adapters/store_migration/canonical_format_marker.rs new file mode 100644 index 0000000..319be0e --- /dev/null +++ b/src/adapters/store_migration/canonical_format_marker.rs @@ -0,0 +1,36 @@ +//! This boundary module owns canonical version-2 store-format marker bytes. + +use super::{StoreFormatMarkerDigest, format_marker_decoder, format_marker_encoder}; + +/// Owned canonical version-2 store-format marker. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalStoreFormatMarker { + encoded: [u8; format_marker_decoder::ENCODED_LENGTH], + digest: StoreFormatMarkerDigest, +} + +impl CanonicalStoreFormatMarker { + /// Constructs the one registered version-2 marker. + pub fn version_two() -> Self { + format_marker_encoder::version_two() + } + + /// Returns the canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the identity of all marker bytes. + pub const fn digest(&self) -> StoreFormatMarkerDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: [u8; format_marker_decoder::ENCODED_LENGTH], + digest: StoreFormatMarkerDigest, + ) -> Self { + Self { encoded, digest } + } +} diff --git a/src/adapters/store_migration/format_definition_digest.rs b/src/adapters/store_migration/format_definition_digest.rs new file mode 100644 index 0000000..f9b29bb --- /dev/null +++ b/src/adapters/store_migration/format_definition_digest.rs @@ -0,0 +1,25 @@ +//! This module owns the registered version-2 format-definition digest. + +/// Identity of one registered store-format definition. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreFormatDefinitionDigest([u8; 32]); + +impl StoreFormatDefinitionDigest { + /// Digest of the frozen `keep.segment-store/v2` definition. + pub const VERSION_TWO: Self = Self([ + 0x32, 0x38, 0x1f, 0x1a, 0xc3, 0x32, 0xd1, 0x27, 0x7a, 0x7e, 0x1f, 0xaf, 0x8f, 0x11, 0x57, + 0x69, 0x93, 0xcb, 0x55, 0xb7, 0xe8, 0x5d, 0x2a, 0x11, 0x0b, 0x74, 0xdc, 0x9c, 0x3b, 0x87, + 0x34, 0x27, + ]); + + /// Returns the raw digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/format_marker_decode_error.rs b/src/adapters/store_migration/format_marker_decode_error.rs new file mode 100644 index 0000000..5102209 --- /dev/null +++ b/src/adapters/store_migration/format_marker_decode_error.rs @@ -0,0 +1,63 @@ +//! This boundary module owns store-format marker decoding failures. + +/// Failure to decode and admit one version-2 store-format marker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreFormatMarkerDecodeError { + /// The input was not exactly one complete marker. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed record length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The marker carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// Reserved bytes were nonzero. + NonZeroReserved { + /// Observed reserved field. + observed: u32, + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// The format-definition digest was not the registered version-2 value. + DefinitionDigestMismatch { + /// Registered version-2 definition digest. + expected: [u8; 32], + /// Observed definition digest. + observed: [u8; 32], + }, + /// The maximum namespace count was noncanonical. + InvalidMaximumNamespaceCount { + /// Required namespace bound. + expected: u32, + /// Observed namespace bound. + observed: u32, + }, +} diff --git a/src/adapters/store_migration/format_marker_decode_error_display.rs b/src/adapters/store_migration/format_marker_decode_error_display.rs new file mode 100644 index 0000000..435d2e3 --- /dev/null +++ b/src/adapters/store_migration/format_marker_decode_error_display.rs @@ -0,0 +1,50 @@ +//! This boundary module owns store-format marker decode diagnostics. + +use std::{error::Error, fmt}; + +use super::StoreFormatMarkerDecodeError; + +impl fmt::Display for StoreFormatMarkerDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "store-format marker has {observed} bytes; expected {expected}" + ), + Self::InvalidMagic { observed } => { + write!( + formatter, + "invalid store-format marker magic {observed:02x?}" + ) + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported store-format marker version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "store-format marker record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => write!( + formatter, + "unsupported store-format marker flags {observed:#010x}" + ), + Self::NonZeroReserved { observed } => write!( + formatter, + "store-format marker reserved field is nonzero: {observed:#010x}" + ), + Self::ChecksumMismatch { .. } => { + formatter.write_str("store-format marker checksum mismatch") + } + Self::DefinitionDigestMismatch { .. } => { + formatter.write_str("store-format definition digest mismatch") + } + Self::InvalidMaximumNamespaceCount { expected, observed } => write!( + formatter, + "store-format maximum namespace count {observed}; expected {expected}" + ), + } + } +} + +impl Error for StoreFormatMarkerDecodeError {} diff --git a/src/adapters/store_migration/format_marker_decoder.rs b/src/adapters/store_migration/format_marker_decoder.rs new file mode 100644 index 0000000..75ca8f1 --- /dev/null +++ b/src/adapters/store_migration/format_marker_decoder.rs @@ -0,0 +1,137 @@ +//! This boundary module owns store-format marker decoding order. + +use super::{ + AdmittedStoreFormatMarker, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, + StoreFormatMarkerDigest, +}; +use crate::RetentionManifest; + +pub(super) const ENCODED_LENGTH: usize = 96; +pub(super) const CHECKSUM_OFFSET: usize = 64; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:STORE:V2\0\0\0"; +pub(super) const VERSION: u16 = 2; +pub(super) const RECORD_LENGTH: u16 = 96; +const CHECKSUM_DOMAIN: &[u8] = b"keep.segment-store-marker-checksum/v2\0"; +const DIGEST_DOMAIN: &[u8] = b"keep.store-format-marker/v2\0"; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, StoreFormatMarkerDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let definition_hash = read_array(encoded, 24)?; + let definition_digest = StoreFormatDefinitionDigest::from_hash(definition_hash); + if definition_digest != StoreFormatDefinitionDigest::VERSION_TWO { + return Err(StoreFormatMarkerDecodeError::DefinitionDigestMismatch { + expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), + observed: definition_hash, + }); + } + let maximum_namespace_count = read_u32(encoded, 56)?; + if maximum_namespace_count != RetentionManifest::MAXIMUM_ENTRY_COUNT { + return Err(StoreFormatMarkerDecodeError::InvalidMaximumNamespaceCount { + expected: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed: maximum_namespace_count, + }); + } + Ok(AdmittedStoreFormatMarker::admitted( + encoded, + definition_digest, + digest(encoded), + )) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreFormatMarkerDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(StoreFormatMarkerDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(StoreFormatMarkerDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(StoreFormatMarkerDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(StoreFormatMarkerDecodeError::UnsupportedFlags { observed: flags }); + } + let reserved = read_u32(encoded, 60)?; + if reserved != 0 { + return Err(StoreFormatMarkerDecodeError::NonZeroReserved { observed: reserved }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), StoreFormatMarkerDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or_else(|| wrong_length(encoded))?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = checksum(preimage); + if observed == expected { + Ok(()) + } else { + Err(StoreFormatMarkerDecodeError::ChecksumMismatch { expected, observed }) + } +} + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + hash(CHECKSUM_DOMAIN, preimage) +} + +pub(super) fn digest(encoded: &[u8]) -> StoreFormatMarkerDigest { + StoreFormatMarkerDigest::from_hash(hash(DIGEST_DOMAIN, encoded)) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} + +const fn require_length(encoded: &[u8]) -> Result<(), StoreFormatMarkerDecodeError> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(wrong_length(encoded)) + } +} + +const fn wrong_length(encoded: &[u8]) -> StoreFormatMarkerDecodeError { + StoreFormatMarkerDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + } +} + +fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], StoreFormatMarkerDecodeError> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(wrong_length(encoded)); + }; + let bytes = encoded + .get(offset..end) + .ok_or_else(|| wrong_length(encoded))?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) +} diff --git a/src/adapters/store_migration/format_marker_digest.rs b/src/adapters/store_migration/format_marker_digest.rs new file mode 100644 index 0000000..39fc208 --- /dev/null +++ b/src/adapters/store_migration/format_marker_digest.rs @@ -0,0 +1,18 @@ +//! This module owns version-2 store-format marker identity. + +/// Domain-separated identity of all canonical store-format marker bytes. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreFormatMarkerDigest([u8; 32]); + +impl StoreFormatMarkerDigest { + /// Returns the raw digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/format_marker_encoder.rs b/src/adapters/store_migration/format_marker_encoder.rs new file mode 100644 index 0000000..14223b8 --- /dev/null +++ b/src/adapters/store_migration/format_marker_encoder.rs @@ -0,0 +1,27 @@ +//! This boundary module owns canonical version-2 format-marker encoding. + +use super::{ + CanonicalStoreFormatMarker, StoreFormatDefinitionDigest, format_marker_decoder as format, +}; +use crate::RetentionManifest; + +pub(super) fn version_two() -> CanonicalStoreFormatMarker { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + let (magic, remaining) = preimage.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, remaining) = remaining.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, remaining) = remaining.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, remaining) = remaining.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (definition_digest, remaining) = remaining.split_at_mut(32); + definition_digest.copy_from_slice(StoreFormatDefinitionDigest::VERSION_TWO.as_bytes()); + let (maximum_namespace_count, remaining) = remaining.split_at_mut(4); + maximum_namespace_count.copy_from_slice(&RetentionManifest::MAXIMUM_ENTRY_COUNT.to_be_bytes()); + let (_reserved, _complete) = remaining.split_at_mut(4); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + let digest = format::digest(&encoded); + CanonicalStoreFormatMarker::admitted(encoded, digest) +} diff --git a/src/lib.rs b/src/lib.rs index 91cd10a..118d258 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,8 +29,10 @@ //! with a blocking storage capability port are available. Storage-independent //! preparation binds preflight to exact canonical manifest and head successors. //! Ordered publication revalidates authority, executes all durability phases, -//! and returns a complete receipt. Filesystem execution, recovery, and garbage -//! collection remain intentionally absent. +//! and returns a complete receipt. The exact version-2 store-format marker has +//! canonical encoding, registered-definition admission, checksum verification, +//! and domain-separated identity. Filesystem migration, retention execution, +//! recovery, and garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -49,41 +51,41 @@ mod retention; pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, - FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, - FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + AdmittedStoreFormatMarker, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, + CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, + CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, + CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, + CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, + CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, + FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, + RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -98,6 +100,7 @@ pub use adapters::{ SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, diff --git a/tests/store_format_marker.rs b/tests/store_format_marker.rs new file mode 100644 index 0000000..bc23f60 --- /dev/null +++ b/tests/store_format_marker.rs @@ -0,0 +1,168 @@ +//! Canonical version-2 store-format marker laws. + +mod support; + +use std::io; + +use keep::{ + AdmittedStoreFormatMarker, CanonicalStoreFormatMarker, StoreFormatDefinitionDigest, + StoreFormatMarkerDecodeError, +}; + +const FORMAT_MARKER: &str = include_str!("../conformance/segment-store/v2/format-marker.hex"); +const MARKER_DIGEST: [u8; 32] = [ + 0x4b, 0x06, 0x3c, 0x32, 0x90, 0x85, 0xab, 0xde, 0xbe, 0x86, 0xb2, 0x56, 0xd5, 0x31, 0xb1, 0x12, + 0xc7, 0xea, 0x33, 0xcb, 0x2f, 0x54, 0x5c, 0xaa, 0x40, 0xa7, 0xa8, 0x69, 0xff, 0x33, 0x37, 0xce, +]; + +#[test] +fn marker_reproduces_the_frozen_version_two_record() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let admitted = AdmittedStoreFormatMarker::decode(&bytes)?; + let canonical = CanonicalStoreFormatMarker::version_two(); + + assert_eq!(admitted.encoded(), bytes); + assert_eq!( + admitted.definition_digest(), + StoreFormatDefinitionDigest::VERSION_TWO + ); + assert_eq!(admitted.digest().as_bytes(), &MARKER_DIGEST); + assert_eq!(canonical.encoded(), bytes); + assert_eq!(canonical.digest(), admitted.digest()); + Ok(()) +} + +#[test] +fn marker_framing_has_exact_first_refusals() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + AdmittedStoreFormatMarker::decode(&truncated), + Err(StoreFormatMarkerDecodeError::WrongLength { + expected: 96, + observed: 95, + }) + )); + + assert_fixed_refusal( + 0, + StoreFormatMarkerDecodeError::InvalidMagic { + observed: mutated_array::<16>(&bytes, 0, 0)?, + }, + )?; + assert_fixed_refusal( + 17, + StoreFormatMarkerDecodeError::UnsupportedVersion { + expected: 2, + observed: 3, + }, + )?; + assert_fixed_refusal( + 19, + StoreFormatMarkerDecodeError::InvalidRecordLength { + expected: 96, + observed: 97, + }, + )?; + assert_fixed_refusal( + 23, + StoreFormatMarkerDecodeError::UnsupportedFlags { observed: 1 }, + )?; + assert_fixed_refusal( + 63, + StoreFormatMarkerDecodeError::NonZeroReserved { observed: 1 }, + )?; + Ok(()) +} + +#[test] +fn checksum_precedes_registered_marker_semantics() -> Result<(), Box> { + let mut definition = fixture_bytes()?; + flip_byte(&mut definition, 24)?; + assert!(matches!( + AdmittedStoreFormatMarker::decode(&definition), + Err(StoreFormatMarkerDecodeError::ChecksumMismatch { .. }) + )); + refresh_checksum(&mut definition)?; + assert!(matches!( + AdmittedStoreFormatMarker::decode(&definition), + Err(StoreFormatMarkerDecodeError::DefinitionDigestMismatch { .. }) + )); + + let mut namespace_limit = fixture_bytes()?; + namespace_limit + .get_mut(56..60) + .ok_or_else(|| io::Error::other("marker lacks namespace limit"))? + .copy_from_slice(&4_095_u32.to_be_bytes()); + refresh_checksum(&mut namespace_limit)?; + assert_eq!( + AdmittedStoreFormatMarker::decode(&namespace_limit), + Err(StoreFormatMarkerDecodeError::InvalidMaximumNamespaceCount { + expected: 4_096, + observed: 4_095, + }) + ); + + let mut checksum = fixture_bytes()?; + flip_byte(&mut checksum, 95)?; + assert!(matches!( + AdmittedStoreFormatMarker::decode(&checksum), + Err(StoreFormatMarkerDecodeError::ChecksumMismatch { .. }) + )); + Ok(()) +} + +fn assert_fixed_refusal( + offset: usize, + expected: StoreFormatMarkerDecodeError, +) -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, offset)?; + assert_eq!(AdmittedStoreFormatMarker::decode(&bytes), Err(expected)); + Ok(()) +} + +fn mutated_array( + bytes: &[u8], + offset: usize, + relative: usize, +) -> Result<[u8; WIDTH], io::Error> { + let end = offset + .checked_add(WIDTH) + .ok_or_else(|| io::Error::other("marker field offset overflow"))?; + let mut observed = <[u8; WIDTH]>::try_from( + bytes + .get(offset..end) + .ok_or_else(|| io::Error::other("marker lacks fixed field"))?, + ) + .map_err(|_| io::Error::other("marker field width mismatch"))?; + let byte = observed + .get_mut(relative) + .ok_or_else(|| io::Error::other("marker mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(observed) +} + +fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { + let byte = bytes + .get_mut(offset) + .ok_or_else(|| io::Error::other("marker mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(()) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, checksum) = bytes + .split_at_mut_checked(64) + .ok_or_else(|| io::Error::other("marker lacks checksum boundary"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.segment-store-marker-checksum/v2\0"); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} + +fn fixture_bytes() -> Result, io::Error> { + support::decode_hex(FORMAT_MARKER.trim_end()) +} From fc20097f4e0cc5cd88d3ea6e03d539a030e515f8 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:04:19 -0700 Subject: [PATCH 31/50] Add: Admit store migration intents --- CHANGELOG.md | 12 +- README.md | 22 +- docs/formats/segment-store-v2/recovery.md | 8 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 17 ++ .../admitted_migration_intent.rs | 122 +++++++++++ .../immutable_pool_inventory_digest.rs | 20 ++ .../store_migration/migration_intent_bytes.rs | 54 +++++ .../migration_intent_decode_error.rs | 84 ++++++++ .../migration_intent_decode_error_display.rs | 64 ++++++ .../migration_intent_decoder.rs | 181 ++++++++++++++++ .../migration_intent_digest.rs | 18 ++ .../store_migration/store_identifier.rs | 18 ++ .../store_migration/store_root_identity.rs | 38 ++++ src/lib.rs | 48 +++-- tests/store_migration_intent.rs | 200 ++++++++++++++++++ tests/store_migration_intent/fixture.rs | 33 +++ 17 files changed, 898 insertions(+), 43 deletions(-) create mode 100644 src/adapters/store_migration/admitted_migration_intent.rs create mode 100644 src/adapters/store_migration/immutable_pool_inventory_digest.rs create mode 100644 src/adapters/store_migration/migration_intent_bytes.rs create mode 100644 src/adapters/store_migration/migration_intent_decode_error.rs create mode 100644 src/adapters/store_migration/migration_intent_decode_error_display.rs create mode 100644 src/adapters/store_migration/migration_intent_decoder.rs create mode 100644 src/adapters/store_migration/migration_intent_digest.rs create mode 100644 src/adapters/store_migration/store_identifier.rs create mode 100644 src/adapters/store_migration/store_root_identity.rs create mode 100644 tests/store_migration_intent.rs create mode 100644 tests/store_migration_intent/fixture.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2afa12d..beb0ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- The version-2 store-format marker now has exact canonical encoding, - registered-definition admission, checksum verification, and domain-separated - identity. Retention transition preflight combines expected-generation - planning with deterministic closure verification against one pinned catalog; - authority-revalidated 17-phase orchestration returns an unforgeable - complete-coordinate receipt after durable cleanup. +- The version-2 store-format marker now has exact canonical encoding and + admission, while migration intents admit exact catalog, predecessor, root, + definition, store-identity, checksum, and digest coordinates. Retention + preflight combines expected-generation planning with deterministic closure + verification; authority-revalidated 17-phase orchestration returns an + unforgeable complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 00f27e8..7406736 100644 --- a/README.md +++ b/README.md @@ -115,17 +115,17 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Canonical version-2 store-format marker encoding and admission are -implemented. Version-2 retention values; canonical in-memory root, global -manifest, and retention-head codecs; storage-independent expected-state -transition planning; deterministic bounded closure verification against a -pinned catalog; a combined transition preflight proof; and the exact 17-phase -publication vocabulary with a blocking storage capability port are -implemented. Private-field proofs retain every receipt coordinate. Ordered -storage-port orchestration revalidates current authority, executes all 17 -durability phases, and returns a consequential complete-coordinate receipt. -Filesystem migration, retention execution, recovery, compaction, and garbage -collection remain planned. +power loss. Canonical version-2 store-format marker encoding and exact +migration-intent admission are implemented. Version-2 retention values; +canonical in-memory root, global manifest, and retention-head codecs; +storage-independent expected-state transition planning; deterministic bounded +closure verification against a pinned catalog; a combined transition preflight +proof; and the exact 17-phase publication vocabulary with a blocking storage +capability port are implemented. Private-field proofs retain every receipt +coordinate. Ordered storage-port orchestration revalidates current authority, +executes all 17 durability phases, and returns a consequential +complete-coordinate receipt. Filesystem migration, retention execution, +recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index ee9bcb3..4f7ad6b 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -60,10 +60,10 @@ The format-definition digest is BLAKE3-256 of its domain followed by the exact corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. -`CanonicalStoreFormatMarker` produces the one registered marker, and -`AdmittedStoreFormatMarker` admits exact canonical bytes only after framing, -checksum, definition, and namespace-bound validation. Store detection and -filesystem migration remain absent. +`CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. +`AdmittedStoreMigrationIntent` admits the exact intent framing, checksum, catalog coordinates, predecessor law, definition, and derived store identity. +These record boundaries do not prove the named live inventory or physical root, +detect the store version, or execute filesystem migration. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index c84c9cf..afb0409 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact format-marker codec and `tests/store_format_marker.rs`; intent and receipt admission remain | In progress in #19 | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact marker and intent admission in `tests/store_format_marker.rs` and `tests/store_migration_intent.rs`; receipt admission remains | In progress in #19 | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 87d54bc..19effb7 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -1,6 +1,7 @@ //! Canonical version-2 store migration record adapters. mod admitted_format_marker; +mod admitted_migration_intent; mod canonical_format_marker; mod format_definition_digest; mod format_marker_decode_error; @@ -8,9 +9,25 @@ mod format_marker_decode_error_display; mod format_marker_decoder; mod format_marker_digest; mod format_marker_encoder; +mod immutable_pool_inventory_digest; +mod migration_intent_bytes; +mod migration_intent_decode_error; +mod migration_intent_decode_error_display; +mod migration_intent_decoder; +mod migration_intent_digest; +mod store_identifier; +mod store_root_identity; pub use admitted_format_marker::AdmittedStoreFormatMarker; +pub use admitted_migration_intent::AdmittedStoreMigrationIntent; pub use canonical_format_marker::CanonicalStoreFormatMarker; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; +pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; +pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; +pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use store_identifier::StoreIdentifier; +pub use store_root_identity::{ + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, +}; diff --git a/src/adapters/store_migration/admitted_migration_intent.rs b/src/adapters/store_migration/admitted_migration_intent.rs new file mode 100644 index 0000000..fa625b1 --- /dev/null +++ b/src/adapters/store_migration/admitted_migration_intent.rs @@ -0,0 +1,122 @@ +//! This boundary module owns admitted store-migration intent evidence. + +use super::{ + ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, + StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, migration_intent_decoder, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Semantic fields admitted from one canonical migration intent. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct StoreMigrationIntentFields { + pub(super) catalog_generation: CatalogGeneration, + pub(super) catalog_length: CatalogLength, + pub(super) catalog_digest: CatalogDigest, + pub(super) predecessor_catalog_digest: Option, + pub(super) inventory_digest: ImmutablePoolInventoryDigest, + pub(super) root_device_identity: StoreRootDeviceIdentity, + pub(super) root_mount_identity: StoreRootMountIdentity, + pub(super) root_file_identity: StoreRootFileIdentity, + pub(super) target_definition_digest: StoreFormatDefinitionDigest, + pub(super) store_identifier: StoreIdentifier, +} + +/// Borrowed canonical version-2 store-migration intent. +/// +/// Admission proves record framing, integrity, internal generation laws, the +/// registered target definition, and deterministic store identity. It does not +/// prove that the named catalog, inventory, or physical root is current. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdmittedStoreMigrationIntent<'encoded> { + encoded: &'encoded [u8], + fields: StoreMigrationIntentFields, + digest: StoreMigrationIntentDigest, +} + +impl<'encoded> AdmittedStoreMigrationIntent<'encoded> { + /// Decodes and admits one exact canonical migration intent. + /// + /// # Errors + /// + /// Returns [`StoreMigrationIntentDecodeError`] for invalid framing, + /// integrity, catalog coordinates, predecessor state, definition identity, + /// or store identity. + pub fn decode(encoded: &'encoded [u8]) -> Result { + migration_intent_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the positive catalog generation named by the intent. + pub const fn catalog_generation(&self) -> CatalogGeneration { + self.fields.catalog_generation + } + + /// Returns the exact admitted catalog byte length. + pub const fn catalog_length(&self) -> CatalogLength { + self.fields.catalog_length + } + + /// Returns the catalog digest named by the intent. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.fields.catalog_digest + } + + /// Returns the generation-relative predecessor digest. + pub const fn predecessor_catalog_digest(&self) -> Option { + self.fields.predecessor_catalog_digest + } + + /// Returns the immutable-pool inventory digest named by the intent. + pub const fn inventory_digest(&self) -> ImmutablePoolInventoryDigest { + self.fields.inventory_digest + } + + /// Returns the serialized root device coordinate named by the intent. + pub const fn root_device_identity(&self) -> StoreRootDeviceIdentity { + self.fields.root_device_identity + } + + /// Returns the serialized root mount coordinate named by the intent. + pub const fn root_mount_identity(&self) -> StoreRootMountIdentity { + self.fields.root_mount_identity + } + + /// Returns the serialized root file coordinate named by the intent. + pub const fn root_file_identity(&self) -> StoreRootFileIdentity { + self.fields.root_file_identity + } + + /// Returns the registered target format-definition digest. + pub const fn target_definition_digest(&self) -> StoreFormatDefinitionDigest { + self.fields.target_definition_digest + } + + /// Returns the deterministic logical store identity. + pub const fn store_identifier(&self) -> StoreIdentifier { + self.fields.store_identifier + } + + /// Returns the domain-separated identity of all intent bytes. + pub const fn digest(&self) -> StoreMigrationIntentDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + fields: StoreMigrationIntentFields, + digest: StoreMigrationIntentDigest, + ) -> Self { + Self { + encoded, + fields, + digest, + } + } +} diff --git a/src/adapters/store_migration/immutable_pool_inventory_digest.rs b/src/adapters/store_migration/immutable_pool_inventory_digest.rs new file mode 100644 index 0000000..188c0aa --- /dev/null +++ b/src/adapters/store_migration/immutable_pool_inventory_digest.rs @@ -0,0 +1,20 @@ +//! This module owns immutable-pool inventory identity. + +/// Digest coordinate naming one canonical complete immutable-pool inventory. +/// +/// Intent admission does not prove that a current inventory has this digest. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct ImmutablePoolInventoryDigest([u8; 32]); + +impl ImmutablePoolInventoryDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_admitted(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/adapters/store_migration/migration_intent_bytes.rs b/src/adapters/store_migration/migration_intent_bytes.rs new file mode 100644 index 0000000..4f6906b --- /dev/null +++ b/src/adapters/store_migration/migration_intent_bytes.rs @@ -0,0 +1,54 @@ +//! This boundary module owns fixed-width migration-intent field access. + +use super::StoreMigrationIntentDecodeError; + +pub(super) const ENCODED_LENGTH: usize = 256; + +pub(super) const fn require_length(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(wrong_length(encoded)) + } +} + +pub(super) const fn wrong_length(encoded: &[u8]) -> StoreMigrationIntentDecodeError { + StoreMigrationIntentDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + } +} + +pub(super) fn read_u16( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], StoreMigrationIntentDecodeError> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(wrong_length(encoded)); + }; + let bytes = encoded + .get(offset..end) + .ok_or_else(|| wrong_length(encoded))?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) +} diff --git a/src/adapters/store_migration/migration_intent_decode_error.rs b/src/adapters/store_migration/migration_intent_decode_error.rs new file mode 100644 index 0000000..5d6c1e3 --- /dev/null +++ b/src/adapters/store_migration/migration_intent_decode_error.rs @@ -0,0 +1,84 @@ +//! This boundary module owns store-migration intent decoding failures. + +use crate::{CatalogGenerationError, CatalogLengthError}; + +/// Failure to decode and admit one version-2 migration intent. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationIntentDecodeError { + /// The input was not exactly one complete intent. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The record-length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The intent carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// The catalog generation was not positive. + InvalidCatalogGeneration { + /// Observed generation. + observed: u64, + /// Precise generation refusal. + source: CatalogGenerationError, + }, + /// The catalog length was outside the canonical version-1 grammar. + InvalidCatalogLength { + /// Observed length. + observed: u64, + /// Precise catalog-length refusal. + source: CatalogLengthError, + }, + /// Generation 1 carried a forbidden predecessor. + NonZeroInitialPredecessor { + /// Observed nonzero predecessor digest. + observed: [u8; 32], + }, + /// A later generation omitted its required predecessor. + MissingSuccessorPredecessor { + /// Observed later generation. + generation: u64, + }, + /// The target definition was not the registered version-2 definition. + DefinitionDigestMismatch { + /// Registered version-2 definition digest. + expected: [u8; 32], + /// Observed target definition digest. + observed: [u8; 32], + }, + /// The stored identifier did not match the deterministic derivation. + StoreIdentifierMismatch { + /// Identifier derived from the admitted semantic fields. + expected: [u8; 32], + /// Identifier stored in the record. + observed: [u8; 32], + }, +} diff --git a/src/adapters/store_migration/migration_intent_decode_error_display.rs b/src/adapters/store_migration/migration_intent_decode_error_display.rs new file mode 100644 index 0000000..daeebd2 --- /dev/null +++ b/src/adapters/store_migration/migration_intent_decode_error_display.rs @@ -0,0 +1,64 @@ +//! This boundary module owns migration-intent error formatting and sources. + +use std::error::Error; +use std::fmt; + +use super::StoreMigrationIntentDecodeError; + +impl fmt::Display for StoreMigrationIntentDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "migration intent requires {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { .. } => formatter.write_str("invalid migration-intent magic"), + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported migration-intent version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "migration-intent record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported migration-intent flags {observed:#010x}" + ) + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("migration-intent checksum mismatch") + } + Self::InvalidCatalogGeneration { observed, .. } => { + write!(formatter, "invalid migration catalog generation {observed}") + } + Self::InvalidCatalogLength { observed, .. } => { + write!(formatter, "invalid migration catalog length {observed}") + } + Self::NonZeroInitialPredecessor { .. } => { + formatter.write_str("initial migration catalog forbids a predecessor") + } + Self::MissingSuccessorPredecessor { generation } => write!( + formatter, + "migration catalog generation {generation} requires a predecessor" + ), + Self::DefinitionDigestMismatch { .. } => { + formatter.write_str("migration target definition digest mismatch") + } + Self::StoreIdentifierMismatch { .. } => { + formatter.write_str("migration store identifier mismatch") + } + } + } +} + +impl Error for StoreMigrationIntentDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::InvalidCatalogGeneration { source, .. } => Some(source), + Self::InvalidCatalogLength { source, .. } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs new file mode 100644 index 0000000..0108c7a --- /dev/null +++ b/src/adapters/store_migration/migration_intent_decoder.rs @@ -0,0 +1,181 @@ +//! This boundary module owns store-migration intent decoding order. + +use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_intent_bytes::{ + read_array, read_u16, read_u32, read_u64, require_length, wrong_length, +}; +use super::{ + AdmittedStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, + StoreIdentifier, StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +const CHECKSUM_OFFSET: usize = 224; +const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; +const VERSION: u16 = 2; +const RECORD_LENGTH: u16 = 256; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; +const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; +const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; +const ZERO_DIGEST: [u8; 32] = [0; 32]; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, StoreMigrationIntentDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let catalog_generation = read_catalog_generation(encoded)?; + let catalog_length = read_catalog_length(encoded)?; + let catalog_digest = CatalogDigest::from_validated(read_array(encoded, 40)?); + let predecessor_catalog_digest = + read_predecessor(catalog_generation, read_array(encoded, 72)?)?; + let fields = StoreMigrationIntentFields { + catalog_generation, + catalog_length, + catalog_digest, + predecessor_catalog_digest, + inventory_digest: ImmutablePoolInventoryDigest::from_admitted(read_array(encoded, 104)?), + root_device_identity: StoreRootDeviceIdentity::from_admitted(read_u64(encoded, 136)?), + root_mount_identity: StoreRootMountIdentity::from_admitted(read_u64(encoded, 144)?), + root_file_identity: StoreRootFileIdentity::from_admitted(read_u64(encoded, 152)?), + target_definition_digest: read_definition_digest(encoded)?, + store_identifier: StoreIdentifier::from_hash(read_array(encoded, 192)?), + }; + verify_store_identifier(&fields)?; + Ok(AdmittedStoreMigrationIntent::admitted( + encoded, + fields, + digest(encoded), + )) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(StoreMigrationIntentDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(StoreMigrationIntentDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(StoreMigrationIntentDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(StoreMigrationIntentDecodeError::UnsupportedFlags { observed: flags }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or_else(|| wrong_length(encoded))?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = hash(CHECKSUM_DOMAIN, &[preimage]); + if observed == expected { + Ok(()) + } else { + Err(StoreMigrationIntentDecodeError::ChecksumMismatch { expected, observed }) + } +} + +fn read_catalog_generation( + encoded: &[u8], +) -> Result { + let observed = read_u64(encoded, 24)?; + CatalogGeneration::new(observed).map_err(|source| { + StoreMigrationIntentDecodeError::InvalidCatalogGeneration { observed, source } + }) +} + +fn read_catalog_length(encoded: &[u8]) -> Result { + let observed = read_u64(encoded, 32)?; + CatalogLength::new(observed).map_err(|source| { + StoreMigrationIntentDecodeError::InvalidCatalogLength { observed, source } + }) +} + +fn read_predecessor( + generation: CatalogGeneration, + observed: [u8; 32], +) -> Result, StoreMigrationIntentDecodeError> { + if generation.get() == 1 { + return if observed == ZERO_DIGEST { + Ok(None) + } else { + Err(StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { observed }) + }; + } + if observed == ZERO_DIGEST { + return Err( + StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { + generation: generation.get(), + }, + ); + } + Ok(Some(CatalogDigest::from_validated(observed))) +} + +fn read_definition_digest( + encoded: &[u8], +) -> Result { + let observed = read_array(encoded, 160)?; + if observed == *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes() { + Ok(StoreFormatDefinitionDigest::VERSION_TWO) + } else { + Err(StoreMigrationIntentDecodeError::DefinitionDigestMismatch { + expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), + observed, + }) + } +} + +fn verify_store_identifier( + fields: &StoreMigrationIntentFields, +) -> Result<(), StoreMigrationIntentDecodeError> { + let predecessor = fields + .predecessor_catalog_digest + .as_ref() + .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); + let expected = hash( + STORE_IDENTIFIER_DOMAIN, + &[ + &fields.catalog_generation.get().to_be_bytes(), + &fields.catalog_length.get().to_be_bytes(), + fields.catalog_digest.as_bytes(), + predecessor, + fields.inventory_digest.as_bytes(), + fields.target_definition_digest.as_bytes(), + ], + ); + let observed = *fields.store_identifier.as_bytes(); + if observed == expected { + Ok(()) + } else { + Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { expected, observed }) + } +} + +fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { + StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) +} + +fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + for field in fields { + hasher.update(field); + } + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/store_migration/migration_intent_digest.rs b/src/adapters/store_migration/migration_intent_digest.rs new file mode 100644 index 0000000..8cce8fc --- /dev/null +++ b/src/adapters/store_migration/migration_intent_digest.rs @@ -0,0 +1,18 @@ +//! This module owns canonical migration-intent identity. + +/// Domain-separated digest of one complete canonical migration intent. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreMigrationIntentDigest([u8; 32]); + +impl StoreMigrationIntentDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/store_identifier.rs b/src/adapters/store_migration/store_identifier.rs new file mode 100644 index 0000000..27f6197 --- /dev/null +++ b/src/adapters/store_migration/store_identifier.rs @@ -0,0 +1,18 @@ +//! This module owns deterministic logical store identity. + +/// Logical identity derived from admitted version-1 state and the v2 format. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreIdentifier([u8; 32]); + +impl StoreIdentifier { + /// Returns the exact identifier bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/store_root_identity.rs b/src/adapters/store_migration/store_root_identity.rs new file mode 100644 index 0000000..39bc4bf --- /dev/null +++ b/src/adapters/store_migration/store_root_identity.rs @@ -0,0 +1,38 @@ +//! This module owns physical store-root recovery coordinates. + +macro_rules! root_identity { + ($name:ident, $documentation:literal) => { + #[doc = $documentation] + #[must_use] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct $name(u64); + + impl $name { + /// Returns the exact serialized platform coordinate. + /// + /// This value remains a comparison coordinate until a platform + /// adapter revalidates it against the opened store root. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } + + pub(super) const fn from_admitted(value: u64) -> Self { + Self(value) + } + } + }; +} + +root_identity!( + StoreRootDeviceIdentity, + "Platform device identity bound into a migration intent." +); +root_identity!( + StoreRootMountIdentity, + "Platform mount identity bound into a migration intent." +); +root_identity!( + StoreRootFileIdentity, + "Platform file identity bound into a migration intent." +); diff --git a/src/lib.rs b/src/lib.rs index 118d258..c09acfc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,10 @@ //! Ordered publication revalidates authority, executes all durability phases, //! and returns a complete receipt. The exact version-2 store-format marker has //! canonical encoding, registered-definition admission, checksum verification, -//! and domain-separated identity. Filesystem migration, retention execution, +//! and domain-separated identity. Migration-intent admission validates its +//! framing, checksum, catalog and predecessor grammar, registered definition, +//! deterministic store identity, and typed recovery coordinates. Live +//! inventory and root revalidation, filesystem migration, retention execution, //! recovery, and garbage collection remain intentionally absent. #[cfg(test)] @@ -51,24 +54,25 @@ mod retention; pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - AdmittedStoreFormatMarker, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, - CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, - CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, - CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, - CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, - CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, - CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, - FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, - FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, BlobIdBinaryParseError, + BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, + CanonicalStoreFormatMarker, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, + ImmutablePoolInventoryDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, @@ -101,8 +105,10 @@ pub use adapters::{ SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + StoreIdentifier, StoreInitializationError, StoreInitializationPhase, + StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, + StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, + StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, diff --git a/tests/store_migration_intent.rs b/tests/store_migration_intent.rs new file mode 100644 index 0000000..2040286 --- /dev/null +++ b/tests/store_migration_intent.rs @@ -0,0 +1,200 @@ +//! Canonical version-2 store-migration intent laws. + +#[path = "store_migration_intent/fixture.rs"] +mod fixture; +mod support; + +use std::io; + +use fixture::{CATALOG_DIGEST, INTENT_DIGEST, INVENTORY_DIGEST, STORE_IDENTIFIER, fixture_bytes}; +use keep::{ + AdmittedStoreMigrationIntent, CatalogGeneration, CatalogGenerationError, CatalogLength, + CatalogLengthError, StoreFormatDefinitionDigest, StoreMigrationIntentDecodeError, +}; + +#[test] +fn intent_admits_every_frozen_coordinate() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let intent = AdmittedStoreMigrationIntent::decode(&bytes)?; + + assert_eq!(intent.encoded(), bytes); + assert_eq!(intent.catalog_generation(), CatalogGeneration::new(1)?); + assert_eq!(intent.catalog_length(), CatalogLength::new(352)?); + assert_eq!(intent.catalog_digest().as_bytes(), &CATALOG_DIGEST); + assert_eq!(intent.predecessor_catalog_digest(), None); + assert_eq!(intent.inventory_digest().as_bytes(), &INVENTORY_DIGEST); + assert_eq!(intent.root_device_identity().get(), 1); + assert_eq!(intent.root_mount_identity().get(), 2); + assert_eq!(intent.root_file_identity().get(), 3); + assert_eq!( + intent.target_definition_digest(), + StoreFormatDefinitionDigest::VERSION_TWO + ); + assert_eq!(intent.store_identifier().as_bytes(), &STORE_IDENTIFIER); + assert_eq!(intent.digest().as_bytes(), &INTENT_DIGEST); + Ok(()) +} + +#[test] +fn intent_framing_has_exact_first_refusals() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert_eq!( + AdmittedStoreMigrationIntent::decode(&truncated), + Err(StoreMigrationIntentDecodeError::WrongLength { + expected: 256, + observed: 255, + }) + ); + let mut extended = bytes.clone(); + extended.push(0); + assert_eq!( + AdmittedStoreMigrationIntent::decode(&extended), + Err(StoreMigrationIntentDecodeError::WrongLength { + expected: 256, + observed: 257, + }) + ); + assert_fixed_refusal( + 0, + StoreMigrationIntentDecodeError::InvalidMagic { + observed: mutated_array(&bytes, 0, 0)?, + }, + )?; + assert_fixed_refusal( + 17, + StoreMigrationIntentDecodeError::UnsupportedVersion { + expected: 2, + observed: 3, + }, + )?; + assert_fixed_refusal( + 19, + StoreMigrationIntentDecodeError::InvalidRecordLength { + expected: 256, + observed: 257, + }, + )?; + assert_fixed_refusal( + 23, + StoreMigrationIntentDecodeError::UnsupportedFlags { observed: 1 }, + )?; + Ok(()) +} + +#[test] +fn checksum_and_semantic_laws_have_exact_precedence() -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, 31)?; + assert!(matches!( + AdmittedStoreMigrationIntent::decode(&bytes), + Err(StoreMigrationIntentDecodeError::ChecksumMismatch { .. }) + )); + refresh_checksum(&mut bytes)?; + assert_eq!( + AdmittedStoreMigrationIntent::decode(&bytes), + Err(StoreMigrationIntentDecodeError::InvalidCatalogGeneration { + observed: 0, + source: CatalogGenerationError::Zero, + }) + ); + + assert_semantic_refusal( + 39, + StoreMigrationIntentDecodeError::InvalidCatalogLength { + observed: 353, + source: CatalogLengthError::NotCongruent { observed: 353 }, + }, + )?; + assert_semantic_refusal( + 103, + StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { + observed: mutated_array(&fixture_bytes()?, 72, 31)?, + }, + )?; + + let mut successor = fixture_bytes()?; + let generation = successor + .get_mut(31) + .ok_or_else(|| io::Error::other("intent lacks generation field"))?; + *generation = 2; + refresh_checksum(&mut successor)?; + assert_eq!( + AdmittedStoreMigrationIntent::decode(&successor), + Err(StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { generation: 2 }) + ); + + assert_semantic_refusal( + 160, + StoreMigrationIntentDecodeError::DefinitionDigestMismatch { + expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), + observed: mutated_array(&fixture_bytes()?, 160, 0)?, + }, + )?; + assert_semantic_refusal( + 223, + StoreMigrationIntentDecodeError::StoreIdentifierMismatch { + expected: STORE_IDENTIFIER, + observed: mutated_array(&fixture_bytes()?, 192, 31)?, + }, + )?; + Ok(()) +} + +fn assert_fixed_refusal( + offset: usize, + expected: StoreMigrationIntentDecodeError, +) -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, offset)?; + assert_eq!(AdmittedStoreMigrationIntent::decode(&bytes), Err(expected)); + Ok(()) +} + +fn assert_semantic_refusal( + offset: usize, + expected: StoreMigrationIntentDecodeError, +) -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, offset)?; + refresh_checksum(&mut bytes)?; + assert_eq!(AdmittedStoreMigrationIntent::decode(&bytes), Err(expected)); + Ok(()) +} + +fn mutated_array( + bytes: &[u8], + offset: usize, + relative: usize, +) -> Result<[u8; WIDTH], io::Error> { + let end = offset + .checked_add(WIDTH) + .ok_or_else(|| io::Error::other("intent field offset overflow"))?; + let field = bytes + .get(offset..end) + .ok_or_else(|| io::Error::other("intent lacks fixed field"))?; + let mut observed = <[u8; WIDTH]>::try_from(field) + .map_err(|_| io::Error::other("intent field width mismatch"))?; + flip_byte(&mut observed, relative)?; + Ok(observed) +} + +fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { + let byte = bytes + .get_mut(offset) + .ok_or_else(|| io::Error::other("intent mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(()) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, checksum) = bytes + .split_at_mut_checked(224) + .ok_or_else(|| io::Error::other("intent lacks checksum boundary"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.store-migration-intent-checksum/v2\0"); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} diff --git a/tests/store_migration_intent/fixture.rs b/tests/store_migration_intent/fixture.rs new file mode 100644 index 0000000..83da59d --- /dev/null +++ b/tests/store_migration_intent/fixture.rs @@ -0,0 +1,33 @@ +//! This module owns frozen migration-intent fixture values. +#![allow( + clippy::redundant_pub_crate, + reason = "the parent integration-test module consumes this private fixture" +)] + +use std::io; + +use super::support; + +const MIGRATION_INTENT: &str = + include_str!("../../conformance/segment-store/v2/migration-intent.hex"); + +pub(super) const CATALOG_DIGEST: [u8; 32] = [ + 0x04, 0xb8, 0x25, 0x19, 0xb0, 0x39, 0x9b, 0xae, 0xfd, 0x0b, 0x9c, 0x0f, 0x32, 0xa8, 0x71, 0x05, + 0x2e, 0x4c, 0x47, 0xe3, 0xa0, 0x02, 0x26, 0xab, 0x03, 0xb2, 0x16, 0x61, 0x47, 0x0f, 0x73, 0x20, +]; +pub(super) const INVENTORY_DIGEST: [u8; 32] = [ + 0x40, 0xbf, 0x5d, 0x49, 0xc3, 0x48, 0x47, 0xac, 0x9c, 0xf4, 0x6a, 0x25, 0x6f, 0x34, 0x3c, 0xee, + 0x80, 0xcd, 0x98, 0x0d, 0x14, 0x05, 0xd2, 0xdd, 0x02, 0xce, 0xff, 0x8f, 0x58, 0xd6, 0x74, 0xf9, +]; +pub(super) const STORE_IDENTIFIER: [u8; 32] = [ + 0x0c, 0xd9, 0xd3, 0xdf, 0xbe, 0xc9, 0xb3, 0x49, 0xfe, 0x42, 0xd2, 0x14, 0x75, 0x27, 0x1b, 0x0e, + 0x8d, 0xe2, 0x3c, 0x04, 0x34, 0x40, 0xd6, 0x42, 0x7a, 0x1c, 0x37, 0x89, 0x8a, 0xd1, 0xdd, 0x79, +]; +pub(super) const INTENT_DIGEST: [u8; 32] = [ + 0xa1, 0x5a, 0x00, 0x00, 0x02, 0x19, 0xdf, 0x20, 0x97, 0x9d, 0xa3, 0x64, 0x19, 0x04, 0x6e, 0xae, + 0x9a, 0x0b, 0xa9, 0x98, 0x64, 0x5f, 0xbf, 0xe3, 0x08, 0xea, 0x43, 0x35, 0xa8, 0x32, 0x6b, 0x44, +]; + +pub(super) fn fixture_bytes() -> Result, io::Error> { + support::decode_hex(MIGRATION_INTENT.trim_end()) +} From 94092ac0ad8742dfeb15fc36c70f848225cf69fd Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:26:18 -0700 Subject: [PATCH 32/50] Add: Admit store migration receipts --- CHANGELOG.md | 12 +- README.md | 16 +- docs/formats/segment-store-v2/recovery.md | 6 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 17 +- .../admitted_migration_receipt.rs | 97 +++++++++++ .../empty_disposition_set_digest.rs | 18 ++ .../initial_gc_state_digest.rs | 18 ++ .../initial_retention_state_digest.rs | 18 ++ .../store_migration/migration_intent_bytes.rs | 54 ------ .../migration_intent_decoder.rs | 2 +- .../migration_receipt_decode_error.rs | 100 +++++++++++ .../migration_receipt_decode_error_display.rs | 63 +++++++ .../migration_receipt_decoder.rs | 157 ++++++++++++++++++ .../migration_receipt_initial_state.rs | 61 +++++++ .../store_migration/migration_record_bytes.rs | 69 ++++++++ .../migration_synchronization_mask.rs | 20 +++ src/lib.rs | 41 ++--- tests/store_migration_receipt.rs | 96 +++++++++++ tests/store_migration_receipt/binding_laws.rs | 104 ++++++++++++ tests/store_migration_receipt/fixture.rs | 38 +++++ tests/store_migration_receipt/harness.rs | 107 ++++++++++++ 22 files changed, 1023 insertions(+), 93 deletions(-) create mode 100644 src/adapters/store_migration/admitted_migration_receipt.rs create mode 100644 src/adapters/store_migration/empty_disposition_set_digest.rs create mode 100644 src/adapters/store_migration/initial_gc_state_digest.rs create mode 100644 src/adapters/store_migration/initial_retention_state_digest.rs delete mode 100644 src/adapters/store_migration/migration_intent_bytes.rs create mode 100644 src/adapters/store_migration/migration_receipt_decode_error.rs create mode 100644 src/adapters/store_migration/migration_receipt_decode_error_display.rs create mode 100644 src/adapters/store_migration/migration_receipt_decoder.rs create mode 100644 src/adapters/store_migration/migration_receipt_initial_state.rs create mode 100644 src/adapters/store_migration/migration_record_bytes.rs create mode 100644 src/adapters/store_migration/migration_synchronization_mask.rs create mode 100644 tests/store_migration_receipt.rs create mode 100644 tests/store_migration_receipt/binding_laws.rs create mode 100644 tests/store_migration_receipt/fixture.rs create mode 100644 tests/store_migration_receipt/harness.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index beb0ef4..a105add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- The version-2 store-format marker now has exact canonical encoding and - admission, while migration intents admit exact catalog, predecessor, root, - definition, store-identity, checksum, and digest coordinates. Retention - preflight combines expected-generation planning with deterministic closure - verification; authority-revalidated 17-phase orchestration returns an - unforgeable complete-coordinate receipt after durable cleanup. +- Version-2 marker, migration-intent, and completion-receipt admission now bind + exact catalog, predecessor, root, definition, store, empty-state, checksum, + digest, and synchronization-mask coordinates. Retention preflight combines + expected-generation planning with deterministic closure verification; + authority-revalidated 17-phase orchestration returns an unforgeable + complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 7406736..499bc51 100644 --- a/README.md +++ b/README.md @@ -116,14 +116,14 @@ recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Canonical version-2 store-format marker encoding and exact -migration-intent admission are implemented. Version-2 retention values; -canonical in-memory root, global manifest, and retention-head codecs; -storage-independent expected-state transition planning; deterministic bounded -closure verification against a pinned catalog; a combined transition preflight -proof; and the exact 17-phase publication vocabulary with a blocking storage -capability port are implemented. Private-field proofs retain every receipt -coordinate. Ordered storage-port orchestration revalidates current authority, -executes all 17 durability phases, and returns a consequential +migration-intent and completion-receipt admission are implemented. Version-2 +retention values; canonical in-memory root, global manifest, and retention-head +codecs; storage-independent expected-state transition planning; deterministic +bounded closure verification against a pinned catalog; a combined transition +preflight proof; and the exact 17-phase publication vocabulary with a blocking +storage capability port are implemented. Private-field proofs retain every +receipt coordinate. Ordered storage-port orchestration revalidates current +authority, executes all 17 durability phases, and returns a consequential complete-coordinate receipt. Filesystem migration, retention execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 4f7ad6b..f631c23 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,9 +61,9 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`AdmittedStoreMigrationIntent` admits the exact intent framing, checksum, catalog coordinates, predecessor law, definition, and derived store identity. -These record boundaries do not prove the named live inventory or physical root, -detect the store version, or execute filesystem migration. +`AdmittedStoreMigrationIntent` admits intent integrity and identity; `AdmittedStoreMigrationReceipt` binds that intent, the marker, registered empty states, and all synchronization bits. +These record boundaries do not prove the named live inventory, physical root, +store version, or execution of filesystem migration. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index afb0409..9d85a84 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact marker and intent admission in `tests/store_format_marker.rs` and `tests/store_migration_intent.rs`; receipt admission remains | In progress in #19 | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs` | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 19effb7..2021c2a 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -2,7 +2,9 @@ mod admitted_format_marker; mod admitted_migration_intent; +mod admitted_migration_receipt; mod canonical_format_marker; +mod empty_disposition_set_digest; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; @@ -10,23 +12,36 @@ mod format_marker_decoder; mod format_marker_digest; mod format_marker_encoder; mod immutable_pool_inventory_digest; -mod migration_intent_bytes; +mod initial_gc_state_digest; +mod initial_retention_state_digest; mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_receipt_decode_error; +mod migration_receipt_decode_error_display; +mod migration_receipt_decoder; +mod migration_receipt_initial_state; +mod migration_record_bytes; +mod migration_synchronization_mask; mod store_identifier; mod store_root_identity; pub use admitted_format_marker::AdmittedStoreFormatMarker; pub use admitted_migration_intent::AdmittedStoreMigrationIntent; +pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use empty_disposition_set_digest::EmptyDispositionSetDigest; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; +pub use initial_gc_state_digest::InitialGcStateDigest; +pub use initial_retention_state_digest::InitialRetentionStateDigest; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; +pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; pub use store_root_identity::{ StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, diff --git a/src/adapters/store_migration/admitted_migration_receipt.rs b/src/adapters/store_migration/admitted_migration_receipt.rs new file mode 100644 index 0000000..6abf88c --- /dev/null +++ b/src/adapters/store_migration/admitted_migration_receipt.rs @@ -0,0 +1,97 @@ +//! This boundary module owns admitted store-migration completion evidence. + +use super::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, EmptyDispositionSetDigest, + InitialGcStateDigest, InitialRetentionStateDigest, MigrationSynchronizationMask, + StoreFormatMarkerDigest, StoreIdentifier, StoreMigrationIntentDigest, + StoreMigrationReceiptDecodeError, migration_receipt_decoder, +}; + +/// Semantic fields admitted from one canonical migration receipt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct StoreMigrationReceiptFields { + pub(super) intent_digest: StoreMigrationIntentDigest, + pub(super) store_identifier: StoreIdentifier, + pub(super) format_marker_digest: StoreFormatMarkerDigest, + pub(super) initial_retention_state_digest: InitialRetentionStateDigest, + pub(super) initial_gc_state_digest: InitialGcStateDigest, + pub(super) empty_disposition_set_digest: EmptyDispositionSetDigest, + pub(super) synchronization_mask: MigrationSynchronizationMask, +} + +/// Borrowed canonical version-2 store-migration receipt. +/// +/// Admission proves record integrity, exact binding to caller-supplied admitted +/// intent and marker evidence, registered empty-state digests, and the complete +/// synchronization mask. It does not prove that filesystem transitions +/// actually occurred; production recovery must establish that separately. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdmittedStoreMigrationReceipt<'encoded> { + encoded: &'encoded [u8], + fields: StoreMigrationReceiptFields, +} + +impl<'encoded> AdmittedStoreMigrationReceipt<'encoded> { + /// Decodes and admits one exact receipt bound to `intent` and `marker`. + /// + /// # Errors + /// + /// Returns [`StoreMigrationReceiptDecodeError`] for invalid framing, + /// integrity, binding, registered state digest, or synchronization bits. + pub fn decode( + encoded: &'encoded [u8], + intent: &AdmittedStoreMigrationIntent<'_>, + marker: &AdmittedStoreFormatMarker<'_>, + ) -> Result { + migration_receipt_decoder::decode(encoded, intent, marker) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the exact bound migration-intent digest. + pub const fn intent_digest(&self) -> StoreMigrationIntentDigest { + self.fields.intent_digest + } + + /// Returns the exact bound logical store identity. + pub const fn store_identifier(&self) -> StoreIdentifier { + self.fields.store_identifier + } + + /// Returns the exact bound format-marker digest. + pub const fn format_marker_digest(&self) -> StoreFormatMarkerDigest { + self.fields.format_marker_digest + } + + /// Returns the registered empty retention-state digest. + pub const fn initial_retention_state_digest(&self) -> InitialRetentionStateDigest { + self.fields.initial_retention_state_digest + } + + /// Returns the registered empty garbage-collection-state digest. + pub const fn initial_gc_state_digest(&self) -> InitialGcStateDigest { + self.fields.initial_gc_state_digest + } + + /// Returns the registered empty recovery-disposition-set digest. + pub const fn empty_disposition_set_digest(&self) -> EmptyDispositionSetDigest { + self.fields.empty_disposition_set_digest + } + + /// Returns the complete admitted synchronization mask. + pub const fn synchronization_mask(&self) -> MigrationSynchronizationMask { + self.fields.synchronization_mask + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + fields: StoreMigrationReceiptFields, + ) -> Self { + Self { encoded, fields } + } +} diff --git a/src/adapters/store_migration/empty_disposition_set_digest.rs b/src/adapters/store_migration/empty_disposition_set_digest.rs new file mode 100644 index 0000000..ad59041 --- /dev/null +++ b/src/adapters/store_migration/empty_disposition_set_digest.rs @@ -0,0 +1,18 @@ +//! This module owns the registered empty recovery-disposition-set identity. + +/// Registered identity of an empty version-2 recovery-disposition set. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct EmptyDispositionSetDigest([u8; 32]); + +impl EmptyDispositionSetDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/initial_gc_state_digest.rs b/src/adapters/store_migration/initial_gc_state_digest.rs new file mode 100644 index 0000000..394c875 --- /dev/null +++ b/src/adapters/store_migration/initial_gc_state_digest.rs @@ -0,0 +1,18 @@ +//! This module owns the registered initial garbage-collection-state identity. + +/// Registered identity of empty version-2 garbage-collection state. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct InitialGcStateDigest([u8; 32]); + +impl InitialGcStateDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/initial_retention_state_digest.rs b/src/adapters/store_migration/initial_retention_state_digest.rs new file mode 100644 index 0000000..2dc9cd2 --- /dev/null +++ b/src/adapters/store_migration/initial_retention_state_digest.rs @@ -0,0 +1,18 @@ +//! This module owns the registered initial retention-state identity. + +/// Registered identity of empty version-2 retention state. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct InitialRetentionStateDigest([u8; 32]); + +impl InitialRetentionStateDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/migration_intent_bytes.rs b/src/adapters/store_migration/migration_intent_bytes.rs deleted file mode 100644 index 4f6906b..0000000 --- a/src/adapters/store_migration/migration_intent_bytes.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! This boundary module owns fixed-width migration-intent field access. - -use super::StoreMigrationIntentDecodeError; - -pub(super) const ENCODED_LENGTH: usize = 256; - -pub(super) const fn require_length(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { - if encoded.len() == ENCODED_LENGTH { - Ok(()) - } else { - Err(wrong_length(encoded)) - } -} - -pub(super) const fn wrong_length(encoded: &[u8]) -> StoreMigrationIntentDecodeError { - StoreMigrationIntentDecodeError::WrongLength { - expected: ENCODED_LENGTH, - observed: encoded.len(), - } -} - -pub(super) fn read_u16( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u16::from_be_bytes) -} - -pub(super) fn read_u32( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u32::from_be_bytes) -} - -pub(super) fn read_u64( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u64::from_be_bytes) -} - -pub(super) fn read_array( - encoded: &[u8], - offset: usize, -) -> Result<[u8; WIDTH], StoreMigrationIntentDecodeError> { - let Some(end) = offset.checked_add(WIDTH) else { - return Err(wrong_length(encoded)); - }; - let bytes = encoded - .get(offset..end) - .ok_or_else(|| wrong_length(encoded))?; - <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) -} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs index 0108c7a..7dabc5c 100644 --- a/src/adapters/store_migration/migration_intent_decoder.rs +++ b/src/adapters/store_migration/migration_intent_decoder.rs @@ -1,7 +1,7 @@ //! This boundary module owns store-migration intent decoding order. use super::admitted_migration_intent::StoreMigrationIntentFields; -use super::migration_intent_bytes::{ +use super::migration_record_bytes::{ read_array, read_u16, read_u32, read_u64, require_length, wrong_length, }; use super::{ diff --git a/src/adapters/store_migration/migration_receipt_decode_error.rs b/src/adapters/store_migration/migration_receipt_decode_error.rs new file mode 100644 index 0000000..609a38c --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_decode_error.rs @@ -0,0 +1,100 @@ +//! This boundary module owns store-migration receipt decoding failures. + +/// Failure to decode and admit one version-2 migration receipt. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationReceiptDecodeError { + /// The input was not exactly one complete receipt. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The record-length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The receipt carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// The receipt did not bind the supplied admitted intent. + IntentDigestMismatch { + /// Supplied intent digest. + expected: [u8; 32], + /// Receipt intent digest. + observed: [u8; 32], + }, + /// The receipt did not bind the intent's store identifier. + StoreIdentifierMismatch { + /// Supplied intent store identifier. + expected: [u8; 32], + /// Receipt store identifier. + observed: [u8; 32], + }, + /// The receipt did not bind the supplied admitted marker. + FormatMarkerDigestMismatch { + /// Supplied marker digest. + expected: [u8; 32], + /// Receipt marker digest. + observed: [u8; 32], + }, + /// The initial retention-state digest was not registered. + InitialRetentionStateDigestMismatch { + /// Registered empty-state digest. + expected: [u8; 32], + /// Receipt digest. + observed: [u8; 32], + }, + /// The initial garbage-collection-state digest was not registered. + InitialGcStateDigestMismatch { + /// Registered empty-state digest. + expected: [u8; 32], + /// Receipt digest. + observed: [u8; 32], + }, + /// The empty recovery-disposition-set digest was not registered. + EmptyDispositionSetDigestMismatch { + /// Registered empty-set digest. + expected: [u8; 32], + /// Receipt digest. + observed: [u8; 32], + }, + /// The synchronization mask carried unknown bits. + UnsupportedSynchronizationBits { + /// Complete supported bit set. + supported: u64, + /// Observed mask. + observed: u64, + }, + /// The synchronization mask omitted one or more mandatory bits. + IncompleteSynchronizationMask { + /// Complete required bit set. + required: u64, + /// Observed mask. + observed: u64, + }, +} diff --git a/src/adapters/store_migration/migration_receipt_decode_error_display.rs b/src/adapters/store_migration/migration_receipt_decode_error_display.rs new file mode 100644 index 0000000..16cc473 --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_decode_error_display.rs @@ -0,0 +1,63 @@ +//! This boundary module owns migration-receipt error formatting. + +use std::error::Error; +use std::fmt; + +use super::StoreMigrationReceiptDecodeError; + +impl fmt::Display for StoreMigrationReceiptDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "migration receipt requires {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { .. } => formatter.write_str("invalid migration-receipt magic"), + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported migration-receipt version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "migration-receipt record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported migration-receipt flags {observed:#010x}" + ) + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("migration-receipt checksum mismatch") + } + Self::IntentDigestMismatch { .. } => { + formatter.write_str("migration-receipt intent digest mismatch") + } + Self::StoreIdentifierMismatch { .. } => { + formatter.write_str("migration-receipt store identifier mismatch") + } + Self::FormatMarkerDigestMismatch { .. } => { + formatter.write_str("migration-receipt format-marker digest mismatch") + } + Self::InitialRetentionStateDigestMismatch { .. } => { + formatter.write_str("migration-receipt initial retention-state digest mismatch") + } + Self::InitialGcStateDigestMismatch { .. } => { + formatter.write_str("migration-receipt initial GC-state digest mismatch") + } + Self::EmptyDispositionSetDigestMismatch { .. } => { + formatter.write_str("migration-receipt empty disposition-set digest mismatch") + } + Self::UnsupportedSynchronizationBits { observed, .. } => write!( + formatter, + "migration-receipt synchronization mask has unknown bits: {observed:#018x}" + ), + Self::IncompleteSynchronizationMask { observed, .. } => write!( + formatter, + "migration-receipt synchronization mask is incomplete: {observed:#018x}" + ), + } + } +} + +impl Error for StoreMigrationReceiptDecodeError {} diff --git a/src/adapters/store_migration/migration_receipt_decoder.rs b/src/adapters/store_migration/migration_receipt_decoder.rs new file mode 100644 index 0000000..308308d --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_decoder.rs @@ -0,0 +1,157 @@ +//! This boundary module owns store-migration receipt decoding order. + +use super::admitted_migration_receipt::StoreMigrationReceiptFields; +use super::migration_receipt_initial_state::{ + read_empty_disposition_digest, read_initial_gc_digest, read_initial_retention_digest, +}; +use super::migration_record_bytes::{ + read_array, read_u16, read_u32, read_u64, require_length, wrong_length, +}; +use super::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + MigrationSynchronizationMask, StoreMigrationReceiptDecodeError, +}; + +const CHECKSUM_OFFSET: usize = 224; +const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; +const VERSION: u16 = 2; +const RECORD_LENGTH: u16 = 256; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; + +pub(super) fn decode<'encoded>( + encoded: &'encoded [u8], + intent: &AdmittedStoreMigrationIntent<'_>, + marker: &AdmittedStoreFormatMarker<'_>, +) -> Result, StoreMigrationReceiptDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let intent_digest = bind_intent_digest(encoded, intent)?; + let store_identifier = bind_store_identifier(encoded, intent)?; + let format_marker_digest = bind_marker_digest(encoded, marker)?; + let initial_retention_state_digest = read_initial_retention_digest(encoded)?; + let initial_gc_state_digest = read_initial_gc_digest(encoded)?; + let empty_disposition_set_digest = read_empty_disposition_digest(encoded)?; + let synchronization_mask = read_synchronization_mask(encoded)?; + Ok(AdmittedStoreMigrationReceipt::admitted( + encoded, + StoreMigrationReceiptFields { + intent_digest, + store_identifier, + format_marker_digest, + initial_retention_state_digest, + initial_gc_state_digest, + empty_disposition_set_digest, + synchronization_mask, + }, + )) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(StoreMigrationReceiptDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(StoreMigrationReceiptDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(StoreMigrationReceiptDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(StoreMigrationReceiptDecodeError::UnsupportedFlags { observed: flags }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or_else(|| wrong_length(encoded))?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = hash(CHECKSUM_DOMAIN, preimage); + if observed == expected { + Ok(()) + } else { + Err(StoreMigrationReceiptDecodeError::ChecksumMismatch { expected, observed }) + } +} + +fn bind_intent_digest( + encoded: &[u8], + intent: &AdmittedStoreMigrationIntent<'_>, +) -> Result { + let expected = *intent.digest().as_bytes(); + let observed = read_array(encoded, 24)?; + if observed == expected { + Ok(intent.digest()) + } else { + Err(StoreMigrationReceiptDecodeError::IntentDigestMismatch { expected, observed }) + } +} + +fn bind_store_identifier( + encoded: &[u8], + intent: &AdmittedStoreMigrationIntent<'_>, +) -> Result { + let expected = *intent.store_identifier().as_bytes(); + let observed = read_array(encoded, 56)?; + if observed == expected { + Ok(intent.store_identifier()) + } else { + Err(StoreMigrationReceiptDecodeError::StoreIdentifierMismatch { expected, observed }) + } +} + +fn bind_marker_digest( + encoded: &[u8], + marker: &AdmittedStoreFormatMarker<'_>, +) -> Result { + let expected = *marker.digest().as_bytes(); + let observed = read_array(encoded, 88)?; + if observed == expected { + Ok(marker.digest()) + } else { + Err(StoreMigrationReceiptDecodeError::FormatMarkerDigestMismatch { expected, observed }) + } +} + +fn read_synchronization_mask( + encoded: &[u8], +) -> Result { + let observed = read_u64(encoded, 216)?; + let supported = MigrationSynchronizationMask::COMPLETE_BITS; + if observed & !supported != 0 { + return Err( + StoreMigrationReceiptDecodeError::UnsupportedSynchronizationBits { + supported, + observed, + }, + ); + } + if observed != supported { + return Err( + StoreMigrationReceiptDecodeError::IncompleteSynchronizationMask { + required: supported, + observed, + }, + ); + } + Ok(MigrationSynchronizationMask::complete()) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/store_migration/migration_receipt_initial_state.rs b/src/adapters/store_migration/migration_receipt_initial_state.rs new file mode 100644 index 0000000..01a43d5 --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_initial_state.rs @@ -0,0 +1,61 @@ +//! This boundary module owns registered empty-state receipt admission. + +use super::migration_record_bytes::read_array; +use super::{ + EmptyDispositionSetDigest, InitialGcStateDigest, InitialRetentionStateDigest, + StoreMigrationReceiptDecodeError, +}; + +const INITIAL_RETENTION_DOMAIN: &[u8] = b"keep.initial-retention-state/v2\0"; +const INITIAL_GC_DOMAIN: &[u8] = b"keep.initial-gc-state/v2\0"; +const EMPTY_DISPOSITION_DOMAIN: &[u8] = b"keep.empty-disposition-set/v2\0"; + +pub(super) fn read_initial_retention_digest( + encoded: &[u8], +) -> Result { + let expected = digest(INITIAL_RETENTION_DOMAIN); + let observed = read_array(encoded, 120)?; + if observed == expected { + Ok(InitialRetentionStateDigest::from_hash(expected)) + } else { + Err( + StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { + expected, + observed, + }, + ) + } +} + +pub(super) fn read_initial_gc_digest( + encoded: &[u8], +) -> Result { + let expected = digest(INITIAL_GC_DOMAIN); + let observed = read_array(encoded, 152)?; + if observed == expected { + Ok(InitialGcStateDigest::from_hash(expected)) + } else { + Err(StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { expected, observed }) + } +} + +pub(super) fn read_empty_disposition_digest( + encoded: &[u8], +) -> Result { + let expected = digest(EMPTY_DISPOSITION_DOMAIN); + let observed = read_array(encoded, 184)?; + if observed == expected { + Ok(EmptyDispositionSetDigest::from_hash(expected)) + } else { + Err( + StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { + expected, + observed, + }, + ) + } +} + +fn digest(domain: &[u8]) -> [u8; 32] { + *blake3::hash(domain).as_bytes() +} diff --git a/src/adapters/store_migration/migration_record_bytes.rs b/src/adapters/store_migration/migration_record_bytes.rs new file mode 100644 index 0000000..774469d --- /dev/null +++ b/src/adapters/store_migration/migration_record_bytes.rs @@ -0,0 +1,69 @@ +//! This boundary module owns fixed-width store-migration record field access. + +use super::{StoreMigrationIntentDecodeError, StoreMigrationReceiptDecodeError}; + +const ENCODED_LENGTH: usize = 256; + +pub(super) trait MigrationRecordDecodeError: Sized { + fn wrong_length(expected: usize, observed: usize) -> Self; +} + +impl MigrationRecordDecodeError for StoreMigrationIntentDecodeError { + fn wrong_length(expected: usize, observed: usize) -> Self { + Self::WrongLength { expected, observed } + } +} + +impl MigrationRecordDecodeError for StoreMigrationReceiptDecodeError { + fn wrong_length(expected: usize, observed: usize) -> Self { + Self::WrongLength { expected, observed } + } +} + +pub(super) fn require_length( + encoded: &[u8], +) -> Result<(), Error> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(wrong_length(encoded)) + } +} + +pub(super) fn wrong_length(encoded: &[u8]) -> Error { + Error::wrong_length(ENCODED_LENGTH, encoded.len()) +} + +pub(super) fn read_u16( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], Error> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(wrong_length(encoded)); + }; + let bytes = encoded + .get(offset..end) + .ok_or_else(|| wrong_length(encoded))?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) +} diff --git a/src/adapters/store_migration/migration_synchronization_mask.rs b/src/adapters/store_migration/migration_synchronization_mask.rs new file mode 100644 index 0000000..47c46cd --- /dev/null +++ b/src/adapters/store_migration/migration_synchronization_mask.rs @@ -0,0 +1,20 @@ +//! This module owns completed migration synchronization evidence. + +/// Closed set of mandatory migration synchronization transitions. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct MigrationSynchronizationMask(u64); + +impl MigrationSynchronizationMask { + pub(super) const COMPLETE_BITS: u64 = 0x03ff; + + /// Returns the complete synchronization bit set. + #[must_use] + pub const fn bits(self) -> u64 { + self.0 + } + + pub(super) const fn complete() -> Self { + Self(Self::COMPLETE_BITS) + } +} diff --git a/src/lib.rs b/src/lib.rs index c09acfc..fb536f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,9 +33,11 @@ //! canonical encoding, registered-definition admission, checksum verification, //! and domain-separated identity. Migration-intent admission validates its //! framing, checksum, catalog and predecessor grammar, registered definition, -//! deterministic store identity, and typed recovery coordinates. Live -//! inventory and root revalidation, filesystem migration, retention execution, -//! recovery, and garbage collection remain intentionally absent. +//! deterministic store identity, and typed recovery coordinates. Completion +//! receipts bind an admitted intent and marker, registered empty-state digests, +//! and the complete synchronization mask. Live inventory and root revalidation, +//! filesystem migration, retention execution, recovery, and garbage collection +//! remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -54,26 +56,27 @@ mod retention; pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, BlobIdBinaryParseError, - BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, - CanonicalStoreFormatMarker, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, + CanonicalPublicationHead, CanonicalStoreFormatMarker, CatalogAdmissionError, + CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, + CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, + FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, - ImmutablePoolInventoryDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + ImmutablePoolInventoryDigest, InitialGcStateDigest, InitialRetentionStateDigest, + LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, + LayoutIdTextParseError, MigrationSynchronizationMask, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, @@ -107,8 +110,8 @@ pub use adapters::{ StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, - StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, + StoreMigrationIntentDigest, StoreMigrationReceiptDecodeError, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, diff --git a/tests/store_migration_receipt.rs b/tests/store_migration_receipt.rs new file mode 100644 index 0000000..bbacfd2 --- /dev/null +++ b/tests/store_migration_receipt.rs @@ -0,0 +1,96 @@ +//! Canonical version-2 store-migration receipt laws. + +#[path = "store_migration_receipt/binding_laws.rs"] +mod binding_laws; +#[path = "store_migration_receipt/fixture.rs"] +mod fixture; +#[path = "store_migration_receipt/harness.rs"] +mod harness; +mod support; + +use fixture::{ + DISPOSITION_DIGEST, INITIAL_GC_DIGEST, INITIAL_RETENTION_DIGEST, intent_bytes, marker_bytes, + receipt_bytes, +}; +use harness::{assert_fixed_refusal, assert_receipt_refusal, mutated_array}; +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + StoreMigrationReceiptDecodeError, +}; + +#[test] +fn receipt_admits_every_frozen_completion_coordinate() -> Result<(), Box> { + let intent_bytes = intent_bytes()?; + let marker_bytes = marker_bytes()?; + let receipt_bytes = receipt_bytes()?; + let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; + let receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker)?; + + assert_eq!(receipt.encoded(), receipt_bytes); + assert_eq!(receipt.intent_digest(), intent.digest()); + assert_eq!(receipt.store_identifier(), intent.store_identifier()); + assert_eq!(receipt.format_marker_digest(), marker.digest()); + assert_eq!( + receipt.initial_retention_state_digest().as_bytes(), + &INITIAL_RETENTION_DIGEST + ); + assert_eq!( + receipt.initial_gc_state_digest().as_bytes(), + &INITIAL_GC_DIGEST + ); + assert_eq!( + receipt.empty_disposition_set_digest().as_bytes(), + &DISPOSITION_DIGEST + ); + assert_eq!(receipt.synchronization_mask().bits(), 0x03ff); + Ok(()) +} + +#[test] +fn receipt_framing_has_exact_first_refusals() -> Result<(), Box> { + let bytes = receipt_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert_receipt_refusal( + &truncated, + StoreMigrationReceiptDecodeError::WrongLength { + expected: 256, + observed: 255, + }, + )?; + let mut extended = bytes.clone(); + extended.push(0); + assert_receipt_refusal( + &extended, + StoreMigrationReceiptDecodeError::WrongLength { + expected: 256, + observed: 257, + }, + )?; + assert_fixed_refusal( + 0, + StoreMigrationReceiptDecodeError::InvalidMagic { + observed: mutated_array(&bytes, 0, 0)?, + }, + )?; + assert_fixed_refusal( + 17, + StoreMigrationReceiptDecodeError::UnsupportedVersion { + expected: 2, + observed: 3, + }, + )?; + assert_fixed_refusal( + 19, + StoreMigrationReceiptDecodeError::InvalidRecordLength { + expected: 256, + observed: 257, + }, + )?; + assert_fixed_refusal( + 23, + StoreMigrationReceiptDecodeError::UnsupportedFlags { observed: 1 }, + )?; + Ok(()) +} diff --git a/tests/store_migration_receipt/binding_laws.rs b/tests/store_migration_receipt/binding_laws.rs new file mode 100644 index 0000000..c7a8cc0 --- /dev/null +++ b/tests/store_migration_receipt/binding_laws.rs @@ -0,0 +1,104 @@ +//! This module owns migration-receipt integrity and binding laws. + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + StoreMigrationReceiptDecodeError, +}; + +use super::fixture::{ + DISPOSITION_DIGEST, INITIAL_GC_DIGEST, INITIAL_RETENTION_DIGEST, intent_bytes, marker_bytes, + receipt_bytes, +}; +use super::harness::{ + assert_semantic_refusal, decode_receipt, digest_intent, flip_byte, mutated_array, + refresh_checksum, +}; + +#[test] +fn receipt_integrity_and_binding_have_exact_precedence() -> Result<(), Box> { + let mut checksum = receipt_bytes()?; + flip_byte(&mut checksum, 24)?; + assert!(matches!( + decode_receipt(&checksum)?, + Err(StoreMigrationReceiptDecodeError::ChecksumMismatch { .. }) + )); + + let intent = intent_bytes()?; + let marker = marker_bytes()?; + assert_semantic_refusal( + 24, + StoreMigrationReceiptDecodeError::IntentDigestMismatch { + expected: digest_intent(&intent), + observed: mutated_array(&receipt_bytes()?, 24, 0)?, + }, + )?; + assert_semantic_refusal( + 56, + StoreMigrationReceiptDecodeError::StoreIdentifierMismatch { + expected: *AdmittedStoreMigrationIntent::decode(&intent)? + .store_identifier() + .as_bytes(), + observed: mutated_array(&receipt_bytes()?, 56, 0)?, + }, + )?; + assert_semantic_refusal( + 88, + StoreMigrationReceiptDecodeError::FormatMarkerDigestMismatch { + expected: *AdmittedStoreFormatMarker::decode(&marker)? + .digest() + .as_bytes(), + observed: mutated_array(&receipt_bytes()?, 88, 0)?, + }, + )?; + assert_semantic_refusal( + 120, + StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { + expected: INITIAL_RETENTION_DIGEST, + observed: mutated_array(&receipt_bytes()?, 120, 0)?, + }, + )?; + assert_semantic_refusal( + 152, + StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { + expected: INITIAL_GC_DIGEST, + observed: mutated_array(&receipt_bytes()?, 152, 0)?, + }, + )?; + assert_semantic_refusal( + 184, + StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { + expected: DISPOSITION_DIGEST, + observed: mutated_array(&receipt_bytes()?, 184, 0)?, + }, + )?; + assert_semantic_refusal( + 221, + StoreMigrationReceiptDecodeError::UnsupportedSynchronizationBits { + supported: 0x03ff, + observed: 0x0001_03ff, + }, + )?; + assert_semantic_refusal( + 222, + StoreMigrationReceiptDecodeError::IncompleteSynchronizationMask { + required: 0x03ff, + observed: 0x02ff, + }, + )?; + + let mut alternative_intent = intent; + flip_byte(&mut alternative_intent, 159)?; + refresh_checksum( + &mut alternative_intent, + 224, + b"keep.store-migration-intent-checksum/v2\0", + )?; + let alternative = AdmittedStoreMigrationIntent::decode(&alternative_intent)?; + let receipt = receipt_bytes()?; + let marker = AdmittedStoreFormatMarker::decode(&marker)?; + assert!(matches!( + AdmittedStoreMigrationReceipt::decode(&receipt, &alternative, &marker), + Err(StoreMigrationReceiptDecodeError::IntentDigestMismatch { .. }) + )); + Ok(()) +} diff --git a/tests/store_migration_receipt/fixture.rs b/tests/store_migration_receipt/fixture.rs new file mode 100644 index 0000000..10a6ff2 --- /dev/null +++ b/tests/store_migration_receipt/fixture.rs @@ -0,0 +1,38 @@ +//! This module owns frozen migration-receipt fixture values. +#![allow( + clippy::redundant_pub_crate, + reason = "the parent integration-test module consumes this private fixture" +)] + +use std::io; + +use super::support; + +const RECEIPT: &str = include_str!("../../conformance/segment-store/v2/migration-receipt.hex"); +const INTENT: &str = include_str!("../../conformance/segment-store/v2/migration-intent.hex"); +const MARKER: &str = include_str!("../../conformance/segment-store/v2/format-marker.hex"); + +pub(super) const INITIAL_RETENTION_DIGEST: [u8; 32] = [ + 0xd5, 0x2f, 0x1f, 0x02, 0x2e, 0xdb, 0x1d, 0xe7, 0xb8, 0x40, 0xc5, 0xbf, 0x8f, 0xb5, 0x5d, 0xe7, + 0x93, 0x2c, 0xa6, 0x93, 0x70, 0xae, 0x85, 0xe2, 0xbe, 0xe4, 0x17, 0x91, 0x43, 0x79, 0x2b, 0xc3, +]; +pub(super) const INITIAL_GC_DIGEST: [u8; 32] = [ + 0xba, 0x0e, 0xa2, 0x00, 0xa5, 0xb0, 0x67, 0x41, 0x56, 0x4c, 0x43, 0xa7, 0x9a, 0x91, 0x94, 0x5b, + 0xef, 0x0b, 0x0f, 0xac, 0x51, 0xc9, 0x60, 0xea, 0x4f, 0x82, 0x07, 0x09, 0x4f, 0x3e, 0x1e, 0x31, +]; +pub(super) const DISPOSITION_DIGEST: [u8; 32] = [ + 0xa8, 0x02, 0x59, 0xfc, 0xd1, 0x23, 0x72, 0x03, 0xea, 0x6c, 0x6c, 0xc5, 0x06, 0x55, 0x14, 0xab, + 0xde, 0xb0, 0x1d, 0xa6, 0x03, 0xc3, 0x19, 0x4b, 0x09, 0x6a, 0x04, 0x5c, 0xf6, 0x94, 0xc9, 0x5a, +]; + +pub(super) fn receipt_bytes() -> Result, io::Error> { + support::decode_hex(RECEIPT.trim_end()) +} + +pub(super) fn intent_bytes() -> Result, io::Error> { + support::decode_hex(INTENT.trim_end()) +} + +pub(super) fn marker_bytes() -> Result, io::Error> { + support::decode_hex(MARKER.trim_end()) +} diff --git a/tests/store_migration_receipt/harness.rs b/tests/store_migration_receipt/harness.rs new file mode 100644 index 0000000..296b863 --- /dev/null +++ b/tests/store_migration_receipt/harness.rs @@ -0,0 +1,107 @@ +//! This module owns migration-receipt mutation and admission test mechanics. +#![allow( + clippy::redundant_pub_crate, + reason = "sibling private test modules consume this harness" +)] + +use std::io; + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + StoreMigrationReceiptDecodeError, +}; + +use super::fixture::{intent_bytes, marker_bytes, receipt_bytes}; + +pub(super) fn assert_fixed_refusal( + offset: usize, + expected: StoreMigrationReceiptDecodeError, +) -> Result<(), Box> { + let mut bytes = receipt_bytes()?; + flip_byte(&mut bytes, offset)?; + assert_receipt_refusal(&bytes, expected) +} + +pub(super) fn assert_semantic_refusal( + offset: usize, + expected: StoreMigrationReceiptDecodeError, +) -> Result<(), Box> { + let mut bytes = receipt_bytes()?; + flip_byte(&mut bytes, offset)?; + refresh_checksum( + &mut bytes, + 224, + b"keep.store-migration-receipt-checksum/v2\0", + )?; + assert_receipt_refusal(&bytes, expected) +} + +pub(super) fn assert_receipt_refusal( + bytes: &[u8], + expected: StoreMigrationReceiptDecodeError, +) -> Result<(), Box> { + assert_eq!(decode_receipt(bytes)?, Err(expected)); + Ok(()) +} + +pub(super) fn decode_receipt( + bytes: &[u8], +) -> Result< + Result, StoreMigrationReceiptDecodeError>, + Box, +> { + let intent_bytes = intent_bytes()?; + let marker_bytes = marker_bytes()?; + let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; + Ok(AdmittedStoreMigrationReceipt::decode( + bytes, &intent, &marker, + )) +} + +pub(super) fn mutated_array( + bytes: &[u8], + offset: usize, + relative: usize, +) -> Result<[u8; WIDTH], io::Error> { + let end = offset + .checked_add(WIDTH) + .ok_or_else(|| io::Error::other("receipt field offset overflow"))?; + let field = bytes + .get(offset..end) + .ok_or_else(|| io::Error::other("receipt lacks fixed field"))?; + let mut observed = <[u8; WIDTH]>::try_from(field) + .map_err(|_| io::Error::other("receipt field width mismatch"))?; + flip_byte(&mut observed, relative)?; + Ok(observed) +} + +pub(super) fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { + let byte = bytes + .get_mut(offset) + .ok_or_else(|| io::Error::other("receipt mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(()) +} + +pub(super) fn refresh_checksum( + bytes: &mut [u8], + offset: usize, + domain: &[u8], +) -> Result<(), io::Error> { + let (preimage, checksum) = bytes + .split_at_mut_checked(offset) + .ok_or_else(|| io::Error::other("record lacks checksum boundary"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} + +pub(super) fn digest_intent(bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.store-migration-intent/v2\0"); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} From 5e541b0e84789afb7c1f0da972401182de1efc7f Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:39:40 -0700 Subject: [PATCH 33/50] Test: Fuzz store migration records --- CHANGELOG.md | 10 +- docs/formats/segment-store-v2/requirements.md | 2 +- fuzz/Cargo.toml | 7 ++ fuzz/README.md | 5 + fuzz/fuzz_targets/migration_format.rs | 52 ++++++++++ xtask/src/fuzz_campaign/target/tests.rs | 1 + xtask/src/fuzz_seed_corpus.rs | 3 + xtask/src/fuzz_seed_corpus/migration_seeds.rs | 94 +++++++++++++++++++ xtask/src/fuzz_seed_corpus/retention_seeds.rs | 25 +---- .../segment_store_v2_fixture.rs | 27 ++++++ .../fuzz_seed_corpus/tests/materialization.rs | 19 ++-- .../parser_fuzz_laws.rs | 23 +++++ 12 files changed, 233 insertions(+), 35 deletions(-) create mode 100644 fuzz/fuzz_targets/migration_format.rs create mode 100644 xtask/src/fuzz_seed_corpus/migration_seeds.rs create mode 100644 xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a105add..6b3bb30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, migration-intent, and completion-receipt admission now bind - exact catalog, predecessor, root, definition, store, empty-state, checksum, - digest, and synchronization-mask coordinates. Retention preflight combines +- Version-2 marker, migration-intent, and completion-receipt admission now binds + exact catalog, predecessor, root, definition, store, empty-state, and checksum, + digest, and synchronization-mask coordinates; a seeded migration fuzz + surface drives all three exact decoders. Retention preflight combines expected-generation planning with deterministic closure verification; - authority-revalidated 17-phase orchestration returns an unforgeable - complete-coordinate receipt after durable cleanup. + authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 9d85a84..bff404f 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs` | Implemented | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index b041049..88feb48 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -117,6 +117,13 @@ test = false doc = false bench = false +[[bin]] +name = "migration_format" +path = "fuzz_targets/migration_format.rs" +test = false +doc = false +bench = false + [[bin]] name = "repository_json" path = "fuzz_targets/repository_json.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 8b6af19..115dc98 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -70,6 +70,11 @@ retention-manifest, and retention-head decoders. The canonical one-root generation keeps mutations inside framing, semantic, ordering, checksum, and digest validation; every admitted value must retain its exact input bytes. +The `migration_format` seeds select the public format-marker, migration-intent, +and completion-receipt decoders. The receipt seed carries its exact marker and +intent dependencies so mutations exercise integrity and cross-record binding; +every admitted value must retain its exact input bytes. + The `segment_format` seeds select the public segment-header, record-header, complete-record, seal, and complete-segment boundaries. Canonical empty, one-record, and bundled segments keep mutations inside the nested parsers; diff --git a/fuzz/fuzz_targets/migration_format.rs b/fuzz/fuzz_targets/migration_format.rs new file mode 100644 index 0000000..739c906 --- /dev/null +++ b/fuzz/fuzz_targets/migration_format.rs @@ -0,0 +1,52 @@ +#![no_main] + +//! This target owns canonical store-migration record parser fuzzing. + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, +}; +use libfuzzer_sys::fuzz_target; + +const MARKER_BYTES: usize = 96; +const INTENT_BYTES: usize = 256; + +fuzz_target!(|bytes: &[u8]| { + let Some((&selector, input)) = bytes.split_first() else { + return; + }; + match selector { + 0 => marker(input), + 1 => intent(input), + _ => receipt(input), + } +}); + +fn marker(input: &[u8]) { + if let Ok(marker) = AdmittedStoreFormatMarker::decode(input) { + assert_eq!(marker.encoded(), input); + } +} + +fn intent(input: &[u8]) { + if let Ok(intent) = AdmittedStoreMigrationIntent::decode(input) { + assert_eq!(intent.encoded(), input); + } +} + +fn receipt(input: &[u8]) { + let Some((marker_bytes, remainder)) = input.split_at_checked(MARKER_BYTES) else { + return; + }; + let Some((intent_bytes, receipt_bytes)) = remainder.split_at_checked(INTENT_BYTES) else { + return; + }; + let (Ok(marker), Ok(intent)) = ( + AdmittedStoreFormatMarker::decode(marker_bytes), + AdmittedStoreMigrationIntent::decode(intent_bytes), + ) else { + return; + }; + if let Ok(receipt) = AdmittedStoreMigrationReceipt::decode(receipt_bytes, &intent, &marker) { + assert_eq!(receipt.encoded(), receipt_bytes); + } +} diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index 7b72f99..b43744d 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -30,6 +30,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "fast_cdc", "golden_protocol", "layout_record", + "migration_format", "repository_json", "retention_format", "segment_format", diff --git a/xtask/src/fuzz_seed_corpus.rs b/xtask/src/fuzz_seed_corpus.rs index 1455e04..3635322 100644 --- a/xtask/src/fuzz_seed_corpus.rs +++ b/xtask/src/fuzz_seed_corpus.rs @@ -5,8 +5,10 @@ mod cdc_seeds; mod filesystem; mod identity_seeds; mod layout_seeds; +mod migration_seeds; mod retention_seeds; mod segment_seeds; +mod segment_store_v2_fixture; use std::error::Error; use std::fmt; @@ -70,6 +72,7 @@ pub(super) fn prepare(repository_root: &Path) -> Result<(), FuzzSeedError> { seeds.extend(cdc_seeds::seeds()?); seeds.extend(golden_protocol_seeds_from(&files)?); seeds.extend(layout_seeds::seeds(&files)?); + seeds.extend(migration_seeds::seeds(&files)?); seeds.extend(retention_seeds::seeds(&files)?); seeds.extend(segment_seeds::seeds(&files)?); files.write_seeds(&seeds) diff --git a/xtask/src/fuzz_seed_corpus/migration_seeds.rs b/xtask/src/fuzz_seed_corpus/migration_seeds.rs new file mode 100644 index 0000000..30d01b8 --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/migration_seeds.rs @@ -0,0 +1,94 @@ +//! This module owns canonical store-migration record fuzz seeds. + +use super::filesystem::RepositoryFiles; +use super::segment_store_v2_fixture; +use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; + +const FORMAT_MARKER_FIXTURE: &str = "format-marker.hex"; +const MIGRATION_INTENT_FIXTURE: &str = "migration-intent.hex"; +const MIGRATION_RECEIPT_FIXTURE: &str = "migration-receipt.hex"; + +pub(super) const FIXTURES: [(u8, &str); 3] = [ + (0, FORMAT_MARKER_FIXTURE), + (1, MIGRATION_INTENT_FIXTURE), + (2, MIGRATION_RECEIPT_FIXTURE), +]; + +pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { + let [ + (marker_selector, marker_fixture), + (intent_selector, intent_fixture), + (receipt_selector, receipt_fixture), + ] = FIXTURES; + let marker = segment_store_v2_fixture::read_hex(files, marker_fixture)?; + let intent = segment_store_v2_fixture::read_hex(files, intent_fixture)?; + let receipt = segment_store_v2_fixture::read_hex(files, receipt_fixture)?; + Ok(vec![ + Seed::new( + "migration_format", + "format-marker", + prefixed(marker_selector, &marker)?, + )?, + Seed::new( + "migration_format", + "migration-intent", + prefixed(intent_selector, &intent)?, + )?, + Seed::new( + "migration_format", + "migration-receipt", + receipt_seed(receipt_selector, &marker, &intent, &receipt)?, + )?, + ]) +} + +fn receipt_seed( + selector: u8, + marker: &[u8], + intent: &[u8], + receipt: &[u8], +) -> Result, FuzzSeedError> { + let payload_bytes = marker + .len() + .checked_add(intent.len()) + .and_then(|length| length.checked_add(receipt.len())) + .ok_or_else(|| FuzzSeedError::violation("migration receipt seed length overflow"))?; + let framed_bytes = payload_bytes + .checked_add(1) + .ok_or_else(|| FuzzSeedError::violation("migration receipt seed length overflow"))?; + if framed_bytes > MAX_SEED_BYTES { + return Err(FuzzSeedError::violation( + "migration receipt seed exceeds the input bound", + )); + } + let mut payload = Vec::with_capacity(payload_bytes); + payload.extend_from_slice(marker); + payload.extend_from_slice(intent); + payload.extend_from_slice(receipt); + prefixed(selector, &payload) +} + +#[cfg(test)] +mod tests { + use super::{FuzzSeedError, MAX_SEED_BYTES, receipt_seed}; + + #[test] + fn receipt_seed_frames_dependencies_before_the_receipt() -> Result<(), FuzzSeedError> { + let seed = receipt_seed(2, b"marker", b"intent", b"receipt")?; + assert_eq!(seed, b"\x02markerintentreceipt"); + Ok(()) + } + + #[test] + fn receipt_seed_refuses_before_allocating_above_the_seed_bound() -> Result<(), FuzzSeedError> { + let oversized_marker = vec![0; MAX_SEED_BYTES]; + let Err(FuzzSeedError::Violation(message)) = receipt_seed(2, &oversized_marker, &[], &[]) + else { + return Err(FuzzSeedError::violation( + "oversized migration receipt seed was admitted", + )); + }; + assert_eq!(message, "migration receipt seed exceeds the input bound"); + Ok(()) + } +} diff --git a/xtask/src/fuzz_seed_corpus/retention_seeds.rs b/xtask/src/fuzz_seed_corpus/retention_seeds.rs index 152dc16..4f16d57 100644 --- a/xtask/src/fuzz_seed_corpus/retention_seeds.rs +++ b/xtask/src/fuzz_seed_corpus/retention_seeds.rs @@ -1,12 +1,8 @@ //! This module owns canonical retention-record fuzz seeds. -use std::path::Path; - use super::filesystem::RepositoryFiles; -use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; -use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; - -const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v2"; +use super::segment_store_v2_fixture; +use super::{FuzzSeedError, Seed, prefixed}; pub(super) const FIXTURES: [(u8, &str); 3] = [ (0, "one-anchor-root.hex"), @@ -20,7 +16,7 @@ pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> let name = fixture .strip_suffix(".hex") .ok_or_else(|| FuzzSeedError::violation("retention fixture lacks .hex suffix"))?; - let encoded = fixture_bytes(files, fixture)?; + let encoded = segment_store_v2_fixture::read_hex(files, fixture)?; seeds.push(Seed::new( "retention_format", name, @@ -29,18 +25,3 @@ pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> } Ok(seeds) } - -fn fixture_bytes(files: &RepositoryFiles, fixture: &'static str) -> Result, FuzzSeedError> { - let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); - let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; - let lines = framed_lines(&transport, MAX_SEED_BYTES) - .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; - let [encoded] = lines.as_slice() else { - return Err(FuzzSeedError::violation(format!( - "{fixture} must contain exactly one hexadecimal line" - ))); - }; - decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { - FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) - }) -} diff --git a/xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs b/xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs new file mode 100644 index 0000000..8788ac1 --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs @@ -0,0 +1,27 @@ +//! This module owns bounded admission of version-2 hexadecimal seed fixtures. + +use std::path::Path; + +use super::filesystem::RepositoryFiles; +use super::{FuzzSeedError, MAX_SEED_BYTES}; +use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; + +const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v2"; + +pub(super) fn read_hex( + files: &RepositoryFiles, + fixture: &'static str, +) -> Result, FuzzSeedError> { + let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); + let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; + let lines = framed_lines(&transport, MAX_SEED_BYTES) + .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; + let [encoded] = lines.as_slice() else { + return Err(FuzzSeedError::violation(format!( + "{fixture} must contain exactly one hexadecimal line" + ))); + }; + decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { + FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) + }) +} diff --git a/xtask/src/fuzz_seed_corpus/tests/materialization.rs b/xtask/src/fuzz_seed_corpus/tests/materialization.rs index 5ed813f..37dd51d 100644 --- a/xtask/src/fuzz_seed_corpus/tests/materialization.rs +++ b/xtask/src/fuzz_seed_corpus/tests/materialization.rs @@ -4,7 +4,8 @@ use std::collections::BTreeMap; use std::path::Path; use super::super::{ - FuzzSeedError, catalog_seeds, layout_seeds, prepare, retention_seeds, segment_seeds, + FuzzSeedError, catalog_seeds, layout_seeds, migration_seeds, prepare, retention_seeds, + segment_seeds, }; use crate::test_directory::TestDirectory; @@ -41,15 +42,16 @@ fn seed_preparation_materializes_the_complete_deterministic_set() copy_layout_fixtures(source_root, root)?; copy_segment_fixtures(source_root, root)?; copy_catalog_fixtures(source_root, root)?; - copy_retention_fixtures(source_root, root)?; + copy_version_two_fixtures(source_root, root)?; prepare(root)?; let corpus = root.join("fuzz/corpus"); let first = seed_contents(&corpus)?; - assert_eq!(first.len(), 43); + assert_eq!(first.len(), 46); assert_eq!(target_seed_count(&first, "catalog_format/"), 6); assert_eq!(target_seed_count(&first, "golden_protocol/"), 9); assert_eq!(target_seed_count(&first, "layout_record/"), 4); + assert_eq!(target_seed_count(&first, "migration_format/"), 3); assert_eq!(target_seed_count(&first, "retention_format/"), 3); assert_eq!(target_seed_count(&first, "segment_format/"), 8); prepare(root)?; @@ -116,24 +118,27 @@ fn copy_catalog_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeed Ok(()) } -fn copy_retention_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { +fn copy_version_two_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { use std::fs; let retention_directory = root.join("conformance/segment-store/v2"); fs::create_dir_all(&retention_directory).map_err(|source| { FuzzSeedError::io( - "create test retention conformance root", + "create test version-two conformance root", &retention_directory, source, ) })?; - for (_selector, fixture) in retention_seeds::FIXTURES { + let fixtures = retention_seeds::FIXTURES + .into_iter() + .chain(migration_seeds::FIXTURES); + for (_selector, fixture) in fixtures { let source_path = source_root .join("conformance/segment-store/v2") .join(fixture); let destination = retention_directory.join(fixture); fs::copy(&source_path, &destination) - .map_err(|source| FuzzSeedError::io("copy test retention", &destination, source))?; + .map_err(|source| FuzzSeedError::io("copy test version-two", &destination, source))?; } Ok(()) } diff --git a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs index bc2919f..763d4db 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs @@ -29,3 +29,26 @@ fn retention_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box Result<(), Box> { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest must have a repository parent")?; + + assert!( + repository_root + .join("fuzz/fuzz_targets/migration_format.rs") + .is_file() + ); + assert!( + repository_root + .join("xtask/src/fuzz_seed_corpus/migration_seeds.rs") + .is_file() + ); + assert!(FUZZ_MANIFEST.contains("name = \"migration_format\"")); + assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/migration_format.rs\"")); + assert!(FUZZ_GUIDE.contains("The `migration_format` seeds")); + assert!(REQUIREMENTS.contains("`migration_format`")); + Ok(()) +} From 2c93701da16a973b8f74eb98f74cc7997de00888 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:52:32 -0700 Subject: [PATCH 34/50] Add: Define store migration phases --- CHANGELOG.md | 4 +- .../segment-store-v2/migration-crash.md | 3 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 2 + .../store_migration/migration_phase.rs | 114 ++++++++++++++++++ src/lib.rs | 15 +-- tests/store_migration_phase.rs | 101 ++++++++++++++++ 7 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 src/adapters/store_migration/migration_phase.rs create mode 100644 tests/store_migration_phase.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3bb30..a6da96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ after its public API and format compatibility policies are established. - Version-2 marker, migration-intent, and completion-receipt admission now binds exact catalog, predecessor, root, definition, store, empty-state, and checksum, - digest, and synchronization-mask coordinates; a seeded migration fuzz - surface drives all three exact decoders. Retention preflight combines + digest, and synchronization-mask coordinates; migration fuzzing drives all + three decoders, and `StoreMigrationPhase` freezes 21 transitions. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md index 16127b0..b030fd8 100644 --- a/docs/formats/segment-store-v2/migration-crash.md +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -102,3 +102,6 @@ Every identifier requires before, during, and after process-death evidence. prefix length. Restart must classify exact stages, canonical targets, namespace prefix, marker, receipt, and cleanup state without depending on a clock, filesystem iteration order, or file existence alone. + +`StoreMigrationPhase::ALL` freezes the 21 boundaries above in exact order. +Storage execution and process-death evidence remain unimplemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index bff404f..e064fba 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -34,7 +34,7 @@ case is not evidence. | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | `KEEP-CRASH-053..=073` crash-injection matrix | Planned in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary in `tests/store_migration_phase.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 2021c2a..1ec3fbc 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -18,6 +18,7 @@ mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_phase; mod migration_receipt_decode_error; mod migration_receipt_decode_error_display; mod migration_receipt_decoder; @@ -40,6 +41,7 @@ pub use initial_gc_state_digest::InitialGcStateDigest; pub use initial_retention_state_digest::InitialRetentionStateDigest; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; diff --git a/src/adapters/store_migration/migration_phase.rs b/src/adapters/store_migration/migration_phase.rs new file mode 100644 index 0000000..cc14888 --- /dev/null +++ b/src/adapters/store_migration/migration_phase.rs @@ -0,0 +1,114 @@ +//! This boundary module owns exact store-migration durability phases. + +use std::fmt; + +/// Storage transition attempted by version-2 store migration. +/// +/// [`Self::ALL`] corresponds in order to `KEEP-CRASH-053` through +/// `KEEP-CRASH-073`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationPhase { + /// Write the complete canonical `migration.intent.next`. + WriteIntentStage, + /// Synchronize `migration.intent.next`. + SynchronizeIntentStage, + /// Link the synchronized intent stage to `migration.intent`. + LinkIntent, + /// Synchronize the store root after the intent link. + SynchronizeRootAfterIntent, + /// Remove the retained `migration.intent.next`. + RemoveIntentStage, + /// Synchronize the store root after intent-stage cleanup. + SynchronizeRootAfterIntentCleanup, + /// Create or exactly admit the persistent reader fence. + AdmitReaderFence, + /// Create or exactly admit the canonical version-2 directory prefix. + AdmitNamespacePrefix, + /// Synchronize the store root after namespace admission. + SynchronizeRootAfterNamespace, + /// Write the complete canonical `FORMAT.next`. + WriteMarkerStage, + /// Synchronize `FORMAT.next`. + SynchronizeMarkerStage, + /// Link the synchronized marker stage to `FORMAT`. + LinkMarker, + /// Synchronize the store root after the marker link. + SynchronizeRootAfterMarker, + /// Remove the retained `FORMAT.next`. + RemoveMarkerStage, + /// Synchronize the store root after marker-stage cleanup. + SynchronizeRootAfterMarkerCleanup, + /// Write the complete canonical `migration.receipt.next`. + WriteReceiptStage, + /// Synchronize `migration.receipt.next`. + SynchronizeReceiptStage, + /// Link the synchronized receipt stage to `migration.receipt`. + LinkReceipt, + /// Synchronize the store root after the receipt link. + SynchronizeRootAfterReceipt, + /// Remove the retained `migration.receipt.next`. + RemoveReceiptStage, + /// Synchronize the store root after receipt-stage cleanup. + SynchronizeRootAfterReceiptCleanup, +} + +impl StoreMigrationPhase { + /// Every migration phase in normative crash-boundary order. + pub const ALL: [Self; 21] = [ + Self::WriteIntentStage, + Self::SynchronizeIntentStage, + Self::LinkIntent, + Self::SynchronizeRootAfterIntent, + Self::RemoveIntentStage, + Self::SynchronizeRootAfterIntentCleanup, + Self::AdmitReaderFence, + Self::AdmitNamespacePrefix, + Self::SynchronizeRootAfterNamespace, + Self::WriteMarkerStage, + Self::SynchronizeMarkerStage, + Self::LinkMarker, + Self::SynchronizeRootAfterMarker, + Self::RemoveMarkerStage, + Self::SynchronizeRootAfterMarkerCleanup, + Self::WriteReceiptStage, + Self::SynchronizeReceiptStage, + Self::LinkReceipt, + Self::SynchronizeRootAfterReceipt, + Self::RemoveReceiptStage, + Self::SynchronizeRootAfterReceiptCleanup, + ]; +} + +impl fmt::Display for StoreMigrationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::WriteIntentStage => "migration-intent stage write", + Self::SynchronizeIntentStage => "migration-intent stage synchronization", + Self::LinkIntent => "migration-intent canonical link", + Self::SynchronizeRootAfterIntent => "store-root synchronization after intent link", + Self::RemoveIntentStage => "migration-intent stage removal", + Self::SynchronizeRootAfterIntentCleanup => { + "store-root synchronization after intent cleanup" + } + Self::AdmitReaderFence => "persistent reader-fence admission", + Self::AdmitNamespacePrefix => "canonical namespace-prefix admission", + Self::SynchronizeRootAfterNamespace => { + "store-root synchronization after namespace admission" + } + Self::WriteMarkerStage => "format-marker stage write", + Self::SynchronizeMarkerStage => "format-marker stage synchronization", + Self::LinkMarker => "format-marker canonical link", + Self::SynchronizeRootAfterMarker => "store-root synchronization after marker link", + Self::RemoveMarkerStage => "format-marker stage removal", + Self::SynchronizeRootAfterMarkerCleanup => { + "store-root synchronization after marker cleanup" + } + Self::WriteReceiptStage => "migration-receipt stage write", + Self::SynchronizeReceiptStage => "migration-receipt stage synchronization", + Self::LinkReceipt => "migration-receipt canonical link", + Self::SynchronizeRootAfterReceipt => "store-root synchronization after receipt link", + Self::RemoveReceiptStage => "migration-receipt stage removal", + Self::SynchronizeRootAfterReceiptCleanup => "final store-root synchronization", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index fb536f3..c4d86a8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,13 +110,14 @@ pub use adapters::{ StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreMigrationReceiptDecodeError, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + StoreMigrationIntentDigest, StoreMigrationPhase, StoreMigrationReceiptDecodeError, + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, + WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, + classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, + classify_recovery_segment_stage, execute_recovery_next_head_finalization, + execute_recovery_segment_resume, execute_recovery_stage_completion, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; diff --git a/tests/store_migration_phase.rs b/tests/store_migration_phase.rs new file mode 100644 index 0000000..978cad1 --- /dev/null +++ b/tests/store_migration_phase.rs @@ -0,0 +1,101 @@ +//! Ordered version-2 store-migration durability phase laws. + +use keep::StoreMigrationPhase; + +const EXPECTED: [(StoreMigrationPhase, &str); 21] = [ + ( + StoreMigrationPhase::WriteIntentStage, + "migration-intent stage write", + ), + ( + StoreMigrationPhase::SynchronizeIntentStage, + "migration-intent stage synchronization", + ), + ( + StoreMigrationPhase::LinkIntent, + "migration-intent canonical link", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterIntent, + "store-root synchronization after intent link", + ), + ( + StoreMigrationPhase::RemoveIntentStage, + "migration-intent stage removal", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, + "store-root synchronization after intent cleanup", + ), + ( + StoreMigrationPhase::AdmitReaderFence, + "persistent reader-fence admission", + ), + ( + StoreMigrationPhase::AdmitNamespacePrefix, + "canonical namespace-prefix admission", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterNamespace, + "store-root synchronization after namespace admission", + ), + ( + StoreMigrationPhase::WriteMarkerStage, + "format-marker stage write", + ), + ( + StoreMigrationPhase::SynchronizeMarkerStage, + "format-marker stage synchronization", + ), + ( + StoreMigrationPhase::LinkMarker, + "format-marker canonical link", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterMarker, + "store-root synchronization after marker link", + ), + ( + StoreMigrationPhase::RemoveMarkerStage, + "format-marker stage removal", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, + "store-root synchronization after marker cleanup", + ), + ( + StoreMigrationPhase::WriteReceiptStage, + "migration-receipt stage write", + ), + ( + StoreMigrationPhase::SynchronizeReceiptStage, + "migration-receipt stage synchronization", + ), + ( + StoreMigrationPhase::LinkReceipt, + "migration-receipt canonical link", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterReceipt, + "store-root synchronization after receipt link", + ), + ( + StoreMigrationPhase::RemoveReceiptStage, + "migration-receipt stage removal", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup, + "final store-root synchronization", + ), +]; + +#[test] +fn migration_phases_are_complete_ordered_and_stably_named() { + assert_eq!( + StoreMigrationPhase::ALL, + EXPECTED.map(|(phase, _name)| phase) + ); + for (phase, name) in EXPECTED { + assert_eq!(phase.to_string(), name); + } +} From 89429a5bbde841fa768547307d22ebb92a24cee9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 03:24:16 -0700 Subject: [PATCH 35/50] Add: Stream store migration inventory --- CHANGELOG.md | 3 +- .../segment-store-v2/migration-inventory.md | 6 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/admitted_segment.rs | 4 + src/adapters/store_migration.rs | 10 ++ .../migration_inventory_entry.rs | 51 +++++++ .../migration_inventory_entry_count.rs | 35 +++++ .../migration_inventory_entry_count_error.rs | 29 ++++ .../migration_inventory_error.rs | 65 +++++++++ .../migration_inventory_hasher.rs | 91 ++++++++++++ src/lib.rs | 4 +- tests/store_migration_inventory.rs | 135 ++++++++++++++++++ 12 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 src/adapters/store_migration/migration_inventory_entry.rs create mode 100644 src/adapters/store_migration/migration_inventory_entry_count.rs create mode 100644 src/adapters/store_migration/migration_inventory_entry_count_error.rs create mode 100644 src/adapters/store_migration/migration_inventory_error.rs create mode 100644 src/adapters/store_migration/migration_inventory_hasher.rs create mode 100644 tests/store_migration_inventory.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a6da96b..12d980e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ after its public API and format compatibility policies are established. - Version-2 marker, migration-intent, and completion-receipt admission now binds exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all - three decoders, and `StoreMigrationPhase` freezes 21 transitions. Retention preflight combines + three decoders, streamed inventory is bounded, and `StoreMigrationPhase` + freezes 21 transitions. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md index b50501e..f970abb 100644 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -47,3 +47,9 @@ do not enter the digest. The exact one-segment, one-catalog input and its canonical entries are frozen in the version-2 corpus [`inventory.tsv`](../../../conformance/segment-store/v2/inventory.tsv). + +`StoreMigrationInventoryEntry` derives canonical bytes only from admitted +artifacts. `StoreMigrationInventoryHasher` requires the bounded entry count +before streaming, retains only the preceding entry, refuses duplicate or +out-of-order evidence, and reproduces the frozen digest. Capability-relative +filesystem inventory and mutation revalidation remain unimplemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e064fba..f282725 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,7 +30,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | diff --git a/src/adapters/admitted_segment.rs b/src/adapters/admitted_segment.rs index f800819..862e8a2 100644 --- a/src/adapters/admitted_segment.rs +++ b/src/adapters/admitted_segment.rs @@ -51,6 +51,10 @@ impl<'a> AdmittedSegment<'a> { self.seal.digest() } + pub(super) const fn segment_length(&self) -> u64 { + self.seal.segment_length() + } + /// Returns a revalidating iterator over records in physical order. #[must_use] pub const fn records(&self) -> SegmentRecords<'a> { diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 1ec3fbc..b1dbf9f 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -18,6 +18,11 @@ mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_inventory_entry; +mod migration_inventory_entry_count; +mod migration_inventory_entry_count_error; +mod migration_inventory_error; +mod migration_inventory_hasher; mod migration_phase; mod migration_receipt_decode_error; mod migration_receipt_decode_error_display; @@ -41,6 +46,11 @@ pub use initial_gc_state_digest::InitialGcStateDigest; pub use initial_retention_state_digest::InitialRetentionStateDigest; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use migration_inventory_entry::StoreMigrationInventoryEntry; +pub use migration_inventory_entry_count::StoreMigrationInventoryEntryCount; +pub use migration_inventory_entry_count_error::StoreMigrationInventoryEntryCountError; +pub use migration_inventory_error::StoreMigrationInventoryError; +pub use migration_inventory_hasher::StoreMigrationInventoryHasher; pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; pub use migration_synchronization_mask::MigrationSynchronizationMask; diff --git a/src/adapters/store_migration/migration_inventory_entry.rs b/src/adapters/store_migration/migration_inventory_entry.rs new file mode 100644 index 0000000..1ce8bc0 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_entry.rs @@ -0,0 +1,51 @@ +//! This boundary module owns canonical migration inventory entries. + +use crate::{AdmittedCatalog, AdmittedSegment}; + +const SEGMENT_KIND: u8 = 1; +const CATALOG_KIND: u8 = 2; +const ENCODED_LENGTH: usize = 56; + +/// Canonical physical coordinate for one admitted version-1 pool artifact. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct StoreMigrationInventoryEntry([u8; ENCODED_LENGTH]); + +impl StoreMigrationInventoryEntry { + /// Constructs the canonical entry for one completely admitted segment. + pub const fn from_segment(segment: &AdmittedSegment<'_>) -> Self { + Self(encode( + SEGMENT_KIND, + 0, + segment.segment_length(), + segment.digest().as_bytes(), + )) + } + + /// Constructs the canonical entry for one completely admitted catalog. + pub const fn from_catalog(catalog: &AdmittedCatalog<'_, '_>) -> Self { + Self(encode( + CATALOG_KIND, + catalog.generation().get(), + catalog.length().get(), + catalog.digest().as_bytes(), + )) + } + + /// Returns the exact 56 canonical bytes. + pub const fn encoded(&self) -> &[u8; ENCODED_LENGTH] { + &self.0 + } +} + +const fn encode(kind: u8, generation: u64, length: u64, digest: &[u8; 32]) -> [u8; ENCODED_LENGTH] { + let mut encoded = [0_u8; ENCODED_LENGTH]; + let (kind_and_reserved, remainder) = encoded.split_at_mut(8); + kind_and_reserved.copy_from_slice(&[kind, 0, 0, 0, 0, 0, 0, 0]); + let (generation_bytes, remainder) = remainder.split_at_mut(8); + generation_bytes.copy_from_slice(&generation.to_be_bytes()); + let (length_bytes, digest_bytes) = remainder.split_at_mut(8); + length_bytes.copy_from_slice(&length.to_be_bytes()); + digest_bytes.copy_from_slice(digest); + encoded +} diff --git a/src/adapters/store_migration/migration_inventory_entry_count.rs b/src/adapters/store_migration/migration_inventory_entry_count.rs new file mode 100644 index 0000000..e0bc3c7 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_entry_count.rs @@ -0,0 +1,35 @@ +//! This module owns the bounded migration inventory entry count. + +use super::StoreMigrationInventoryEntryCountError; + +/// Exact number of canonical entries expected in one migration inventory. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct StoreMigrationInventoryEntryCount(u32); + +impl StoreMigrationInventoryEntryCount { + /// Largest number of immutable-pool entries admitted by version 2. + pub const MAXIMUM: u32 = 2_097_152; + + /// Admits one exact entry count, including an empty inventory. + /// + /// # Errors + /// + /// Returns [`StoreMigrationInventoryEntryCountError`] above + /// [`Self::MAXIMUM`]. + pub const fn new(value: u32) -> Result { + if value <= Self::MAXIMUM { + Ok(Self(value)) + } else { + Err(StoreMigrationInventoryEntryCountError::AboveMaximum { + observed: value, + maximum: Self::MAXIMUM, + }) + } + } + + /// Returns the exact admitted count. + pub const fn get(self) -> u32 { + self.0 + } +} diff --git a/src/adapters/store_migration/migration_inventory_entry_count_error.rs b/src/adapters/store_migration/migration_inventory_entry_count_error.rs new file mode 100644 index 0000000..6f44336 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_entry_count_error.rs @@ -0,0 +1,29 @@ +//! This boundary module owns migration inventory entry-count refusals. + +use std::error::Error; +use std::fmt; + +/// Failure to admit a bounded migration inventory entry count. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationInventoryEntryCountError { + /// The requested count exceeds the immutable protocol maximum. + AboveMaximum { + /// Count supplied by the caller. + observed: u32, + /// Largest count admitted by the protocol. + maximum: u32, + }, +} + +impl fmt::Display for StoreMigrationInventoryEntryCountError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AboveMaximum { observed, maximum } => write!( + formatter, + "migration inventory entry count {observed} exceeds maximum {maximum}" + ), + } + } +} + +impl Error for StoreMigrationInventoryEntryCountError {} diff --git a/src/adapters/store_migration/migration_inventory_error.rs b/src/adapters/store_migration/migration_inventory_error.rs new file mode 100644 index 0000000..e1d35b7 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_error.rs @@ -0,0 +1,65 @@ +//! This boundary module owns streamed migration inventory refusals. + +use std::error::Error; +use std::fmt; + +use super::{StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount}; + +/// Failure to stream one bounded canonical migration inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationInventoryError { + /// An entry would exceed the declared inventory count. + EntryCountExceeded { + /// Exact count declared before hashing. + expected: StoreMigrationInventoryEntryCount, + /// Count that the attempted entry would produce. + observed: u32, + }, + /// The same canonical entry appeared more than once. + Duplicate { + /// Repeated canonical entry. + entry: StoreMigrationInventoryEntry, + }, + /// Canonical entry order moved backward. + OutOfOrder { + /// Last entry admitted before the refusal. + previous: StoreMigrationInventoryEntry, + /// Entry observed after `previous`. + observed: StoreMigrationInventoryEntry, + }, + /// Finalization observed fewer entries than declared. + Incomplete { + /// Exact count declared before hashing. + expected: StoreMigrationInventoryEntryCount, + /// Exact number of entries admitted. + observed: u32, + }, + /// Checked observed-count arithmetic overflowed. + EntryCountOverflow, +} + +impl fmt::Display for StoreMigrationInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EntryCountExceeded { expected, observed } => write!( + formatter, + "migration inventory expected {} entries but observed at least {observed}", + expected.get() + ), + Self::Duplicate { .. } => formatter.write_str("duplicate migration inventory entry"), + Self::OutOfOrder { .. } => { + formatter.write_str("migration inventory entries are out of canonical order") + } + Self::Incomplete { expected, observed } => write!( + formatter, + "migration inventory expected {} entries but observed {observed}", + expected.get() + ), + Self::EntryCountOverflow => { + formatter.write_str("migration inventory entry count overflow") + } + } + } +} + +impl Error for StoreMigrationInventoryError {} diff --git a/src/adapters/store_migration/migration_inventory_hasher.rs b/src/adapters/store_migration/migration_inventory_hasher.rs new file mode 100644 index 0000000..f802e35 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_hasher.rs @@ -0,0 +1,91 @@ +//! This boundary module owns streamed canonical migration inventory identity. + +use super::{ + ImmutablePoolInventoryDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, + StoreMigrationInventoryError, +}; + +const DOMAIN: &[u8] = b"keep.store-v1-pool-inventory/v2\0"; + +/// In-progress bounded digest over one declared canonical pool inventory. +/// +/// Entries must be supplied in complete canonical byte order. The hasher +/// retains only the preceding entry and never materializes the complete +/// encoded inventory. +#[must_use] +pub struct StoreMigrationInventoryHasher { + expected: StoreMigrationInventoryEntryCount, + observed: u32, + previous: Option, + hasher: blake3::Hasher, +} + +impl StoreMigrationInventoryHasher { + /// Begins one inventory whose exact count is known before entry streaming. + pub fn new(expected: StoreMigrationInventoryEntryCount) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(DOMAIN); + hasher.update(&expected.get().to_be_bytes()); + Self { + expected, + observed: 0, + previous: None, + hasher, + } + } + + /// Admits and hashes the next exact canonical entry. + /// + /// # Errors + /// + /// Returns [`StoreMigrationInventoryError`] for count excess, a duplicate, + /// noncanonical order, or checked count overflow. + pub fn push( + &mut self, + entry: StoreMigrationInventoryEntry, + ) -> Result<(), StoreMigrationInventoryError> { + let observed = self + .observed + .checked_add(1) + .ok_or(StoreMigrationInventoryError::EntryCountOverflow)?; + if observed > self.expected.get() { + return Err(StoreMigrationInventoryError::EntryCountExceeded { + expected: self.expected, + observed, + }); + } + if let Some(previous) = self.previous { + if entry == previous { + return Err(StoreMigrationInventoryError::Duplicate { entry }); + } + if entry < previous { + return Err(StoreMigrationInventoryError::OutOfOrder { + previous, + observed: entry, + }); + } + } + self.hasher.update(entry.encoded()); + self.previous = Some(entry); + self.observed = observed; + Ok(()) + } + + /// Finalizes only after the declared number of entries was admitted. + /// + /// # Errors + /// + /// Returns [`StoreMigrationInventoryError::Incomplete`] when fewer entries + /// were supplied than declared. + pub fn finish(self) -> Result { + if self.observed != self.expected.get() { + return Err(StoreMigrationInventoryError::Incomplete { + expected: self.expected, + observed: self.observed, + }); + } + Ok(ImmutablePoolInventoryDigest::from_admitted( + *self.hasher.finalize().as_bytes(), + )) + } +} diff --git a/src/lib.rs b/src/lib.rs index c4d86a8..bacb7a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,7 +110,9 @@ pub use adapters::{ StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreMigrationPhase, StoreMigrationReceiptDecodeError, + StoreMigrationIntentDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, + StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, + StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, diff --git a/tests/store_migration_inventory.rs b/tests/store_migration_inventory.rs new file mode 100644 index 0000000..ffe49fc --- /dev/null +++ b/tests/store_migration_inventory.rs @@ -0,0 +1,135 @@ +//! Canonical version-1 immutable-pool migration inventory laws. + +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, ChecksummedCatalog, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, + StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, + StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, + StoreMigrationInventoryHasher, +}; +use support::decode_hex; + +const SEGMENT: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const SEGMENT_ENTRY: &str = concat!( + "0100000000000000", + "0000000000000000", + "0000000000000151", + "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc", +); +const CATALOG_ENTRY: &str = concat!( + "0200000000000000", + "0000000000000001", + "0000000000000160", + "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320", +); +const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; + +#[test] +fn frozen_inventory_entries_and_digest_are_exact() -> Result<(), Box> { + let (segment, catalog) = frozen_entries()?; + assert_eq!(segment.encoded().as_slice(), decode_hex(SEGMENT_ENTRY)?); + assert_eq!(catalog.encoded().as_slice(), decode_hex(CATALOG_ENTRY)?); + + let count = StoreMigrationInventoryEntryCount::new(2)?; + let mut inventory = StoreMigrationInventoryHasher::new(count); + inventory.push(segment)?; + inventory.push(catalog)?; + let digest = inventory.finish()?; + assert_eq!(digest.as_bytes().as_slice(), decode_hex(INVENTORY_DIGEST)?); + Ok(()) +} + +#[test] +fn inventory_refuses_duplicate_and_out_of_order_entries() -> Result<(), Box> { + let (segment, catalog) = frozen_entries()?; + let count = StoreMigrationInventoryEntryCount::new(2)?; + + let mut duplicate = StoreMigrationInventoryHasher::new(count); + duplicate.push(segment)?; + assert_eq!( + duplicate.push(segment), + Err(StoreMigrationInventoryError::Duplicate { entry: segment }) + ); + + let mut out_of_order = StoreMigrationInventoryHasher::new(count); + out_of_order.push(catalog)?; + assert_eq!( + out_of_order.push(segment), + Err(StoreMigrationInventoryError::OutOfOrder { + previous: catalog, + observed: segment, + }) + ); + Ok(()) +} + +#[test] +fn inventory_refuses_count_overrun_and_incomplete_finalization() -> Result<(), Box> { + let (segment, catalog) = frozen_entries()?; + let one = StoreMigrationInventoryEntryCount::new(1)?; + let two = StoreMigrationInventoryEntryCount::new(2)?; + + let mut overrun = StoreMigrationInventoryHasher::new(one); + overrun.push(segment)?; + assert_eq!( + overrun.push(catalog), + Err(StoreMigrationInventoryError::EntryCountExceeded { + expected: one, + observed: 2, + }) + ); + + let mut incomplete = StoreMigrationInventoryHasher::new(two); + incomplete.push(segment)?; + assert_eq!( + incomplete.finish(), + Err(StoreMigrationInventoryError::Incomplete { + expected: two, + observed: 1, + }) + ); + Ok(()) +} + +#[test] +fn inventory_count_has_the_exact_protocol_bound() { + assert_eq!( + StoreMigrationInventoryEntryCount::new(0).map(StoreMigrationInventoryEntryCount::get), + Ok(0) + ); + assert_eq!( + StoreMigrationInventoryEntryCount::new(StoreMigrationInventoryEntryCount::MAXIMUM) + .map(StoreMigrationInventoryEntryCount::get), + Ok(StoreMigrationInventoryEntryCount::MAXIMUM) + ); + assert_eq!( + StoreMigrationInventoryEntryCount::new(2_097_153), + Err(StoreMigrationInventoryEntryCountError::AboveMaximum { + observed: 2_097_153, + maximum: 2_097_152, + }) + ); +} + +fn frozen_entries() +-> Result<(StoreMigrationInventoryEntry, StoreMigrationInventoryEntry), Box> { + let segment_bytes = fixture(SEGMENT)?; + let catalog_bytes = fixture(CATALOG)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segment_entry = StoreMigrationInventoryEntry::from_segment(&segment); + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?.admit(&[segment])?; + let catalog_entry = StoreMigrationInventoryEntry::from_catalog(&catalog); + Ok((segment_entry, catalog_entry)) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 72b09182254f5b549d94e02222787713f3acaec7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 03:40:33 -0700 Subject: [PATCH 36/50] Add: Construct store migration intents --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 2 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 4 + .../canonical_migration_intent.rs | 67 +++++++++++++ .../migration_intent_decoder.rs | 79 ++++++--------- .../migration_intent_encoder.rs | 84 ++++++++++++++++ .../migration_intent_format.rs | 61 ++++++++++++ src/lib.rs | 21 ++-- tests/store_migration_intent_encoding.rs | 95 +++++++++++++++++++ 10 files changed, 352 insertions(+), 65 deletions(-) create mode 100644 src/adapters/store_migration/canonical_migration_intent.rs create mode 100644 src/adapters/store_migration/migration_intent_encoder.rs create mode 100644 src/adapters/store_migration/migration_intent_format.rs create mode 100644 tests/store_migration_intent_encoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 12d980e..94ceadd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, migration-intent, and completion-receipt admission now binds +- Version-2 marker, canonical migration-intent construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index f631c23..b652509 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`AdmittedStoreMigrationIntent` admits intent integrity and identity; `AdmittedStoreMigrationReceipt` binds that intent, the marker, registered empty states, and all synchronization bits. +`CanonicalStoreMigrationIntent` reproduces intent bytes from typed coordinates; `AdmittedStoreMigrationIntent` admits integrity and identity; `AdmittedStoreMigrationReceipt` binds the intent, marker, empty states, and synchronization bits. These record boundaries do not prove the named live inventory, physical root, store version, or execution of filesystem migration. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index f282725..072dc6c 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; seeded `migration_format` fuzz target | Implemented | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical intent construction in `tests/store_migration_intent_encoding.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index b1dbf9f..ee2ab13 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -4,6 +4,7 @@ mod admitted_format_marker; mod admitted_migration_intent; mod admitted_migration_receipt; mod canonical_format_marker; +mod canonical_migration_intent; mod empty_disposition_set_digest; mod format_definition_digest; mod format_marker_decode_error; @@ -18,6 +19,8 @@ mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_intent_encoder; +mod migration_intent_format; mod migration_inventory_entry; mod migration_inventory_entry_count; mod migration_inventory_entry_count_error; @@ -37,6 +40,7 @@ pub use admitted_format_marker::AdmittedStoreFormatMarker; pub use admitted_migration_intent::AdmittedStoreMigrationIntent; pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use canonical_migration_intent::CanonicalStoreMigrationIntent; pub use empty_disposition_set_digest::EmptyDispositionSetDigest; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs new file mode 100644 index 0000000..f6e2303 --- /dev/null +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -0,0 +1,67 @@ +//! This boundary module owns canonical owned migration-intent bytes. + +use super::{ + ImmutablePoolInventoryDigest, StoreIdentifier, StoreMigrationIntentDigest, + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + migration_intent_encoder, migration_intent_format, +}; +use crate::CatalogSnapshot; + +/// Owned canonical version-2 store-migration intent. +/// +/// Construction preserves admitted catalog coordinates and serializes the +/// supplied inventory and physical-root coordinates. It does not prove that +/// the inventory or physical root remains current. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalStoreMigrationIntent { + encoded: [u8; migration_intent_format::ENCODED_LENGTH], + digest: StoreMigrationIntentDigest, + store_identifier: StoreIdentifier, +} + +impl CanonicalStoreMigrationIntent { + /// Constructs one canonical intent from typed migration coordinates. + pub fn from_snapshot( + snapshot: &CatalogSnapshot<'_, '_, '_>, + inventory_digest: ImmutablePoolInventoryDigest, + root_device_identity: StoreRootDeviceIdentity, + root_mount_identity: StoreRootMountIdentity, + root_file_identity: StoreRootFileIdentity, + ) -> Self { + migration_intent_encoder::encode( + snapshot, + inventory_digest, + root_device_identity, + root_mount_identity, + root_file_identity, + ) + } + + /// Returns the exact canonical intent bytes. + pub const fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the domain-separated identity of all intent bytes. + pub const fn digest(&self) -> StoreMigrationIntentDigest { + self.digest + } + + /// Returns the deterministic logical store identity. + pub const fn store_identifier(&self) -> StoreIdentifier { + self.store_identifier + } + + pub(super) const fn admitted( + encoded: [u8; migration_intent_format::ENCODED_LENGTH], + digest: StoreMigrationIntentDigest, + store_identifier: StoreIdentifier, + ) -> Self { + Self { + encoded, + digest, + store_identifier, + } + } +} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs index 7dabc5c..7eeb34e 100644 --- a/src/adapters/store_migration/migration_intent_decoder.rs +++ b/src/adapters/store_migration/migration_intent_decoder.rs @@ -1,25 +1,17 @@ //! This boundary module owns store-migration intent decoding order. use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_intent_format::{self as format, StoreIdentifierFields}; use super::migration_record_bytes::{ read_array, read_u16, read_u32, read_u64, require_length, wrong_length, }; use super::{ AdmittedStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, - StoreIdentifier, StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + StoreIdentifier, StoreMigrationIntentDecodeError, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; -const CHECKSUM_OFFSET: usize = 224; -const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; -const VERSION: u16 = 2; -const RECORD_LENGTH: u16 = 256; -const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; -const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; -const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; -const ZERO_DIGEST: [u8; 32] = [0; 32]; - pub(super) fn decode( encoded: &[u8], ) -> Result, StoreMigrationIntentDecodeError> { @@ -47,26 +39,26 @@ pub(super) fn decode( Ok(AdmittedStoreMigrationIntent::admitted( encoded, fields, - digest(encoded), + format::digest(encoded), )) } fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { let magic = read_array(encoded, 0)?; - if magic != MAGIC { + if magic != format::MAGIC { return Err(StoreMigrationIntentDecodeError::InvalidMagic { observed: magic }); } let version = read_u16(encoded, 16)?; - if version != VERSION { + if version != format::VERSION { return Err(StoreMigrationIntentDecodeError::UnsupportedVersion { - expected: VERSION, + expected: format::VERSION, observed: version, }); } let record_length = read_u16(encoded, 18)?; - if record_length != RECORD_LENGTH { + if record_length != format::RECORD_LENGTH { return Err(StoreMigrationIntentDecodeError::InvalidRecordLength { - expected: RECORD_LENGTH, + expected: format::RECORD_LENGTH, observed: record_length, }); } @@ -79,10 +71,10 @@ fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecod fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { let preimage = encoded - .get(..CHECKSUM_OFFSET) + .get(..format::CHECKSUM_OFFSET) .ok_or_else(|| wrong_length(encoded))?; - let observed = read_array(encoded, CHECKSUM_OFFSET)?; - let expected = hash(CHECKSUM_DOMAIN, &[preimage]); + let observed = read_array(encoded, format::CHECKSUM_OFFSET)?; + let expected = format::checksum(preimage); if observed == expected { Ok(()) } else { @@ -111,13 +103,13 @@ fn read_predecessor( observed: [u8; 32], ) -> Result, StoreMigrationIntentDecodeError> { if generation.get() == 1 { - return if observed == ZERO_DIGEST { + return if observed == format::ZERO_DIGEST { Ok(None) } else { Err(StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { observed }) }; } - if observed == ZERO_DIGEST { + if observed == format::ZERO_DIGEST { return Err( StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { generation: generation.get(), @@ -144,38 +136,21 @@ fn read_definition_digest( fn verify_store_identifier( fields: &StoreMigrationIntentFields, ) -> Result<(), StoreMigrationIntentDecodeError> { - let predecessor = fields - .predecessor_catalog_digest - .as_ref() - .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); - let expected = hash( - STORE_IDENTIFIER_DOMAIN, - &[ - &fields.catalog_generation.get().to_be_bytes(), - &fields.catalog_length.get().to_be_bytes(), - fields.catalog_digest.as_bytes(), - predecessor, - fields.inventory_digest.as_bytes(), - fields.target_definition_digest.as_bytes(), - ], - ); + let expected = format::store_identifier(&StoreIdentifierFields { + catalog_generation: fields.catalog_generation, + catalog_length: fields.catalog_length, + catalog_digest: fields.catalog_digest, + predecessor_catalog_digest: fields.predecessor_catalog_digest, + inventory_digest: fields.inventory_digest, + target_definition_digest: fields.target_definition_digest, + }); let observed = *fields.store_identifier.as_bytes(); - if observed == expected { + if observed == *expected.as_bytes() { Ok(()) } else { - Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { expected, observed }) - } -} - -fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { - StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) -} - -fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(domain); - for field in fields { - hasher.update(field); + Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { + expected: *expected.as_bytes(), + observed, + }) } - *hasher.finalize().as_bytes() } diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs new file mode 100644 index 0000000..a461655 --- /dev/null +++ b/src/adapters/store_migration/migration_intent_encoder.rs @@ -0,0 +1,84 @@ +//! This boundary module owns canonical migration-intent encoding. + +use super::migration_intent_format::StoreIdentifierFields; +use super::{ + CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, + StoreIdentifier, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + migration_intent_format as format, +}; +use crate::CatalogSnapshot; + +#[derive(Clone, Copy)] +struct RootIdentities { + device: StoreRootDeviceIdentity, + mount: StoreRootMountIdentity, + file: StoreRootFileIdentity, +} + +pub(super) fn encode( + snapshot: &CatalogSnapshot<'_, '_, '_>, + inventory_digest: ImmutablePoolInventoryDigest, + root_device_identity: StoreRootDeviceIdentity, + root_mount_identity: StoreRootMountIdentity, + root_file_identity: StoreRootFileIdentity, +) -> CanonicalStoreMigrationIntent { + let fields = StoreIdentifierFields { + catalog_generation: snapshot.generation(), + catalog_length: snapshot.catalog_length(), + catalog_digest: snapshot.catalog_digest(), + predecessor_catalog_digest: snapshot.previous_catalog_digest(), + inventory_digest, + target_definition_digest: StoreFormatDefinitionDigest::VERSION_TWO, + }; + let roots = RootIdentities { + device: root_device_identity, + mount: root_mount_identity, + file: root_file_identity, + }; + let store_identifier = format::store_identifier(&fields); + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + write_preimage(preimage, &fields, roots, store_identifier); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + let digest = format::digest(&encoded); + CanonicalStoreMigrationIntent::admitted(encoded, digest, store_identifier) +} + +fn write_preimage( + output: &mut [u8], + fields: &StoreIdentifierFields, + roots: RootIdentities, + store_identifier: StoreIdentifier, +) { + let (magic, output) = output.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, output) = output.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, output) = output.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, output) = output.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (generation, output) = output.split_at_mut(8); + generation.copy_from_slice(&fields.catalog_generation.get().to_be_bytes()); + let (catalog_length, output) = output.split_at_mut(8); + catalog_length.copy_from_slice(&fields.catalog_length.get().to_be_bytes()); + let (catalog_digest, output) = output.split_at_mut(32); + catalog_digest.copy_from_slice(fields.catalog_digest.as_bytes()); + let predecessor = fields + .predecessor_catalog_digest + .as_ref() + .map_or(&format::ZERO_DIGEST, crate::CatalogDigest::as_bytes); + let (predecessor_digest, output) = output.split_at_mut(32); + predecessor_digest.copy_from_slice(predecessor); + let (inventory_digest, output) = output.split_at_mut(32); + inventory_digest.copy_from_slice(fields.inventory_digest.as_bytes()); + let (device_identity, output) = output.split_at_mut(8); + device_identity.copy_from_slice(&roots.device.get().to_be_bytes()); + let (mount_identity, output) = output.split_at_mut(8); + mount_identity.copy_from_slice(&roots.mount.get().to_be_bytes()); + let (file_identity, output) = output.split_at_mut(8); + file_identity.copy_from_slice(&roots.file.get().to_be_bytes()); + let (definition_digest, store_identifier_slot) = output.split_at_mut(32); + definition_digest.copy_from_slice(fields.target_definition_digest.as_bytes()); + store_identifier_slot.copy_from_slice(store_identifier.as_bytes()); +} diff --git a/src/adapters/store_migration/migration_intent_format.rs b/src/adapters/store_migration/migration_intent_format.rs new file mode 100644 index 0000000..0077bcb --- /dev/null +++ b/src/adapters/store_migration/migration_intent_format.rs @@ -0,0 +1,61 @@ +//! This boundary module owns shared migration-intent format identity. + +use super::{ + ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, + StoreMigrationIntentDigest, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +pub(super) const CHECKSUM_OFFSET: usize = 224; +pub(super) const ENCODED_LENGTH: usize = 256; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; +pub(super) const RECORD_LENGTH: u16 = 256; +pub(super) const VERSION: u16 = 2; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; +const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; +const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; +pub(super) const ZERO_DIGEST: [u8; 32] = [0; 32]; + +pub(super) struct StoreIdentifierFields { + pub(super) catalog_generation: CatalogGeneration, + pub(super) catalog_length: CatalogLength, + pub(super) catalog_digest: CatalogDigest, + pub(super) predecessor_catalog_digest: Option, + pub(super) inventory_digest: ImmutablePoolInventoryDigest, + pub(super) target_definition_digest: StoreFormatDefinitionDigest, +} + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + hash(CHECKSUM_DOMAIN, &[preimage]) +} + +pub(super) fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { + StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) +} + +pub(super) fn store_identifier(fields: &StoreIdentifierFields) -> StoreIdentifier { + let predecessor = fields + .predecessor_catalog_digest + .as_ref() + .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); + StoreIdentifier::from_hash(hash( + STORE_IDENTIFIER_DOMAIN, + &[ + &fields.catalog_generation.get().to_be_bytes(), + &fields.catalog_length.get().to_be_bytes(), + fields.catalog_digest.as_bytes(), + predecessor, + fields.inventory_digest.as_bytes(), + fields.target_definition_digest.as_bytes(), + ], + )) +} + +fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + for field in fields { + hasher.update(field); + } + *hasher.finalize().as_bytes() +} diff --git a/src/lib.rs b/src/lib.rs index bacb7a1..0f49f61 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,16 +58,17 @@ pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CanonicalStoreFormatMarker, CatalogAdmissionError, - CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, - CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, - FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + CanonicalPublicationHead, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, + CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, + CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, + CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, + CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, diff --git a/tests/store_migration_intent_encoding.rs b/tests/store_migration_intent_encoding.rs new file mode 100644 index 0000000..2158dbc --- /dev/null +++ b/tests/store_migration_intent_encoding.rs @@ -0,0 +1,95 @@ +//! Canonical version-2 store-migration intent encoding laws. + +#[path = "store_migration_intent/fixture.rs"] +mod fixture; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, AdmittedStoreMigrationIntent, CanonicalStoreMigrationIntent, + ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, SegmentReadPolicy, + SegmentRecordLimit, +}; +use support::decode_hex; + +const SEGMENT: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_TWO: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +const HEAD_TWO: &str = + include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); +const PREDECESSOR_OFFSET: usize = 72; +const PREDECESSOR_END: usize = 104; + +#[test] +fn admitted_coordinates_reproduce_the_frozen_intent() -> Result<(), Box> { + let expected = fixture::fixture_bytes()?; + let admitted = AdmittedStoreMigrationIntent::decode(&expected)?; + assert_eq!( + admitted.inventory_digest().as_bytes(), + &fixture::INVENTORY_DIGEST + ); + let canonical = canonical_intent(CATALOG, HEAD, &admitted)?; + + assert_eq!(canonical.encoded(), expected); + assert_eq!(canonical.digest(), admitted.digest()); + assert_eq!(canonical.store_identifier(), admitted.store_identifier()); + assert_eq!(canonical.digest().as_bytes(), &fixture::INTENT_DIGEST); + assert_eq!( + canonical.store_identifier().as_bytes(), + &fixture::STORE_IDENTIFIER + ); + Ok(()) +} + +#[test] +fn successor_intent_encodes_the_exact_predecessor() -> Result<(), Box> { + let source_bytes = fixture::fixture_bytes()?; + let source = AdmittedStoreMigrationIntent::decode(&source_bytes)?; + let canonical = canonical_intent(CATALOG_TWO, HEAD_TWO, &source)?; + let admitted = AdmittedStoreMigrationIntent::decode(canonical.encoded())?; + + assert_eq!(admitted.catalog_generation().get(), 2); + assert_eq!( + canonical.encoded().get(PREDECESSOR_OFFSET..PREDECESSOR_END), + Some(fixture::CATALOG_DIGEST.as_slice()) + ); + assert_eq!( + admitted + .predecessor_catalog_digest() + .ok_or("successor intent omitted its predecessor")? + .as_bytes(), + &fixture::CATALOG_DIGEST + ); + Ok(()) +} + +fn canonical_intent( + catalog_hex: &str, + head_hex: &str, + source: &AdmittedStoreMigrationIntent<'_>, +) -> Result> { + let segment_bytes = protocol_fixture(SEGMENT)?; + let catalog_bytes = protocol_fixture(catalog_hex)?; + let head_bytes = protocol_fixture(head_hex)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?.admit(&[segment])?; + let snapshot = ChecksummedPublicationHead::decode(&head_bytes)?.admit(catalog)?; + Ok(CanonicalStoreMigrationIntent::from_snapshot( + &snapshot, + source.inventory_digest(), + source.root_device_identity(), + source.root_mount_identity(), + source.root_file_identity(), + )) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn protocol_fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From ea293f8bb02e64bb14f77c63a23492dd071d4ffc Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 03:52:15 -0700 Subject: [PATCH 37/50] Add: Construct store migration receipts --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 2 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 4 ++ .../canonical_migration_intent.rs | 11 ++++ .../canonical_migration_receipt.rs | 35 +++++++++++++ .../migration_receipt_decoder.rs | 30 ++++------- .../migration_receipt_encoder.rs | 52 +++++++++++++++++++ .../migration_receipt_format.rs | 15 ++++++ .../migration_receipt_initial_state.rs | 27 +++++++--- src/lib.rs | 16 +++--- tests/store_migration_receipt_encoding.rs | 46 ++++++++++++++++ 12 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 src/adapters/store_migration/canonical_migration_receipt.rs create mode 100644 src/adapters/store_migration/migration_receipt_encoder.rs create mode 100644 src/adapters/store_migration/migration_receipt_format.rs create mode 100644 tests/store_migration_receipt_encoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ceadd..9c00eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, canonical migration-intent construction, and record admission bind +- Version-2 marker, canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index b652509..f446cf3 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` reproduces intent bytes from typed coordinates; `AdmittedStoreMigrationIntent` admits integrity and identity; `AdmittedStoreMigrationReceipt` binds the intent, marker, empty states, and synchronization bits. +`CanonicalStoreMigrationIntent` reproduces typed intent bytes; `CanonicalStoreMigrationReceipt` binds canonical intent, marker, empty states, and complete synchronization; admitted record types verify both. These record boundaries do not prove the named live inventory, physical root, store version, or execution of filesystem migration. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 072dc6c..e68ae63 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical intent construction in `tests/store_migration_intent_encoding.rs`; seeded `migration_format` fuzz target | Implemented | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index ee2ab13..2515c97 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -5,6 +5,7 @@ mod admitted_migration_intent; mod admitted_migration_receipt; mod canonical_format_marker; mod canonical_migration_intent; +mod canonical_migration_receipt; mod empty_disposition_set_digest; mod format_definition_digest; mod format_marker_decode_error; @@ -30,6 +31,8 @@ mod migration_phase; mod migration_receipt_decode_error; mod migration_receipt_decode_error_display; mod migration_receipt_decoder; +mod migration_receipt_encoder; +mod migration_receipt_format; mod migration_receipt_initial_state; mod migration_record_bytes; mod migration_synchronization_mask; @@ -41,6 +44,7 @@ pub use admitted_migration_intent::AdmittedStoreMigrationIntent; pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; pub use canonical_format_marker::CanonicalStoreFormatMarker; pub use canonical_migration_intent::CanonicalStoreMigrationIntent; +pub use canonical_migration_receipt::CanonicalStoreMigrationReceipt; pub use empty_disposition_set_digest::EmptyDispositionSetDigest; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs index f6e2303..9c70969 100644 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -21,6 +21,17 @@ pub struct CanonicalStoreMigrationIntent { } impl CanonicalStoreMigrationIntent { + /// Owns the exact bytes and identities of an admitted intent. + pub const fn from_admitted(intent: &super::AdmittedStoreMigrationIntent<'_>) -> Self { + let mut encoded = [0_u8; migration_intent_format::ENCODED_LENGTH]; + encoded.copy_from_slice(intent.encoded()); + Self { + encoded, + digest: intent.digest(), + store_identifier: intent.store_identifier(), + } + } + /// Constructs one canonical intent from typed migration coordinates. pub fn from_snapshot( snapshot: &CatalogSnapshot<'_, '_, '_>, diff --git a/src/adapters/store_migration/canonical_migration_receipt.rs b/src/adapters/store_migration/canonical_migration_receipt.rs new file mode 100644 index 0000000..7c40d86 --- /dev/null +++ b/src/adapters/store_migration/canonical_migration_receipt.rs @@ -0,0 +1,35 @@ +//! This boundary module owns canonical owned migration-receipt bytes. + +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, migration_receipt_encoder, + migration_receipt_format, +}; + +/// Owned canonical version-2 store-migration completion receipt. +/// +/// Construction binds canonical artifacts and the registered complete initial +/// state. It does not prove that the named filesystem transitions occurred. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalStoreMigrationReceipt { + encoded: [u8; migration_receipt_format::ENCODED_LENGTH], +} + +impl CanonicalStoreMigrationReceipt { + /// Constructs the one complete receipt for `intent` and `marker`. + pub fn from_canonical( + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, + ) -> Self { + migration_receipt_encoder::encode(intent, marker) + } + + /// Returns the exact canonical receipt bytes. + pub const fn encoded(&self) -> &[u8] { + &self.encoded + } + + pub(super) const fn admitted(encoded: [u8; migration_receipt_format::ENCODED_LENGTH]) -> Self { + Self { encoded } + } +} diff --git a/src/adapters/store_migration/migration_receipt_decoder.rs b/src/adapters/store_migration/migration_receipt_decoder.rs index 308308d..cd92d71 100644 --- a/src/adapters/store_migration/migration_receipt_decoder.rs +++ b/src/adapters/store_migration/migration_receipt_decoder.rs @@ -10,14 +10,9 @@ use super::migration_record_bytes::{ use super::{ AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, MigrationSynchronizationMask, StoreMigrationReceiptDecodeError, + migration_receipt_format as format, }; -const CHECKSUM_OFFSET: usize = 224; -const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; -const VERSION: u16 = 2; -const RECORD_LENGTH: u16 = 256; -const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; - pub(super) fn decode<'encoded>( encoded: &'encoded [u8], intent: &AdmittedStoreMigrationIntent<'_>, @@ -49,20 +44,20 @@ pub(super) fn decode<'encoded>( fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { let magic = read_array(encoded, 0)?; - if magic != MAGIC { + if magic != format::MAGIC { return Err(StoreMigrationReceiptDecodeError::InvalidMagic { observed: magic }); } let version = read_u16(encoded, 16)?; - if version != VERSION { + if version != format::VERSION { return Err(StoreMigrationReceiptDecodeError::UnsupportedVersion { - expected: VERSION, + expected: format::VERSION, observed: version, }); } let record_length = read_u16(encoded, 18)?; - if record_length != RECORD_LENGTH { + if record_length != format::RECORD_LENGTH { return Err(StoreMigrationReceiptDecodeError::InvalidRecordLength { - expected: RECORD_LENGTH, + expected: format::RECORD_LENGTH, observed: record_length, }); } @@ -75,10 +70,10 @@ fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDeco fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { let preimage = encoded - .get(..CHECKSUM_OFFSET) + .get(..format::CHECKSUM_OFFSET) .ok_or_else(|| wrong_length(encoded))?; - let observed = read_array(encoded, CHECKSUM_OFFSET)?; - let expected = hash(CHECKSUM_DOMAIN, preimage); + let observed = read_array(encoded, format::CHECKSUM_OFFSET)?; + let expected = format::checksum(preimage); if observed == expected { Ok(()) } else { @@ -148,10 +143,3 @@ fn read_synchronization_mask( } Ok(MigrationSynchronizationMask::complete()) } - -fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(domain); - hasher.update(bytes); - *hasher.finalize().as_bytes() -} diff --git a/src/adapters/store_migration/migration_receipt_encoder.rs b/src/adapters/store_migration/migration_receipt_encoder.rs new file mode 100644 index 0000000..9020298 --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_encoder.rs @@ -0,0 +1,52 @@ +//! This boundary module owns canonical migration-receipt encoding. + +use super::migration_receipt_initial_state::{ + empty_disposition_digest, initial_gc_digest, initial_retention_digest, +}; +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + MigrationSynchronizationMask, migration_receipt_format as format, +}; + +pub(super) fn encode( + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, +) -> CanonicalStoreMigrationReceipt { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + write_preimage(preimage, intent, marker); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + CanonicalStoreMigrationReceipt::admitted(encoded) +} + +fn write_preimage( + output: &mut [u8], + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, +) { + let (magic, output) = output.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, output) = output.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, output) = output.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, output) = output.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (intent_digest, output) = output.split_at_mut(32); + intent_digest.copy_from_slice(intent.digest().as_bytes()); + let (store_identifier, output) = output.split_at_mut(32); + store_identifier.copy_from_slice(intent.store_identifier().as_bytes()); + let (marker_digest, output) = output.split_at_mut(32); + marker_digest.copy_from_slice(marker.digest().as_bytes()); + let (retention_digest, output) = output.split_at_mut(32); + retention_digest.copy_from_slice(initial_retention_digest().as_bytes()); + let (gc_digest, output) = output.split_at_mut(32); + gc_digest.copy_from_slice(initial_gc_digest().as_bytes()); + let (disposition_digest, synchronization_mask) = output.split_at_mut(32); + disposition_digest.copy_from_slice(empty_disposition_digest().as_bytes()); + synchronization_mask.copy_from_slice( + &MigrationSynchronizationMask::complete() + .bits() + .to_be_bytes(), + ); +} diff --git a/src/adapters/store_migration/migration_receipt_format.rs b/src/adapters/store_migration/migration_receipt_format.rs new file mode 100644 index 0000000..392ff3d --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_format.rs @@ -0,0 +1,15 @@ +//! This boundary module owns shared migration-receipt framing and integrity. + +pub(super) const CHECKSUM_OFFSET: usize = 224; +pub(super) const ENCODED_LENGTH: usize = 256; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; +pub(super) const RECORD_LENGTH: u16 = 256; +pub(super) const VERSION: u16 = 2; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(CHECKSUM_DOMAIN); + hasher.update(preimage); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/store_migration/migration_receipt_initial_state.rs b/src/adapters/store_migration/migration_receipt_initial_state.rs index 01a43d5..d89af3e 100644 --- a/src/adapters/store_migration/migration_receipt_initial_state.rs +++ b/src/adapters/store_migration/migration_receipt_initial_state.rs @@ -13,10 +13,11 @@ const EMPTY_DISPOSITION_DOMAIN: &[u8] = b"keep.empty-disposition-set/v2\0"; pub(super) fn read_initial_retention_digest( encoded: &[u8], ) -> Result { - let expected = digest(INITIAL_RETENTION_DOMAIN); + let admitted = initial_retention_digest(); + let expected = *admitted.as_bytes(); let observed = read_array(encoded, 120)?; if observed == expected { - Ok(InitialRetentionStateDigest::from_hash(expected)) + Ok(admitted) } else { Err( StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { @@ -30,10 +31,11 @@ pub(super) fn read_initial_retention_digest( pub(super) fn read_initial_gc_digest( encoded: &[u8], ) -> Result { - let expected = digest(INITIAL_GC_DOMAIN); + let admitted = initial_gc_digest(); + let expected = *admitted.as_bytes(); let observed = read_array(encoded, 152)?; if observed == expected { - Ok(InitialGcStateDigest::from_hash(expected)) + Ok(admitted) } else { Err(StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { expected, observed }) } @@ -42,10 +44,11 @@ pub(super) fn read_initial_gc_digest( pub(super) fn read_empty_disposition_digest( encoded: &[u8], ) -> Result { - let expected = digest(EMPTY_DISPOSITION_DOMAIN); + let admitted = empty_disposition_digest(); + let expected = *admitted.as_bytes(); let observed = read_array(encoded, 184)?; if observed == expected { - Ok(EmptyDispositionSetDigest::from_hash(expected)) + Ok(admitted) } else { Err( StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { @@ -56,6 +59,18 @@ pub(super) fn read_empty_disposition_digest( } } +pub(super) fn initial_retention_digest() -> InitialRetentionStateDigest { + InitialRetentionStateDigest::from_hash(digest(INITIAL_RETENTION_DOMAIN)) +} + +pub(super) fn initial_gc_digest() -> InitialGcStateDigest { + InitialGcStateDigest::from_hash(digest(INITIAL_GC_DOMAIN)) +} + +pub(super) fn empty_disposition_digest() -> EmptyDispositionSetDigest { + EmptyDispositionSetDigest::from_hash(digest(EMPTY_DISPOSITION_DOMAIN)) +} + fn digest(domain: &[u8]) -> [u8; 32] { *blake3::hash(domain).as_bytes() } diff --git a/src/lib.rs b/src/lib.rs index 0f49f61..e6a551a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,14 +59,14 @@ pub use adapters::{ AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, - CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, - CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, - CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, - CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, - CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, + CanonicalStoreMigrationReceipt, CatalogAdmissionError, CatalogAllocationPhase, + CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, diff --git a/tests/store_migration_receipt_encoding.rs b/tests/store_migration_receipt_encoding.rs new file mode 100644 index 0000000..c5f8793 --- /dev/null +++ b/tests/store_migration_receipt_encoding.rs @@ -0,0 +1,46 @@ +//! Canonical version-2 store-migration receipt encoding laws. + +#[path = "store_migration_receipt/fixture.rs"] +mod fixture; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, +}; + +#[test] +fn canonical_completion_receipt_reproduces_every_frozen_field() -> Result<(), Box> { + let expected = fixture::receipt_bytes()?; + let intent_bytes = fixture::intent_bytes()?; + let admitted_intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let intent = CanonicalStoreMigrationIntent::from_admitted(&admitted_intent); + let marker = CanonicalStoreFormatMarker::version_two(); + assert_eq!(marker.encoded(), fixture::marker_bytes()?); + + let canonical = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); + assert_eq!(canonical.encoded(), expected); + + let admitted_marker = AdmittedStoreFormatMarker::decode(marker.encoded())?; + let admitted = AdmittedStoreMigrationReceipt::decode( + canonical.encoded(), + &admitted_intent, + &admitted_marker, + )?; + assert_eq!( + admitted.initial_retention_state_digest().as_bytes(), + &fixture::INITIAL_RETENTION_DIGEST + ); + assert_eq!( + admitted.initial_gc_state_digest().as_bytes(), + &fixture::INITIAL_GC_DIGEST + ); + assert_eq!( + admitted.empty_disposition_set_digest().as_bytes(), + &fixture::DISPOSITION_DIGEST + ); + assert_eq!(admitted.synchronization_mask().bits(), 0x03ff); + Ok(()) +} From 5a38fa75f7178f9451e0a6a8c7204a473fbdcebb Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 04:03:18 -0700 Subject: [PATCH 38/50] Add: Retain canonical migration coordinates --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 2 +- .../canonical_migration_intent.rs | 75 ++++++++++++++++--- .../migration_intent_encoder.rs | 18 ++++- tests/store_migration_intent_encoding.rs | 36 ++++++++- 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c00eb6..783c217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, canonical intent/receipt construction, and record admission bind +- Version-2 marker, typed canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index f446cf3..a5e4444 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` reproduces typed intent bytes; `CanonicalStoreMigrationReceipt` binds canonical intent, marker, empty states, and complete synchronization; admitted record types verify both. +`CanonicalStoreMigrationIntent` retains and reproduces typed intent coordinates; `CanonicalStoreMigrationReceipt` binds intent, marker, empty states, and complete synchronization; admitted record types verify both. These record boundaries do not prove the named live inventory, physical root, store version, or execution of filesystem migration. diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs index 9c70969..28a95ca 100644 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -1,11 +1,12 @@ //! This boundary module owns canonical owned migration-intent bytes. +use super::admitted_migration_intent::StoreMigrationIntentFields; use super::{ - ImmutablePoolInventoryDigest, StoreIdentifier, StoreMigrationIntentDigest, - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, - migration_intent_encoder, migration_intent_format, + ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, + StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, + StoreRootMountIdentity, migration_intent_encoder, migration_intent_format, }; -use crate::CatalogSnapshot; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength, CatalogSnapshot}; /// Owned canonical version-2 store-migration intent. /// @@ -16,8 +17,8 @@ use crate::CatalogSnapshot; #[derive(Clone, Debug, Eq, PartialEq)] pub struct CanonicalStoreMigrationIntent { encoded: [u8; migration_intent_format::ENCODED_LENGTH], + fields: StoreMigrationIntentFields, digest: StoreMigrationIntentDigest, - store_identifier: StoreIdentifier, } impl CanonicalStoreMigrationIntent { @@ -27,8 +28,19 @@ impl CanonicalStoreMigrationIntent { encoded.copy_from_slice(intent.encoded()); Self { encoded, + fields: StoreMigrationIntentFields { + catalog_generation: intent.catalog_generation(), + catalog_length: intent.catalog_length(), + catalog_digest: intent.catalog_digest(), + predecessor_catalog_digest: intent.predecessor_catalog_digest(), + inventory_digest: intent.inventory_digest(), + root_device_identity: intent.root_device_identity(), + root_mount_identity: intent.root_mount_identity(), + root_file_identity: intent.root_file_identity(), + target_definition_digest: intent.target_definition_digest(), + store_identifier: intent.store_identifier(), + }, digest: intent.digest(), - store_identifier: intent.store_identifier(), } } @@ -59,20 +71,65 @@ impl CanonicalStoreMigrationIntent { self.digest } + /// Returns the positive catalog generation named by the intent. + pub const fn catalog_generation(&self) -> CatalogGeneration { + self.fields.catalog_generation + } + + /// Returns the exact catalog byte length named by the intent. + pub const fn catalog_length(&self) -> CatalogLength { + self.fields.catalog_length + } + + /// Returns the catalog digest named by the intent. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.fields.catalog_digest + } + + /// Returns the generation-relative predecessor digest. + pub const fn predecessor_catalog_digest(&self) -> Option { + self.fields.predecessor_catalog_digest + } + + /// Returns the immutable-pool inventory digest named by the intent. + pub const fn inventory_digest(&self) -> ImmutablePoolInventoryDigest { + self.fields.inventory_digest + } + + /// Returns the serialized root device coordinate. + pub const fn root_device_identity(&self) -> StoreRootDeviceIdentity { + self.fields.root_device_identity + } + + /// Returns the serialized root mount coordinate. + pub const fn root_mount_identity(&self) -> StoreRootMountIdentity { + self.fields.root_mount_identity + } + + /// Returns the serialized root file coordinate. + pub const fn root_file_identity(&self) -> StoreRootFileIdentity { + self.fields.root_file_identity + } + + /// Returns the registered target format-definition digest. + pub const fn target_definition_digest(&self) -> StoreFormatDefinitionDigest { + self.fields.target_definition_digest + } + /// Returns the deterministic logical store identity. pub const fn store_identifier(&self) -> StoreIdentifier { - self.store_identifier + self.fields.store_identifier } pub(super) const fn admitted( encoded: [u8; migration_intent_format::ENCODED_LENGTH], + fields: StoreMigrationIntentFields, digest: StoreMigrationIntentDigest, - store_identifier: StoreIdentifier, ) -> Self { Self { encoded, + fields, digest, - store_identifier, } } } diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs index a461655..faa7572 100644 --- a/src/adapters/store_migration/migration_intent_encoder.rs +++ b/src/adapters/store_migration/migration_intent_encoder.rs @@ -1,5 +1,6 @@ //! This boundary module owns canonical migration-intent encoding. +use super::admitted_migration_intent::StoreMigrationIntentFields; use super::migration_intent_format::StoreIdentifierFields; use super::{ CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, @@ -41,7 +42,22 @@ pub(super) fn encode( write_preimage(preimage, &fields, roots, store_identifier); checksum_slot.copy_from_slice(&format::checksum(preimage)); let digest = format::digest(&encoded); - CanonicalStoreMigrationIntent::admitted(encoded, digest, store_identifier) + CanonicalStoreMigrationIntent::admitted( + encoded, + StoreMigrationIntentFields { + catalog_generation: fields.catalog_generation, + catalog_length: fields.catalog_length, + catalog_digest: fields.catalog_digest, + predecessor_catalog_digest: fields.predecessor_catalog_digest, + inventory_digest: fields.inventory_digest, + root_device_identity: roots.device, + root_mount_identity: roots.mount, + root_file_identity: roots.file, + target_definition_digest: fields.target_definition_digest, + store_identifier, + }, + digest, + ) } fn write_preimage( diff --git a/tests/store_migration_intent_encoding.rs b/tests/store_migration_intent_encoding.rs index 2158dbc..34a85a9 100644 --- a/tests/store_migration_intent_encoding.rs +++ b/tests/store_migration_intent_encoding.rs @@ -35,7 +35,7 @@ fn admitted_coordinates_reproduce_the_frozen_intent() -> Result<(), Box Result<(), Box, +) { + assert_eq!( + canonical.catalog_generation(), + admitted.catalog_generation() + ); + assert_eq!(canonical.catalog_length(), admitted.catalog_length()); + assert_eq!(canonical.catalog_digest(), admitted.catalog_digest()); + assert_eq!( + canonical.predecessor_catalog_digest(), + admitted.predecessor_catalog_digest() + ); + assert_eq!(canonical.inventory_digest(), admitted.inventory_digest()); + assert_eq!( + canonical.root_device_identity(), + admitted.root_device_identity() + ); + assert_eq!( + canonical.root_mount_identity(), + admitted.root_mount_identity() + ); + assert_eq!( + canonical.root_file_identity(), + admitted.root_file_identity() + ); + assert_eq!( + canonical.target_definition_digest(), + admitted.target_definition_digest() + ); + assert_eq!(canonical.store_identifier(), admitted.store_identifier()); +} + #[test] fn successor_intent_encodes_the_exact_predecessor() -> Result<(), Box> { let source_bytes = fixture::fixture_bytes()?; From 9b3c711eb933ec9eefcb264026820c92c0a4c3c0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 04:19:08 -0700 Subject: [PATCH 39/50] Add: Define store migration storage port --- CHANGELOG.md | 6 +- docs/formats/segment-store-v2/recovery.md | 6 +- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/store_migration.rs | 2 + .../store_migration/migration_storage.rs | 173 ++++++++++++++++++ src/lib.rs | 14 +- tests/store_migration_storage.rs | 87 +++++++++ .../recording_storage.rs | 146 +++++++++++++++ 8 files changed, 423 insertions(+), 15 deletions(-) create mode 100644 src/adapters/store_migration/migration_storage.rs create mode 100644 tests/store_migration_storage.rs create mode 100644 tests/store_migration_storage/recording_storage.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 783c217..24ee878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,9 @@ after its public API and format compatibility policies are established. exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` - freezes 21 transitions. Retention preflight combines - expected-generation planning with deterministic closure verification; - authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. + freezes 21 transitions behind explicit blocking storage capabilities. + Retention preflight combines expected-generation planning with deterministic + closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index a5e4444..6cefd88 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,9 +61,9 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` retains and reproduces typed intent coordinates; `CanonicalStoreMigrationReceipt` binds intent, marker, empty states, and complete synchronization; admitted record types verify both. -These record boundaries do not prove the named live inventory, physical root, -store version, or execution of filesystem migration. +`CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. +`StoreMigrationStorage` names current-state verification and all 21 blocking durability capabilities but does not prove the live inventory, physical root, +store version, execution, or recovery of filesystem migration. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e68ae63..e9bf0de 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,11 +30,11 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; mandatory verification capability in `tests/store_migration_storage.rs`; filesystem integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary in `tests/store_migration_phase.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary and matching storage capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 2515c97..e73c255 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -35,6 +35,7 @@ mod migration_receipt_encoder; mod migration_receipt_format; mod migration_receipt_initial_state; mod migration_record_bytes; +mod migration_storage; mod migration_synchronization_mask; mod store_identifier; mod store_root_identity; @@ -61,6 +62,7 @@ pub use migration_inventory_error::StoreMigrationInventoryError; pub use migration_inventory_hasher::StoreMigrationInventoryHasher; pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; +pub use migration_storage::StoreMigrationStorage; pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; pub use store_root_identity::{ diff --git a/src/adapters/store_migration/migration_storage.rs b/src/adapters/store_migration/migration_storage.rs new file mode 100644 index 0000000..fed5ec8 --- /dev/null +++ b/src/adapters/store_migration/migration_storage.rs @@ -0,0 +1,173 @@ +//! This boundary module owns blocking store-migration durability capabilities. + +use std::io; + +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, +}; + +/// Blocking storage capabilities for one writer-locked version-2 migration. +/// +/// An implementation must retain exclusive writer authority and one pinned +/// store root for the complete operation. After `verify_current`, each method +/// corresponds to one [`StoreMigrationPhase`](super::StoreMigrationPhase) and +/// must not report success before its durability and verification obligations +/// are complete. +pub trait StoreMigrationStorage { + /// Revalidates the exact version-1 authority bound by `intent`. + /// + /// This must verify the catalog coordinates, inventory, physical root, + /// version-1 format, and absence of migration or version-2 artifacts. + /// + /// # Errors + /// + /// Returns the exact current-state or recovery-required refusal. + fn verify_current(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; + + /// Exclusively creates and completely writes `migration.intent.next`. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_intent_stage(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; + + /// Synchronizes the complete intent stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_intent_stage(&mut self) -> io::Result<()>; + + /// Links and exactly verifies canonical `migration.intent`. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_intent(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; + + /// Synchronizes the store root after the intent link. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_intent(&mut self) -> io::Result<()>; + + /// Removes only the retained intent stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_intent_stage(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after intent-stage cleanup. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()>; + + /// Creates or exactly admits persistent `reader.lock`. + /// + /// # Errors + /// + /// Returns the exact creation, open, or verification failure. + fn admit_reader_fence(&mut self) -> io::Result<()>; + + /// Creates or exactly admits the complete version-2 directory prefix. + /// + /// # Errors + /// + /// Returns the exact namespace creation or admission failure. + fn admit_namespace_prefix(&mut self) -> io::Result<()>; + + /// Synchronizes created namespaces and the store root. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_namespace(&mut self) -> io::Result<()>; + + /// Exclusively creates and completely writes `FORMAT.next`. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_marker_stage(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()>; + + /// Synchronizes the complete marker stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_marker_stage(&mut self) -> io::Result<()>; + + /// Links and exactly verifies canonical `FORMAT`. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_marker(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()>; + + /// Synchronizes the store root after the marker link. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_marker(&mut self) -> io::Result<()>; + + /// Removes only the retained marker stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_marker_stage(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after marker-stage cleanup. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()>; + + /// Exclusively creates and completely writes `migration.receipt.next`. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_receipt_stage(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()>; + + /// Synchronizes the complete receipt stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_receipt_stage(&mut self) -> io::Result<()>; + + /// Links and exactly verifies canonical `migration.receipt`. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_receipt(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()>; + + /// Synchronizes the store root after the receipt link. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_receipt(&mut self) -> io::Result<()>; + + /// Removes only the retained receipt stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_receipt_stage(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after receipt-stage cleanup. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index e6a551a..5446185 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,13 +114,13 @@ pub use adapters::{ StoreMigrationIntentDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, - WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, - classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, execute_recovery_next_head_finalization, - execute_recovery_segment_resume, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, - plan_recovery_next_head_finalization, plan_recovery_segment_resume, + StoreMigrationStorage, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, + assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, + classify_recovery_next_head_stage, classify_recovery_segment_stage, + execute_recovery_next_head_finalization, execute_recovery_segment_resume, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; diff --git a/tests/store_migration_storage.rs b/tests/store_migration_storage.rs new file mode 100644 index 0000000..a1b3142 --- /dev/null +++ b/tests/store_migration_storage.rs @@ -0,0 +1,87 @@ +//! Version-2 store-migration storage capability laws. + +#[path = "store_migration_storage/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; +use std::io; + +use keep::{ + AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, + CanonicalStoreMigrationReceipt, StoreMigrationPhase, StoreMigrationStorage, +}; +use recording_storage::RecordingStorage; + +const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); + +#[test] +fn storage_port_names_every_migration_phase() -> Result<(), Box> { + let intent_bytes = support::decode_hex(INTENT.trim_end())?; + let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let intent = CanonicalStoreMigrationIntent::from_admitted(&admitted); + let marker = CanonicalStoreFormatMarker::version_two(); + let receipt = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); + let mut storage = RecordingStorage::default(); + + exercise_storage(&mut storage, &intent, &marker, &receipt)?; + + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), StoreMigrationPhase::ALL); + Ok(()) +} + +fn exercise_storage( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, + receipt: &CanonicalStoreMigrationReceipt, +) -> io::Result<()> { + storage.verify_current(intent)?; + exercise_intent(storage, intent)?; + exercise_namespace(storage)?; + exercise_marker(storage, marker)?; + exercise_receipt(storage, receipt) +} + +fn exercise_intent( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, +) -> io::Result<()> { + storage.write_intent_stage(intent)?; + storage.synchronize_intent_stage()?; + storage.link_intent(intent)?; + storage.synchronize_root_after_intent()?; + storage.remove_intent_stage()?; + storage.synchronize_root_after_intent_cleanup() +} + +fn exercise_namespace(storage: &mut impl StoreMigrationStorage) -> io::Result<()> { + storage.admit_reader_fence()?; + storage.admit_namespace_prefix()?; + storage.synchronize_root_after_namespace() +} + +fn exercise_marker( + storage: &mut impl StoreMigrationStorage, + marker: &CanonicalStoreFormatMarker, +) -> io::Result<()> { + storage.write_marker_stage(marker)?; + storage.synchronize_marker_stage()?; + storage.link_marker(marker)?; + storage.synchronize_root_after_marker()?; + storage.remove_marker_stage()?; + storage.synchronize_root_after_marker_cleanup() +} + +fn exercise_receipt( + storage: &mut impl StoreMigrationStorage, + receipt: &CanonicalStoreMigrationReceipt, +) -> io::Result<()> { + storage.write_receipt_stage(receipt)?; + storage.synchronize_receipt_stage()?; + storage.link_receipt(receipt)?; + storage.synchronize_root_after_receipt()?; + storage.remove_receipt_stage()?; + storage.synchronize_root_after_receipt_cleanup() +} diff --git a/tests/store_migration_storage/recording_storage.rs b/tests/store_migration_storage/recording_storage.rs new file mode 100644 index 0000000..277d1b3 --- /dev/null +++ b/tests/store_migration_storage/recording_storage.rs @@ -0,0 +1,146 @@ +//! This module owns the store-migration storage test double. + +use std::io; + +use keep::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + StoreMigrationPhase, StoreMigrationStorage, +}; + +#[derive(Default)] +/// Storage port that records every attempted migration phase. +pub struct RecordingStorage { + observed: Vec, + verification_count: usize, +} + +impl RecordingStorage { + /// Returns attempted migration phases in call order. + pub fn observed(&self) -> &[StoreMigrationPhase] { + &self.observed + } + + /// Returns the number of current-state verification attempts. + pub const fn verification_count(&self) -> usize { + self.verification_count + } + + fn record(&mut self, phase: StoreMigrationPhase) { + self.observed.push(phase); + } +} + +impl StoreMigrationStorage for RecordingStorage { + fn verify_current(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + self.verification_count = self + .verification_count + .checked_add(1) + .ok_or_else(|| io::Error::other("verification count overflow"))?; + Ok(()) + } + + fn write_intent_stage(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + self.record(StoreMigrationPhase::WriteIntentStage); + Ok(()) + } + + fn synchronize_intent_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeIntentStage); + Ok(()) + } + + fn link_intent(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + self.record(StoreMigrationPhase::LinkIntent); + Ok(()) + } + + fn synchronize_root_after_intent(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterIntent); + Ok(()) + } + + fn remove_intent_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::RemoveIntentStage); + Ok(()) + } + + fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup); + Ok(()) + } + + fn admit_reader_fence(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::AdmitReaderFence); + Ok(()) + } + + fn admit_namespace_prefix(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::AdmitNamespacePrefix); + Ok(()) + } + + fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace); + Ok(()) + } + + fn write_marker_stage(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { + self.record(StoreMigrationPhase::WriteMarkerStage); + Ok(()) + } + + fn synchronize_marker_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeMarkerStage); + Ok(()) + } + + fn link_marker(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { + self.record(StoreMigrationPhase::LinkMarker); + Ok(()) + } + + fn synchronize_root_after_marker(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterMarker); + Ok(()) + } + + fn remove_marker_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::RemoveMarkerStage); + Ok(()) + } + + fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup); + Ok(()) + } + + fn write_receipt_stage(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { + self.record(StoreMigrationPhase::WriteReceiptStage); + Ok(()) + } + + fn synchronize_receipt_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeReceiptStage); + Ok(()) + } + + fn link_receipt(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { + self.record(StoreMigrationPhase::LinkReceipt); + Ok(()) + } + + fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt); + Ok(()) + } + + fn remove_receipt_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::RemoveReceiptStage); + Ok(()) + } + + fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup); + Ok(()) + } +} From 3fc96642894115c3fef4928be9159d8657c7180b Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 04:33:26 -0700 Subject: [PATCH 40/50] Add: Execute ordered store migration --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 4 +- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/store_migration.rs | 4 + .../store_migration/migration_error.rs | 45 ++++++ .../store_migration/migration_execution.rs | 136 ++++++++++++++++++ src/lib.rs | 24 ++-- tests/store_migration_execution.rs | 97 +++++++++++++ .../recording_storage.rs | 91 ++++++------ 9 files changed, 346 insertions(+), 61 deletions(-) create mode 100644 src/adapters/store_migration/migration_error.rs create mode 100644 src/adapters/store_migration/migration_execution.rs create mode 100644 tests/store_migration_execution.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 24ee878..e907f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ after its public API and format compatibility policies are established. exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` - freezes 21 transitions behind explicit blocking storage capabilities. + freezes 21 transitions with explicit storage and verification-first execution. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 6cefd88..0946883 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -62,8 +62,8 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. `CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. -`StoreMigrationStorage` names current-state verification and all 21 blocking durability capabilities but does not prove the live inventory, physical root, -store version, execution, or recovery of filesystem migration. +`StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. +These boundaries do not prove a filesystem implementation or partial-prefix recovery. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e9bf0de..551f526 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,11 +30,11 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; mandatory verification capability in `tests/store_migration_storage.rs`; filesystem integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; verification-first execution in `tests/store_migration_execution.rs`; filesystem integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary and matching storage capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index e73c255..df65667 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -16,6 +16,8 @@ mod format_marker_encoder; mod immutable_pool_inventory_digest; mod initial_gc_state_digest; mod initial_retention_state_digest; +mod migration_error; +mod migration_execution; mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; @@ -53,6 +55,8 @@ pub use format_marker_digest::StoreFormatMarkerDigest; pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; pub use initial_gc_state_digest::InitialGcStateDigest; pub use initial_retention_state_digest::InitialRetentionStateDigest; +pub use migration_error::StoreMigrationError; +pub use migration_execution::execute_store_migration; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; pub use migration_inventory_entry::StoreMigrationInventoryEntry; diff --git a/src/adapters/store_migration/migration_error.rs b/src/adapters/store_migration/migration_error.rs new file mode 100644 index 0000000..9fc39e2 --- /dev/null +++ b/src/adapters/store_migration/migration_error.rs @@ -0,0 +1,45 @@ +//! This boundary module owns ordered store-migration execution failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::StoreMigrationPhase; + +/// Failure before or during ordered version-2 store migration. +#[derive(Debug)] +pub enum StoreMigrationError { + /// Current version-1 authority could not be revalidated before mutation. + CurrentVerification { + /// Preserved storage refusal. + source: io::Error, + }, + /// One exact durability phase failed. + Storage { + /// Phase attempted when storage refused. + phase: StoreMigrationPhase, + /// Preserved storage refusal. + source: io::Error, + }, +} + +impl fmt::Display for StoreMigrationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentVerification { .. } => { + formatter.write_str("store-migration authority verification failed") + } + Self::Storage { phase, .. } => { + write!(formatter, "store-migration phase {phase} failed") + } + } + } +} + +impl Error for StoreMigrationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::CurrentVerification { source } | Self::Storage { source, .. } => Some(source), + } + } +} diff --git a/src/adapters/store_migration/migration_execution.rs b/src/adapters/store_migration/migration_execution.rs new file mode 100644 index 0000000..065fdd3 --- /dev/null +++ b/src/adapters/store_migration/migration_execution.rs @@ -0,0 +1,136 @@ +//! This boundary module owns ordered version-2 store-migration execution. + +use std::io; + +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + StoreMigrationError, StoreMigrationPhase, StoreMigrationStorage, +}; + +/// Executes one version-2 migration under revalidated version-1 authority. +/// +/// The returned receipt exists only after all canonical artifacts are visible, +/// all retained stages are removed, and final store-root cleanup is synchronized. +/// +/// # Errors +/// +/// Returns [`StoreMigrationError`] for current-state revalidation or the exact +/// failed durability phase. Failure returns no receipt. +pub fn execute_store_migration( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, +) -> Result { + storage + .verify_current(intent) + .map_err(|source| StoreMigrationError::CurrentVerification { source })?; + let marker = CanonicalStoreFormatMarker::version_two(); + let receipt = CanonicalStoreMigrationReceipt::from_canonical(intent, &marker); + execute_intent(storage, intent)?; + execute_namespace(storage)?; + execute_marker(storage, &marker)?; + execute_receipt(storage, &receipt)?; + Ok(receipt) +} + +fn execute_intent( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, +) -> Result<(), StoreMigrationError> { + require( + storage.write_intent_stage(intent), + StoreMigrationPhase::WriteIntentStage, + )?; + require( + storage.synchronize_intent_stage(), + StoreMigrationPhase::SynchronizeIntentStage, + )?; + require(storage.link_intent(intent), StoreMigrationPhase::LinkIntent)?; + require( + storage.synchronize_root_after_intent(), + StoreMigrationPhase::SynchronizeRootAfterIntent, + )?; + require( + storage.remove_intent_stage(), + StoreMigrationPhase::RemoveIntentStage, + )?; + require( + storage.synchronize_root_after_intent_cleanup(), + StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, + ) +} + +fn execute_namespace(storage: &mut impl StoreMigrationStorage) -> Result<(), StoreMigrationError> { + require( + storage.admit_reader_fence(), + StoreMigrationPhase::AdmitReaderFence, + )?; + require( + storage.admit_namespace_prefix(), + StoreMigrationPhase::AdmitNamespacePrefix, + )?; + require( + storage.synchronize_root_after_namespace(), + StoreMigrationPhase::SynchronizeRootAfterNamespace, + ) +} + +fn execute_marker( + storage: &mut impl StoreMigrationStorage, + marker: &CanonicalStoreFormatMarker, +) -> Result<(), StoreMigrationError> { + require( + storage.write_marker_stage(marker), + StoreMigrationPhase::WriteMarkerStage, + )?; + require( + storage.synchronize_marker_stage(), + StoreMigrationPhase::SynchronizeMarkerStage, + )?; + require(storage.link_marker(marker), StoreMigrationPhase::LinkMarker)?; + require( + storage.synchronize_root_after_marker(), + StoreMigrationPhase::SynchronizeRootAfterMarker, + )?; + require( + storage.remove_marker_stage(), + StoreMigrationPhase::RemoveMarkerStage, + )?; + require( + storage.synchronize_root_after_marker_cleanup(), + StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, + ) +} + +fn execute_receipt( + storage: &mut impl StoreMigrationStorage, + receipt: &CanonicalStoreMigrationReceipt, +) -> Result<(), StoreMigrationError> { + require( + storage.write_receipt_stage(receipt), + StoreMigrationPhase::WriteReceiptStage, + )?; + require( + storage.synchronize_receipt_stage(), + StoreMigrationPhase::SynchronizeReceiptStage, + )?; + require( + storage.link_receipt(receipt), + StoreMigrationPhase::LinkReceipt, + )?; + require( + storage.synchronize_root_after_receipt(), + StoreMigrationPhase::SynchronizeRootAfterReceipt, + )?; + require( + storage.remove_receipt_stage(), + StoreMigrationPhase::RemoveReceiptStage, + )?; + require( + storage.synchronize_root_after_receipt_cleanup(), + StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup, + ) +} + +fn require(result: io::Result, phase: StoreMigrationPhase) -> Result { + result.map_err(|source| StoreMigrationError::Storage { phase, source }) +} diff --git a/src/lib.rs b/src/lib.rs index 5446185..2c0f2ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,19 +110,19 @@ pub use adapters::{ SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, - StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, - StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, - StoreMigrationStorage, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, - WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, - assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, - classify_recovery_next_head_stage, classify_recovery_segment_stage, + StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationError, + StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, StoreMigrationInventoryEntry, + StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, + StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, + StoreMigrationReceiptDecodeError, StoreMigrationStorage, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, - plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, - read_recovery_inventory, + execute_recovery_stage_completion, execute_recovery_stage_discard, execute_store_migration, + fingerprint_recovery_stage, initialize_store, plan_recovery_next_head_finalization, + plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, + publish_catalog_generation, read_recovery_inventory, }; pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, diff --git a/tests/store_migration_execution.rs b/tests/store_migration_execution.rs new file mode 100644 index 0000000..1c7f015 --- /dev/null +++ b/tests/store_migration_execution.rs @@ -0,0 +1,97 @@ +//! Version-2 store-migration execution laws. + +#[path = "store_migration_storage/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; +use std::io; + +use keep::{ + AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, + CanonicalStoreMigrationReceipt, StoreMigrationError, StoreMigrationPhase, + execute_store_migration, +}; +use recording_storage::RecordingStorage; + +const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); + +#[test] +fn migration_executes_every_phase_before_returning_its_receipt() -> Result<(), Box> { + let (intent, marker) = artifacts()?; + let expected = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); + let mut storage = RecordingStorage::default(); + + let receipt = execute_store_migration(&mut storage, &intent)?; + + assert_eq!(receipt, expected); + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), StoreMigrationPhase::ALL); + Ok(()) +} + +#[test] +fn current_verification_refuses_before_every_migration_phase() -> Result<(), Box> { + let (intent, _marker) = artifacts()?; + let mut storage = RecordingStorage::verification_failure(); + + let Err(error) = execute_store_migration(&mut storage, &intent) else { + return Err("current-state refusal unexpectedly admitted migration".into()); + }; + + match error { + StoreMigrationError::CurrentVerification { source } => { + assert_eq!(source.kind(), io::ErrorKind::PermissionDenied); + } + other @ StoreMigrationError::Storage { .. } => { + return Err(format!("unexpected migration error: {other}").into()); + } + } + assert_eq!(storage.verification_count(), 1); + assert!(storage.observed().is_empty()); + Ok(()) +} + +#[test] +fn every_phase_failure_stops_before_all_later_mutation() -> Result<(), Box> { + let (intent, _marker) = artifacts()?; + for (index, phase) in StoreMigrationPhase::ALL.into_iter().enumerate() { + let mut storage = RecordingStorage::failing_at(phase); + let Err(error) = execute_store_migration(&mut storage, &intent) else { + return Err("injected refusal unexpectedly admitted migration".into()); + }; + assert_storage_error(error, phase)?; + assert_eq!(storage.verification_count(), 1); + let expected = StoreMigrationPhase::ALL + .get(..=index) + .ok_or("migration phase prefix is out of bounds")?; + assert_eq!(storage.observed(), expected); + } + Ok(()) +} + +fn assert_storage_error( + error: StoreMigrationError, + expected_phase: StoreMigrationPhase, +) -> Result<(), Box> { + match error { + StoreMigrationError::Storage { phase, source } => { + assert_eq!(phase, expected_phase); + assert_eq!(source.kind(), io::ErrorKind::Other); + Ok(()) + } + other @ StoreMigrationError::CurrentVerification { .. } => { + Err(format!("unexpected migration error: {other}").into()) + } + } +} + +fn artifacts() -> Result<(CanonicalStoreMigrationIntent, CanonicalStoreFormatMarker), Box> +{ + let intent_bytes = support::decode_hex(INTENT.trim_end())?; + let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + Ok(( + CanonicalStoreMigrationIntent::from_admitted(&admitted), + CanonicalStoreFormatMarker::version_two(), + )) +} diff --git a/tests/store_migration_storage/recording_storage.rs b/tests/store_migration_storage/recording_storage.rs index 277d1b3..d3dbf25 100644 --- a/tests/store_migration_storage/recording_storage.rs +++ b/tests/store_migration_storage/recording_storage.rs @@ -12,9 +12,27 @@ use keep::{ pub struct RecordingStorage { observed: Vec, verification_count: usize, + fail_at: Option, + verification_failure: Option, } impl RecordingStorage { + /// Creates storage that refuses at one exact migration phase. + pub fn failing_at(phase: StoreMigrationPhase) -> Self { + Self { + fail_at: Some(phase), + ..Self::default() + } + } + + /// Creates storage that refuses current-state verification. + pub fn verification_failure() -> Self { + Self { + verification_failure: Some(io::ErrorKind::PermissionDenied), + ..Self::default() + } + } + /// Returns attempted migration phases in call order. pub fn observed(&self) -> &[StoreMigrationPhase] { &self.observed @@ -25,8 +43,13 @@ impl RecordingStorage { self.verification_count } - fn record(&mut self, phase: StoreMigrationPhase) { + fn record(&mut self, phase: StoreMigrationPhase) -> io::Result<()> { self.observed.push(phase); + if self.fail_at == Some(phase) { + Err(io::Error::other("injected store-migration failure")) + } else { + Ok(()) + } } } @@ -36,111 +59,91 @@ impl StoreMigrationStorage for RecordingStorage { .verification_count .checked_add(1) .ok_or_else(|| io::Error::other("verification count overflow"))?; - Ok(()) + self.verification_failure + .map_or(Ok(()), |kind| Err(kind.into())) } fn write_intent_stage(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteIntentStage); - Ok(()) + self.record(StoreMigrationPhase::WriteIntentStage) } fn synchronize_intent_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeIntentStage); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeIntentStage) } fn link_intent(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkIntent); - Ok(()) + self.record(StoreMigrationPhase::LinkIntent) } fn synchronize_root_after_intent(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterIntent); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterIntent) } fn remove_intent_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveIntentStage); - Ok(()) + self.record(StoreMigrationPhase::RemoveIntentStage) } fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup) } fn admit_reader_fence(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::AdmitReaderFence); - Ok(()) + self.record(StoreMigrationPhase::AdmitReaderFence) } fn admit_namespace_prefix(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::AdmitNamespacePrefix); - Ok(()) + self.record(StoreMigrationPhase::AdmitNamespacePrefix) } fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace) } fn write_marker_stage(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteMarkerStage); - Ok(()) + self.record(StoreMigrationPhase::WriteMarkerStage) } fn synchronize_marker_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeMarkerStage); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeMarkerStage) } fn link_marker(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkMarker); - Ok(()) + self.record(StoreMigrationPhase::LinkMarker) } fn synchronize_root_after_marker(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterMarker); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterMarker) } fn remove_marker_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveMarkerStage); - Ok(()) + self.record(StoreMigrationPhase::RemoveMarkerStage) } fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup) } fn write_receipt_stage(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteReceiptStage); - Ok(()) + self.record(StoreMigrationPhase::WriteReceiptStage) } fn synchronize_receipt_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeReceiptStage); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeReceiptStage) } fn link_receipt(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkReceipt); - Ok(()) + self.record(StoreMigrationPhase::LinkReceipt) } fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt) } fn remove_receipt_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveReceiptStage); - Ok(()) + self.record(StoreMigrationPhase::RemoveReceiptStage) } fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup) } } From 117a92acffb9e02dba7eebaa0a258803ec978f11 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 05:52:48 -0700 Subject: [PATCH 41/50] Add: Inventory filesystem migration pools --- CHANGELOG.md | 6 +- .../segment-store-v2/migration-inventory.md | 10 +- docs/formats/segment-store-v2/recovery.md | 2 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 35 ++++ .../filesystem_inventory_catalog_errors.rs | 107 ++++++++++ .../filesystem_inventory_catalogs.rs | 178 +++++++++++++++++ ...system_inventory_catalogs_refusal_tests.rs | 184 ++++++++++++++++++ ...esystem_inventory_catalogs_test_fixture.rs | 109 +++++++++++ .../filesystem_inventory_catalogs_tests.rs | 35 ++++ .../filesystem_inventory_directory.rs | 96 +++++++++ .../filesystem_inventory_error.rs | 155 +++++++++++++++ .../filesystem_inventory_error_display.rs | 147 ++++++++++++++ .../filesystem_inventory_file.rs | 128 ++++++++++++ .../filesystem_inventory_file_tests.rs | 67 +++++++ .../filesystem_inventory_names.rs | 61 ++++++ .../filesystem_inventory_names_tests.rs | 41 ++++ .../filesystem_inventory_reader.rs | 168 ++++++++++++++++ .../filesystem_inventory_reader_tests.rs | 80 ++++++++ .../filesystem_inventory_segments.rs | 183 +++++++++++++++++ ...system_inventory_segments_refusal_tests.rs | 156 +++++++++++++++ ...esystem_inventory_segments_test_fixture.rs | 79 ++++++++ .../filesystem_inventory_segments_tests.rs | 36 ++++ .../migration_catalog_admission.rs | 106 ++++++++++ .../store_migration/migration_catalog_plan.rs | 51 +++++ .../migration_catalog_records.rs | 93 +++++++++ .../migration_inventory_entry.rs | 11 ++ src/lib.rs | 10 +- 28 files changed, 2326 insertions(+), 10 deletions(-) create mode 100644 src/adapters/store_migration/filesystem_inventory_catalog_errors.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_directory.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_error.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_error_display.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_file.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_file_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_names.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_names_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_reader.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_reader_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments_tests.rs create mode 100644 src/adapters/store_migration/migration_catalog_admission.rs create mode 100644 src/adapters/store_migration/migration_catalog_plan.rs create mode 100644 src/adapters/store_migration/migration_catalog_records.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e907f1e..adf1891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,10 @@ after its public API and format compatibility policies are established. - Version-2 marker, typed canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all - three decoders, streamed inventory is bounded, and `StoreMigrationPhase` - freezes 21 transitions with explicit storage and verification-first execution. + three decoders, streamed inventory is bounded, writer-locked filesystem + inventory completely admits every immutable-pool artifact, and + `StoreMigrationPhase` freezes 21 transitions with explicit storage and + verification-first execution. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md index f970abb..09c233c 100644 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -51,5 +51,11 @@ the version-2 corpus `StoreMigrationInventoryEntry` derives canonical bytes only from admitted artifacts. `StoreMigrationInventoryHasher` requires the bounded entry count before streaming, retains only the preceding entry, refuses duplicate or -out-of-order evidence, and reproduces the frozen digest. Capability-relative -filesystem inventory and mutation revalidation remain unimplemented. +out-of-order evidence, and reproduces the frozen digest. + +`FilesystemStoreMigrationInventoryReader` retains exclusive writer authority +and pinned capabilities for both immutable pools. It inventories every regular +entry, including artifacts not reachable from the current publication head, +and reproduces the frozen digest without retaining every artifact body at +once. Migration-session integration that revalidates this inventory +immediately before the first namespace mutation remains in progress. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 0946883..b526f9b 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -63,7 +63,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. `CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. `StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. -These boundaries do not prove a filesystem implementation or partial-prefix recovery. +`FilesystemStoreMigrationInventoryReader` inventories every version-1 immutable artifact under retained writer authority and pinned pool capabilities; migration-session integration and partial-prefix recovery remain unimplemented. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 551f526..3ea5b2c 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,7 +30,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; verification-first execution in `tests/store_migration_execution.rs`; filesystem integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; verification-first execution in `tests/store_migration_execution.rs`; mutation-time integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index df65667..93cdfc9 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -7,6 +7,33 @@ mod canonical_format_marker; mod canonical_migration_intent; mod canonical_migration_receipt; mod empty_disposition_set_digest; +mod filesystem_inventory_catalog_errors; +mod filesystem_inventory_catalogs; +#[cfg(test)] +mod filesystem_inventory_catalogs_refusal_tests; +#[cfg(test)] +mod filesystem_inventory_catalogs_test_fixture; +#[cfg(test)] +mod filesystem_inventory_catalogs_tests; +mod filesystem_inventory_directory; +mod filesystem_inventory_error; +mod filesystem_inventory_error_display; +mod filesystem_inventory_file; +#[cfg(test)] +mod filesystem_inventory_file_tests; +mod filesystem_inventory_names; +#[cfg(test)] +mod filesystem_inventory_names_tests; +mod filesystem_inventory_reader; +#[cfg(test)] +mod filesystem_inventory_reader_tests; +mod filesystem_inventory_segments; +#[cfg(test)] +mod filesystem_inventory_segments_refusal_tests; +#[cfg(test)] +mod filesystem_inventory_segments_test_fixture; +#[cfg(test)] +mod filesystem_inventory_segments_tests; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; @@ -16,6 +43,9 @@ mod format_marker_encoder; mod immutable_pool_inventory_digest; mod initial_gc_state_digest; mod initial_retention_state_digest; +mod migration_catalog_admission; +mod migration_catalog_plan; +mod migration_catalog_records; mod migration_error; mod migration_execution; mod migration_intent_decode_error; @@ -49,6 +79,11 @@ pub use canonical_format_marker::CanonicalStoreFormatMarker; pub use canonical_migration_intent::CanonicalStoreMigrationIntent; pub use canonical_migration_receipt::CanonicalStoreMigrationReceipt; pub use empty_disposition_set_digest::EmptyDispositionSetDigest; +pub use filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +pub use filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs b/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs new file mode 100644 index 0000000..7a579e2 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs @@ -0,0 +1,107 @@ +//! This module owns filesystem migration catalog-inventory error translation. + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_file::FilesystemInventoryFileError; +use super::migration_catalog_admission::MigrationCatalogAdmissionError; +use crate::adapters::{ + CatalogAdmissionError, CatalogRestartError, RecoveryEntryName, SegmentDigest, +}; + +const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; + +pub(super) fn admission( + name: &RecoveryEntryName, + source: MigrationCatalogAdmissionError, +) -> FilesystemMigrationInventoryError { + match source { + MigrationCatalogAdmissionError::Catalog(source) => catalog_admission(name, *source), + MigrationCatalogAdmissionError::SegmentSource { digest, source } => { + referenced_segment(digest, source) + } + MigrationCatalogAdmissionError::SegmentCoordinate { expected, observed } => { + FilesystemMigrationInventoryError::ReferencedSegment { + digest: expected, + source: Box::new(CatalogRestartError::SegmentCoordinate { expected, observed }), + } + } + } +} + +fn catalog_admission( + name: &RecoveryEntryName, + source: CatalogAdmissionError, +) -> FilesystemMigrationInventoryError { + match source { + CatalogAdmissionError::MissingSegment { digest } => { + FilesystemMigrationInventoryError::ReferencedSegment { + digest, + source: Box::new(CatalogRestartError::CatalogAdmission { + source: Box::new(CatalogAdmissionError::MissingSegment { digest }), + }), + } + } + CatalogAdmissionError::Segment { digest, source } => { + FilesystemMigrationInventoryError::ReferencedSegment { + digest, + source: Box::new(CatalogRestartError::Segment { + expected: digest, + source, + }), + } + } + source => artifact( + name, + CatalogRestartError::CatalogAdmission { + source: Box::new(source), + }, + ), + } +} + +pub(super) fn catalog_file( + name: &RecoveryEntryName, + source: FilesystemInventoryFileError, +) -> FilesystemMigrationInventoryError { + match source { + FilesystemInventoryFileError::Artifact(source) => { + FilesystemMigrationInventoryError::Artifact { + pool: POOL, + name: name.clone(), + source, + } + } + FilesystemInventoryFileError::Changed => { + FilesystemMigrationInventoryError::ArtifactChanged { + pool: POOL, + name: name.clone(), + } + } + } +} + +fn referenced_segment( + digest: SegmentDigest, + source: FilesystemInventoryFileError, +) -> FilesystemMigrationInventoryError { + match source { + FilesystemInventoryFileError::Artifact(source) => { + FilesystemMigrationInventoryError::ReferencedSegment { digest, source } + } + FilesystemInventoryFileError::Changed => { + FilesystemMigrationInventoryError::ReferencedSegmentChanged { digest } + } + } +} + +pub(super) fn artifact( + name: &RecoveryEntryName, + source: CatalogRestartError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Artifact { + pool: POOL, + name: name.clone(), + source: Box::new(source), + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs.rs b/src/adapters/store_migration/filesystem_inventory_catalogs.rs new file mode 100644 index 0000000..147a608 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs.rs @@ -0,0 +1,178 @@ +//! This module owns complete filesystem migration catalog-pool admission. + +use std::collections::TryReserveError; + +use cap_std::fs::Dir; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_catalog_errors; +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_file::{ + self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, +}; +use super::filesystem_inventory_names; +use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; +use super::migration_catalog_admission::{self, MigrationSegmentLoadError}; +use crate::CatalogLength; +use crate::adapters::{ + CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, ChecksummedCatalog, + RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, recovery_pool_name, +}; + +const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; + +pub(super) struct FilesystemMigrationCatalogInventory { + entries: Vec, + names: Vec, + remaining: u32, +} + +impl FilesystemMigrationCatalogInventory { + pub(super) fn entries(&self) -> &[StoreMigrationInventoryEntry] { + &self.entries + } + + pub(super) const fn len(&self) -> usize { + self.entries.len() + } + + pub(super) fn verify_names( + &self, + directory: &Dir, + ) -> Result<(), FilesystemMigrationInventoryError> { + filesystem_inventory_names::verify(directory, POOL, self.remaining, &self.names) + } +} + +pub(super) fn read( + catalogs: &Dir, + segments: &Dir, + admitted_segments: &FilesystemMigrationSegmentInventory, + remaining: u32, + policy: SegmentReadPolicy, +) -> Result { + let names = filesystem_inventory_names::read(catalogs, POOL, remaining)?; + let capacity = names.len(); + let entry_count = u64::try_from(capacity) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool: POOL })?; + let mut entries = reserve(capacity, entry_count)?; + for name in &names { + entries.push(admit(catalogs, segments, admitted_segments, name, policy)?); + } + entries.sort_unstable(); + Ok(FilesystemMigrationCatalogInventory { + entries, + names, + remaining, + }) +} + +fn reserve( + capacity: usize, + entry_count: u64, +) -> Result, FilesystemMigrationInventoryError> { + let mut values = Vec::new(); + values + .try_reserve_exact(capacity) + .map_err(|source| allocation(entry_count, source))?; + Ok(values) +} + +const fn allocation( + entry_count: u64, + source: TryReserveError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Allocation { + pool: POOL, + entry_count, + source, + } +} + +fn admit( + catalogs: &Dir, + segments: &Dir, + admitted_segments: &FilesystemMigrationSegmentInventory, + name: &RecoveryEntryName, + policy: SegmentReadPolicy, +) -> Result { + let (generation, digest) = recovery_pool_name::catalog(name).map_err(|source| { + FilesystemMigrationInventoryError::Name { + pool: POOL, + name: name.clone(), + source, + } + })?; + let encoded = read_catalog(catalogs, name, generation, digest)?; + let catalog = ChecksummedCatalog::decode(&encoded).map_err(|source| { + filesystem_inventory_catalog_errors::artifact(name, CatalogRestartError::Catalog { source }) + })?; + require_coordinate(name, generation, digest, catalog)?; + let admitted = migration_catalog_admission::admit(catalog, policy, |required| { + load_segment(segments, admitted_segments, required) + }) + .map_err(|source| filesystem_inventory_catalog_errors::admission(name, source))?; + Ok(StoreMigrationInventoryEntry::from_migration_catalog( + &admitted, + )) +} + +fn read_catalog( + directory: &Dir, + name: &RecoveryEntryName, + generation: crate::CatalogGeneration, + digest: crate::CatalogDigest, +) -> Result, FilesystemMigrationInventoryError> { + let canonical_name = physical_pool_name::catalog(generation, digest); + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + CatalogRestartPhase::ReadCatalog, + CatalogLength::MAXIMUM.get(), + ); + filesystem_inventory_file::read(directory, &canonical_name, policy) + .map_err(|source| filesystem_inventory_catalog_errors::catalog_file(name, source)) +} + +fn require_coordinate( + name: &RecoveryEntryName, + generation: crate::CatalogGeneration, + digest: crate::CatalogDigest, + catalog: ChecksummedCatalog<'_>, +) -> Result<(), FilesystemMigrationInventoryError> { + if catalog.generation() == generation && catalog.digest() == digest { + return Ok(()); + } + Err(filesystem_inventory_catalog_errors::artifact( + name, + CatalogRestartError::CatalogCoordinate { + expected_generation: generation, + observed_generation: catalog.generation(), + expected_length: catalog.length(), + observed_length: catalog.length(), + expected_digest: digest, + observed_digest: catalog.digest(), + }, + )) +} + +fn load_segment( + directory: &Dir, + admitted: &FilesystemMigrationSegmentInventory, + digest: SegmentDigest, +) -> Result, MigrationSegmentLoadError> { + if !admitted.contains(digest) { + return Err(MigrationSegmentLoadError::Missing); + } + let name = physical_pool_name::segment(digest); + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Segment { digest }, + CatalogRestartPhase::OpenSegment, + CatalogRestartPhase::ReadSegment, + crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH, + ); + filesystem_inventory_file::read(directory, &name, policy) + .map_err(MigrationSegmentLoadError::Source) +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs new file mode 100644 index 0000000..8676d40 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs @@ -0,0 +1,184 @@ +//! Filesystem migration catalog-pool refusal and orphan laws. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_inventory_catalogs; +use super::filesystem_inventory_catalogs_test_fixture::{ + CatalogPoolFixture, empty_segment_bytes, maximum_policy, +}; +use super::filesystem_inventory_error::FilesystemMigrationInventoryError; +use super::filesystem_inventory_segments; +use crate::adapters::{ + AdmittedSegment, CatalogAdmissionError, CatalogRestartError, physical_pool_name, +}; + +#[test] +fn unrelated_orphan_segment_remains_in_exact_pool_inventory() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-orphan-segment")?; + let orphan_bytes = empty_segment_bytes()?; + let orphan = AdmittedSegment::decode(&orphan_bytes, maximum_policy())?; + fs::write( + fixture + .segments_path() + .join(physical_pool_name::segment(orphan.digest())), + &orphan_bytes, + )?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 2, maximum_policy())?; + + let catalogs = filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + )?; + assert_eq!(catalogs.entries().len(), 1); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +#[test] +fn catalog_missing_its_segment_refuses_inventory() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-missing-segment")?; + let missing = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?.digest(); + remove_fixture_segment(&fixture)?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 0, maximum_policy())?; + + let error = require_error(filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::ReferencedSegment { digest, source } = error else { + return Err(io::Error::other("missing segment returned wrong refusal").into()); + }; + assert_eq!(digest, missing); + assert!(matches!( + source.as_ref(), + CatalogRestartError::CatalogAdmission { + source + } if matches!( + source.as_ref(), + CatalogAdmissionError::MissingSegment { digest } if *digest == missing + ) + )); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +#[test] +fn segment_corruption_after_pool_admission_refuses_catalog() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-segment-corruption")?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; + let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + fs::write( + fixture + .segments_path() + .join(physical_pool_name::segment(segment.digest())), + b"corrupt", + )?; + + let error = require_error(filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + ))?; + let (digest, source) = match error { + FilesystemMigrationInventoryError::ReferencedSegment { digest, source } => (digest, source), + other => { + return Err(io::Error::other(format!( + "corrupt referenced segment returned wrong refusal: {other:?}" + )) + .into()); + } + }; + assert_eq!(digest, segment.digest()); + assert!(matches!( + source.as_ref(), + CatalogRestartError::Segment { + expected, + .. + } if *expected == segment.digest() + )); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +#[test] +fn valid_segment_substitution_names_the_referenced_coordinate() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-segment-substitution")?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; + let expected = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + let replacement_bytes = empty_segment_bytes()?; + let replacement = AdmittedSegment::decode(&replacement_bytes, maximum_policy())?; + assert_ne!(expected.digest(), replacement.digest()); + fs::write( + fixture + .segments_path() + .join(physical_pool_name::segment(expected.digest())), + &replacement_bytes, + )?; + + let error = require_error(filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::ReferencedSegment { digest, source } = error else { + return Err(io::Error::other("segment substitution returned wrong refusal").into()); + }; + assert_eq!(digest, expected.digest()); + assert!(matches!( + source.as_ref(), + CatalogRestartError::SegmentCoordinate { + expected: expected_digest, + observed + } if *expected_digest == expected.digest() && *observed == replacement.digest() + )); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +fn remove_fixture_segment(fixture: &CatalogPoolFixture) -> Result<(), Box> { + let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + fs::remove_file( + fixture + .segments_path() + .join(physical_pool_name::segment(segment.digest())), + )?; + Ok(()) +} + +fn require_error( + result: Result, +) -> Result { + result.map_or_else(Ok, |_value| { + Err(io::Error::other( + "filesystem migration catalog inventory unexpectedly succeeded", + )) + }) +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs new file mode 100644 index 0000000..c9a45c3 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs @@ -0,0 +1,109 @@ +//! Deterministic filesystem migration catalog-pool fixture. + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use crate::LayoutEntryLimit; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedSegment, ChecksummedCatalog, SegmentReadPolicy, SegmentRecordLimit, physical_pool_name, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); + +pub(super) struct CatalogPoolFixture { + sandbox: TestDirectory, + segment_bytes: Vec, + catalog_bytes: Vec, +} + +impl CatalogPoolFixture { + pub(super) fn create(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::create_dir(sandbox.path().join("segments"))?; + fs::create_dir(sandbox.path().join("catalogs"))?; + let segment_bytes = decode_hex(SEGMENT_HEX.trim())?; + let catalog_bytes = decode_hex(CATALOG_HEX.trim())?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + fs::write( + sandbox + .path() + .join("segments") + .join(physical_pool_name::segment(segment.digest())), + &segment_bytes, + )?; + fs::write( + sandbox + .path() + .join("catalogs") + .join(physical_pool_name::catalog( + catalog.generation(), + catalog.digest(), + )), + &catalog_bytes, + )?; + Ok(Self { + sandbox, + segment_bytes, + catalog_bytes, + }) + } + + pub(super) fn path(&self) -> &Path { + self.sandbox.path() + } + + pub(super) fn segments_path(&self) -> PathBuf { + self.path().join("segments") + } + + pub(super) fn catalogs_path(&self) -> PathBuf { + self.path().join("catalogs") + } + + pub(super) fn open_segments(&self) -> Result> { + Ok(Dir::open_ambient_dir( + self.segments_path(), + ambient_authority(), + )?) + } + + pub(super) fn open_catalogs(&self) -> Result> { + Ok(Dir::open_ambient_dir( + self.catalogs_path(), + ambient_authority(), + )?) + } + + pub(super) fn segment_bytes(&self) -> &[u8] { + &self.segment_bytes + } + + pub(super) fn catalog_bytes(&self) -> &[u8] { + &self.catalog_bytes + } + + pub(super) fn remove(self) -> Result<(), Box> { + self.sandbox.remove()?; + Ok(()) + } +} + +pub(super) const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +pub(super) fn empty_segment_bytes() -> Result, Box> { + Ok(decode_hex(EMPTY_SEGMENT_HEX.trim())?) +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs new file mode 100644 index 0000000..08aff5b --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs @@ -0,0 +1,35 @@ +//! Filesystem migration catalog-pool admission laws. + +use std::error::Error; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_catalogs; +use super::filesystem_inventory_catalogs_test_fixture::{CatalogPoolFixture, maximum_policy}; +use super::filesystem_inventory_segments; +use crate::adapters::{AdmittedSegment, ChecksummedCatalog}; + +#[test] +fn every_catalog_binds_exact_pool_segment_records() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-inventory")?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; + let catalogs = filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + )?; + let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + let admitted = ChecksummedCatalog::decode(fixture.catalog_bytes())?.admit(&[segment])?; + + assert_eq!( + catalogs.entries(), + &[StoreMigrationInventoryEntry::from_catalog(&admitted)] + ); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_inventory_directory.rs b/src/adapters/store_migration/filesystem_inventory_directory.rs new file mode 100644 index 0000000..c797f72 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_directory.rs @@ -0,0 +1,96 @@ +//! This module owns pinned migration pool-directory identity. + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, Metadata}; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +use crate::adapters::sync_capable_directory; + +pub(super) struct PinnedMigrationPoolDirectory { + pool: MigrationInventoryPool, + name: &'static str, + identity: DirectoryIdentity, + directory: Dir, +} + +impl PinnedMigrationPoolDirectory { + pub(super) fn open( + root: &Dir, + pool: MigrationInventoryPool, + name: &'static str, + ) -> Result { + let directory = sync_capable_directory::open(root, name).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::OpenPool, + source, + } + })?; + let identity = DirectoryIdentity::read(&directory).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::OpenPool, + source, + } + })?; + Ok(Self { + pool, + name, + identity, + directory, + }) + } + + pub(super) fn verify(&self, root: &Dir) -> Result<(), FilesystemMigrationInventoryError> { + let handle = DirectoryIdentity::read(&self.directory).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(self.pool), + operation: FilesystemMigrationInventoryOperation::VerifyPool, + source, + } + })?; + let metadata = root.symlink_metadata(self.name).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(self.pool), + operation: FilesystemMigrationInventoryOperation::VerifyPool, + source, + } + })?; + let current = DirectoryIdentity::from(&metadata); + if metadata.is_dir() && handle == self.identity && current == self.identity { + Ok(()) + } else { + Err(FilesystemMigrationInventoryError::NamespaceChanged { pool: self.pool }) + } + } + + pub(super) const fn directory(&self) -> &Dir { + &self.directory + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DirectoryIdentity { + device: u64, + inode: u64, +} + +impl DirectoryIdentity { + fn read(directory: &Dir) -> std::io::Result { + directory + .dir_metadata() + .map(|metadata| Self::from(&metadata)) + } +} + +impl From<&Metadata> for DirectoryIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_error.rs b/src/adapters/store_migration/filesystem_inventory_error.rs new file mode 100644 index 0000000..df903b2 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_error.rs @@ -0,0 +1,155 @@ +//! This boundary module owns filesystem migration-inventory failures. + +use std::collections::TryReserveError; +use std::io; + +use super::super::{CatalogRestartError, RecoveryEntryName, RecoveryPoolNameError, SegmentDigest}; + +/// Immutable version-1 pool selected during migration inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MigrationInventoryPool { + /// The `segments` immutable pool. + Segments, + /// The `catalogs` immutable pool. + Catalogs, +} + +/// Pinned filesystem namespace observed during migration inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MigrationInventoryNamespace { + /// The writer-authorized store root. + Root, + /// The `segments` immutable pool. + Segments, + /// The `catalogs` immutable pool. + Catalogs, +} + +impl From for MigrationInventoryNamespace { + fn from(pool: MigrationInventoryPool) -> Self { + match pool { + MigrationInventoryPool::Segments => Self::Segments, + MigrationInventoryPool::Catalogs => Self::Catalogs, + } + } +} + +/// Capability-relative directory operation attempted during inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FilesystemMigrationInventoryOperation { + /// Clone the writer-authorized root capability. + CloneRoot, + /// Open one immutable-pool directory without following links. + OpenPool, + /// Revalidate one pinned immutable-pool directory. + VerifyPool, + /// Count entries under the pinned directory capability. + CountEntries, + /// Read exact raw entry names under the pinned directory capability. + ReadEntryNames, +} + +/// Failure to derive exact migration inventory from immutable filesystem pools. +#[derive(Debug)] +pub enum FilesystemMigrationInventoryError { + /// One capability-relative directory operation failed. + Io { + /// Namespace being observed. + namespace: MigrationInventoryNamespace, + /// Exact failed operation. + operation: FilesystemMigrationInventoryOperation, + /// Preserved filesystem source. + source: io::Error, + }, + /// A pinned immutable-pool directory changed identity. + NamespaceChanged { + /// Pool whose canonical directory entry changed. + pool: MigrationInventoryPool, + }, + /// Exact raw pool membership changed during artifact admission. + EntriesChanged { + /// Pool whose entry-name set changed. + pool: MigrationInventoryPool, + }, + /// A pool exceeded the remaining inventory entry budget. + EntryLimitExceeded { + /// Pool being observed. + pool: MigrationInventoryPool, + /// Remaining entry budget. + maximum: u32, + /// Smallest count observed before enumeration stopped. + observed_at_least: u64, + }, + /// The directory entry count changed between bounded passes. + EntryCountChanged { + /// Pool being observed. + pool: MigrationInventoryPool, + /// Count established by the first pass. + expected: u64, + /// Count established by the second pass. + observed: u64, + }, + /// A host entry count did not fit the protocol representation. + EntryCountHostWidth { + /// Pool being observed. + pool: MigrationInventoryPool, + }, + /// Memory could not retain the bounded semantic inventory. + Allocation { + /// Pool being observed. + pool: MigrationInventoryPool, + /// Exact number of entries requested. + entry_count: u64, + /// Preserved allocation source. + source: TryReserveError, + }, + /// One immutable-pool name was not canonical. + Name { + /// Pool containing the entry. + pool: MigrationInventoryPool, + /// Exact raw name that was refused. + name: RecoveryEntryName, + /// Preserved canonical-name refusal. + source: RecoveryPoolNameError, + }, + /// One named immutable artifact could not be completely admitted. + Artifact { + /// Pool containing the artifact. + pool: MigrationInventoryPool, + /// Exact raw canonical name. + name: RecoveryEntryName, + /// Preserved artifact refusal. + source: Box, + }, + /// A canonical artifact changed identity while it was being admitted. + ArtifactChanged { + /// Pool containing the artifact. + pool: MigrationInventoryPool, + /// Exact raw canonical name. + name: RecoveryEntryName, + }, + /// A catalog-referenced segment could not be completely admitted. + ReferencedSegment { + /// Exact segment coordinate required by the catalog. + digest: SegmentDigest, + /// Preserved segment artifact refusal. + source: Box, + }, + /// A catalog-referenced segment changed identity during admission. + ReferencedSegmentChanged { + /// Exact segment coordinate required by the catalog. + digest: SegmentDigest, + }, + /// Combined pool count arithmetic could not be represented. + EntryCountArithmetic, + /// Combined pool count violated the canonical inventory bound. + EntryCount { + /// Preserved canonical count refusal. + source: super::StoreMigrationInventoryEntryCountError, + }, + /// Canonical entry streaming refused the observed inventory. + Canonical { + /// Preserved canonical inventory refusal. + source: Box, + }, +} diff --git a/src/adapters/store_migration/filesystem_inventory_error_display.rs b/src/adapters/store_migration/filesystem_inventory_error_display.rs new file mode 100644 index 0000000..aa955cf --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_error_display.rs @@ -0,0 +1,147 @@ +//! This module owns filesystem migration-inventory error presentation. + +use std::error::Error; +use std::fmt; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; + +impl fmt::Display for MigrationInventoryPool { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Segments => "segments", + Self::Catalogs => "catalogs", + }) + } +} + +impl fmt::Display for FilesystemMigrationInventoryOperation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::CloneRoot => "clone writer-authorized root", + Self::OpenPool => "open pool capability", + Self::VerifyPool => "verify pool capability", + Self::CountEntries => "count entries", + Self::ReadEntryNames => "read entry names", + }) + } +} + +impl fmt::Display for MigrationInventoryNamespace { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Root => "root", + Self::Segments => "segments", + Self::Catalogs => "catalogs", + }) + } +} + +impl fmt::Display for FilesystemMigrationInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { + namespace, + operation, + .. + } => write!( + formatter, + "migration inventory failed to {operation} in {namespace}" + ), + Self::NamespaceChanged { pool } => { + write!( + formatter, + "migration inventory namespace changed for {pool}" + ) + } + Self::EntriesChanged { pool } => { + write!(formatter, "migration inventory entries changed for {pool}") + } + Self::EntryLimitExceeded { + pool, + maximum, + observed_at_least, + } => write!( + formatter, + "migration inventory observed at least {observed_at_least} entries in {pool}, \ + exceeding remaining limit {maximum}" + ), + Self::EntryCountChanged { + pool, + expected, + observed, + } => write!( + formatter, + "migration inventory entry count changed in {pool}: expected {expected}, \ + observed {observed}" + ), + Self::EntryCountHostWidth { pool } => { + write!( + formatter, + "migration inventory count does not fit for {pool}" + ) + } + Self::Allocation { + pool, entry_count, .. + } => write!( + formatter, + "migration inventory could not retain {entry_count} entries for {pool}" + ), + Self::Name { pool, name, .. } => write!( + formatter, + "migration inventory found noncanonical name {:?} in {pool}", + name.as_bytes() + ), + Self::Artifact { pool, name, .. } => write!( + formatter, + "migration inventory could not admit artifact {:?} in {pool}", + name.as_bytes() + ), + Self::ArtifactChanged { pool, name } => write!( + formatter, + "migration inventory artifact {:?} changed identity in {pool}", + name.as_bytes() + ), + Self::ReferencedSegment { digest, .. } => write!( + formatter, + "migration inventory could not admit catalog segment {digest:?}" + ), + Self::ReferencedSegmentChanged { digest } => write!( + formatter, + "migration inventory catalog segment {digest:?} changed identity" + ), + Self::EntryCountArithmetic => { + formatter.write_str("migration inventory entry count overflowed") + } + Self::EntryCount { .. } => { + formatter.write_str("migration inventory entry count was refused") + } + Self::Canonical { .. } => { + formatter.write_str("migration inventory canonical streaming failed") + } + } + } +} + +impl Error for FilesystemMigrationInventoryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Allocation { source, .. } => Some(source), + Self::Name { source, .. } => Some(source), + Self::Artifact { source, .. } | Self::ReferencedSegment { source, .. } => Some(source), + Self::EntryCount { source } => Some(source), + Self::Canonical { source } => Some(source), + Self::EntryLimitExceeded { .. } + | Self::EntryCountChanged { .. } + | Self::EntryCountHostWidth { .. } + | Self::ArtifactChanged { .. } + | Self::ReferencedSegmentChanged { .. } + | Self::NamespaceChanged { .. } + | Self::EntriesChanged { .. } + | Self::EntryCountArithmetic => None, + } + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_file.rs b/src/adapters/store_migration/filesystem_inventory_file.rs new file mode 100644 index 0000000..7008016 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_file.rs @@ -0,0 +1,128 @@ +//! This module owns identity-stable migration artifact reads. + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, File, Metadata}; + +use crate::adapters::{ + CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, catalog_restart_io, +}; + +pub(super) enum FilesystemInventoryFileError { + Artifact(Box), + Changed, +} + +#[derive(Clone, Copy)] +pub(super) struct FilesystemInventoryFilePolicy { + artifact: CatalogRestartArtifact, + open_phase: CatalogRestartPhase, + read_phase: CatalogRestartPhase, + maximum_length: u64, +} + +impl FilesystemInventoryFilePolicy { + pub(super) const fn new( + artifact: CatalogRestartArtifact, + open_phase: CatalogRestartPhase, + read_phase: CatalogRestartPhase, + maximum_length: u64, + ) -> Self { + Self { + artifact, + open_phase, + read_phase, + maximum_length, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FileIdentity { + device: u64, + inode: u64, + length: u64, +} + +impl FileIdentity { + fn read(file: &File, phase: CatalogRestartPhase) -> Result { + file.metadata() + .map(|metadata| Self::from(&metadata)) + .map_err(|source| CatalogRestartError::io(phase, source)) + } +} + +impl From<&Metadata> for FileIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + length: metadata.len(), + } + } +} + +pub(super) fn read( + directory: &Dir, + name: &str, + policy: FilesystemInventoryFilePolicy, +) -> Result, FilesystemInventoryFileError> { + read_with(directory, name, policy, || {}) +} + +pub(super) fn read_with( + directory: &Dir, + name: &str, + policy: FilesystemInventoryFilePolicy, + before_verify: F, +) -> Result, FilesystemInventoryFileError> +where + F: FnOnce(), +{ + let (file, length) = + catalog_restart_io::open_regular(directory, name, policy.artifact, policy.open_phase) + .map_err(artifact_error)?; + if length > policy.maximum_length { + return Err(FilesystemInventoryFileError::Artifact(Box::new( + CatalogRestartError::Length { + artifact: policy.artifact, + minimum: 0, + maximum: policy.maximum_length, + observed: length, + }, + ))); + } + let identity = FileIdentity::read(&file, policy.read_phase).map_err(artifact_error)?; + let retained = file + .try_clone() + .map_err(|source| CatalogRestartError::io(policy.read_phase, source)) + .map_err(artifact_error)?; + let encoded = catalog_restart_io::read_exact(file, policy.artifact, policy.read_phase, length) + .map_err(artifact_error)?; + before_verify(); + verify(directory, name, &retained, identity, policy.read_phase)?; + Ok(encoded) +} + +fn verify( + directory: &Dir, + name: &str, + file: &File, + identity: FileIdentity, + phase: CatalogRestartPhase, +) -> Result<(), FilesystemInventoryFileError> { + let handle = FileIdentity::read(file, phase).map_err(artifact_error)?; + let metadata = directory + .symlink_metadata(name) + .map_err(|source| CatalogRestartError::io(phase, source)) + .map_err(artifact_error)?; + let current = FileIdentity::from(&metadata); + if metadata.is_file() && handle == identity && current == identity { + Ok(()) + } else { + Err(FilesystemInventoryFileError::Changed) + } +} + +fn artifact_error(source: CatalogRestartError) -> FilesystemInventoryFileError { + FilesystemInventoryFileError::Artifact(Box::new(source)) +} diff --git a/src/adapters/store_migration/filesystem_inventory_file_tests.rs b/src/adapters/store_migration/filesystem_inventory_file_tests.rs new file mode 100644 index 0000000..5735950 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_file_tests.rs @@ -0,0 +1,67 @@ +//! Identity-stable migration artifact read laws. + +use std::error::Error; +use std::fs; +use std::io; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_inventory_file::{ + self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, +}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::{CatalogRestartArtifact, CatalogRestartPhase}; + +#[test] +fn replacement_after_read_refuses_the_opened_artifact() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-artifact-replacement")?; + let name = "artifact"; + fs::write(sandbox.path().join(name), b"old bytes")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + + let result = filesystem_inventory_file::read_with(&directory, name, policy(9), || { + replace(sandbox.path(), name); + }); + assert!(matches!(result, Err(FilesystemInventoryFileError::Changed))); + drop(directory); + sandbox.remove()?; + Ok(()) +} + +fn replace(root: &std::path::Path, name: &str) { + let renamed = fs::rename(root.join(name), root.join("replaced")); + let written = fs::write(root.join(name), b"old bytes"); + assert!(renamed.is_ok(), "test replacement rename failed"); + assert!(written.is_ok(), "test replacement write failed"); +} + +#[test] +fn regular_file_read_returns_exact_bytes() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-artifact-read")?; + let name = "artifact"; + fs::write(sandbox.path().join(name), b"exact")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + + let bytes = filesystem_inventory_file::read(&directory, name, policy(5)).map_err(file_error)?; + assert_eq!(bytes, b"exact"); + drop(directory); + sandbox.remove()?; + Ok(()) +} + +fn file_error(error: FilesystemInventoryFileError) -> io::Error { + match error { + FilesystemInventoryFileError::Artifact(source) => io::Error::other(source), + FilesystemInventoryFileError::Changed => io::Error::other("artifact changed"), + } +} + +const fn policy(maximum_length: u64) -> FilesystemInventoryFilePolicy { + FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + CatalogRestartPhase::ReadCatalog, + maximum_length, + ) +} diff --git a/src/adapters/store_migration/filesystem_inventory_names.rs b/src/adapters/store_migration/filesystem_inventory_names.rs new file mode 100644 index 0000000..ce76dcd --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_names.rs @@ -0,0 +1,61 @@ +//! This module owns deterministic bounded migration pool-name scans. + +use cap_std::fs::Dir; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +use crate::adapters::{RecoveryEntryName, filesystem_recovery_inventory_scan}; + +pub(super) fn read( + directory: &Dir, + pool: MigrationInventoryPool, + remaining: u32, +) -> Result, FilesystemMigrationInventoryError> { + let expected = + filesystem_recovery_inventory_scan::count_entries(directory, u64::from(remaining)) + .map_err(|source| FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::CountEntries, + source, + })?; + if expected > u64::from(remaining) { + return Err(FilesystemMigrationInventoryError::EntryLimitExceeded { + pool, + maximum: remaining, + observed_at_least: expected, + }); + } + let mut names = filesystem_recovery_inventory_scan::read_entry_names(directory, expected) + .map_err(|source| FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::ReadEntryNames, + source, + })?; + let observed = u64::try_from(names.len()) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool })?; + if observed != expected { + return Err(FilesystemMigrationInventoryError::EntryCountChanged { + pool, + expected, + observed, + }); + } + names.sort_unstable(); + Ok(names) +} + +pub(super) fn verify( + directory: &Dir, + pool: MigrationInventoryPool, + remaining: u32, + expected: &[RecoveryEntryName], +) -> Result<(), FilesystemMigrationInventoryError> { + let observed = read(directory, pool, remaining)?; + if observed == expected { + Ok(()) + } else { + Err(FilesystemMigrationInventoryError::EntriesChanged { pool }) + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_names_tests.rs b/src/adapters/store_migration/filesystem_inventory_names_tests.rs new file mode 100644 index 0000000..729b0cc --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_names_tests.rs @@ -0,0 +1,41 @@ +//! Migration pool-name revalidation laws. + +use std::error::Error; +use std::fs; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_names; +use crate::adapters::filesystem_test_sandbox::TestDirectory; + +#[test] +fn membership_change_after_scan_refuses_revalidation() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-name-revalidation")?; + fs::write(sandbox.path().join("first"), b"one")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + let expected = + filesystem_inventory_names::read(&directory, MigrationInventoryPool::Segments, 2)?; + fs::write(sandbox.path().join("second"), b"two")?; + + let error = filesystem_inventory_names::verify( + &directory, + MigrationInventoryPool::Segments, + 2, + &expected, + ) + .err() + .ok_or("changed membership unexpectedly passed")?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::EntriesChanged { + pool: MigrationInventoryPool::Segments + } + )); + drop(directory); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_inventory_reader.rs b/src/adapters/store_migration/filesystem_inventory_reader.rs new file mode 100644 index 0000000..06b00ea --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_reader.rs @@ -0,0 +1,168 @@ +//! This module owns writer-locked filesystem migration pool inventory. + +use cap_std::fs::Dir; + +use super::filesystem_inventory_catalogs; +use super::filesystem_inventory_catalogs::FilesystemMigrationCatalogInventory; +use super::filesystem_inventory_directory::PinnedMigrationPoolDirectory; +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +use super::filesystem_inventory_segments; +use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; +use super::{ + ImmutablePoolInventoryDigest, StoreMigrationInventoryEntryCount, StoreMigrationInventoryHasher, +}; +use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock, SegmentReadPolicy}; + +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; + +/// Writer-authorized reader for one exact version-1 immutable-pool inventory. +/// +/// The reader retains the writer lock and pinned root, segment-pool, and +/// catalog-pool capabilities. It performs no protocol mutation. +#[must_use] +pub struct FilesystemStoreMigrationInventoryReader { + root: Dir, + segments: PinnedMigrationPoolDirectory, + catalogs: PinnedMigrationPoolDirectory, + policy: SegmentReadPolicy, + _lock: FilesystemWriterLock, +} + +impl FilesystemStoreMigrationInventoryReader { + /// Pins both immutable pools under admitted exclusive writer authority. + /// + /// The synchronous call performs bounded capability-relative filesystem + /// I/O and allocates no content-sized memory. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationInventoryError`] when the root capability + /// cannot be cloned or either canonical pool is missing, linked, replaced, + /// or not a directory. + pub fn open( + admission: FilesystemPlatformAdmission, + policy: SegmentReadPolicy, + ) -> Result { + let lock = admission.into_lock(); + let root = + lock.clone_directory() + .map_err(|source| FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::Root, + operation: FilesystemMigrationInventoryOperation::CloneRoot, + source, + })?; + let segments = PinnedMigrationPoolDirectory::open( + &root, + MigrationInventoryPool::Segments, + SEGMENTS_NAME, + )?; + let catalogs = PinnedMigrationPoolDirectory::open( + &root, + MigrationInventoryPool::Catalogs, + CATALOGS_NAME, + )?; + Ok(Self { + root, + segments, + catalogs, + policy, + _lock: lock, + }) + } + + /// Derives the exact bounded canonical inventory digest. + /// + /// Every segment and catalog pool entry is named canonically, opened + /// without following links, read under the fixed format bound, verified + /// against its physical coordinate, and completely admitted. Catalog + /// record bindings reopen only referenced admitted segment coordinates, so + /// peak content allocation is bounded by one catalog and one segment. + /// + /// The synchronous call may block on filesystem I/O and performs no + /// protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationInventoryError`] for namespace drift, + /// count drift or overflow, noncanonical names, linked or changed + /// artifacts, malformed content, catalog binding failure, allocation + /// refusal, or canonical digest-stream refusal. + pub fn read(&self) -> Result { + self.verify_directories()?; + let (segments, catalogs) = self.read_pools()?; + let digest = hash_inventory(&segments, &catalogs)?; + segments.verify_names(self.segments.directory())?; + catalogs.verify_names(self.catalogs.directory())?; + self.verify_directories()?; + Ok(digest) + } + + fn read_pools( + &self, + ) -> Result< + ( + FilesystemMigrationSegmentInventory, + FilesystemMigrationCatalogInventory, + ), + FilesystemMigrationInventoryError, + > { + let maximum = StoreMigrationInventoryEntryCount::MAXIMUM; + let segments = + filesystem_inventory_segments::read(self.segments.directory(), maximum, self.policy)?; + let segment_count = host_count(segments.len(), MigrationInventoryPool::Segments)?; + let remaining = maximum + .checked_sub(segment_count) + .ok_or(FilesystemMigrationInventoryError::EntryCountArithmetic)?; + let catalogs = filesystem_inventory_catalogs::read( + self.catalogs.directory(), + self.segments.directory(), + &segments, + remaining, + self.policy, + )?; + Ok((segments, catalogs)) + } + + fn verify_directories(&self) -> Result<(), FilesystemMigrationInventoryError> { + self.segments.verify(&self.root)?; + self.catalogs.verify(&self.root) + } +} + +fn hash_inventory( + segments: &FilesystemMigrationSegmentInventory, + catalogs: &FilesystemMigrationCatalogInventory, +) -> Result { + let segment_count = host_count(segments.len(), MigrationInventoryPool::Segments)?; + let catalog_count = host_count(catalogs.len(), MigrationInventoryPool::Catalogs)?; + let total = segment_count + .checked_add(catalog_count) + .ok_or(FilesystemMigrationInventoryError::EntryCountArithmetic)?; + let count = StoreMigrationInventoryEntryCount::new(total) + .map_err(|source| FilesystemMigrationInventoryError::EntryCount { source })?; + let mut hasher = StoreMigrationInventoryHasher::new(count); + for entry in segments.entries().iter().chain(catalogs.entries()) { + hasher.push(*entry).map_err(canonical_error)?; + } + hasher.finish().map_err(canonical_error) +} + +fn canonical_error( + source: super::StoreMigrationInventoryError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Canonical { + source: Box::new(source), + } +} + +fn host_count( + count: usize, + pool: MigrationInventoryPool, +) -> Result { + u32::try_from(count) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool }) +} diff --git a/src/adapters/store_migration/filesystem_inventory_reader_tests.rs b/src/adapters/store_migration/filesystem_inventory_reader_tests.rs new file mode 100644 index 0000000..5087431 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_reader_tests.rs @@ -0,0 +1,80 @@ +//! Writer-locked filesystem migration inventory laws. + +use std::error::Error; +use std::fs; + +use super::filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; +use super::{FilesystemMigrationInventoryError, MigrationInventoryPool}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedSegment, ChecksummedCatalog, FilesystemPlatformAdmission, physical_pool_name, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; + +#[test] +fn writer_locked_pools_reproduce_the_frozen_inventory_digest() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-inventory-reader")?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + let segment_bytes = decode_hex(SEGMENT_HEX.trim())?; + let catalog_bytes = decode_hex(CATALOG_HEX.trim())?; + let policy = super::filesystem_inventory_catalogs_test_fixture::maximum_policy(); + let segment = AdmittedSegment::decode(&segment_bytes, policy)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + fs::write( + sandbox + .path() + .join("segments") + .join(physical_pool_name::segment(segment.digest())), + &segment_bytes, + )?; + fs::write( + sandbox + .path() + .join("catalogs") + .join(physical_pool_name::catalog( + catalog.generation(), + catalog.digest(), + )), + &catalog_bytes, + )?; + + let reader = FilesystemStoreMigrationInventoryReader::open(admission, policy)?; + let digest = reader.read()?; + assert_eq!(digest.as_bytes().as_slice(), decode_hex(INVENTORY_DIGEST)?); + drop(reader); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn replaced_pool_directory_refuses_before_artifact_reads() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-inventory-directory-replacement")?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + let policy = super::filesystem_inventory_catalogs_test_fixture::maximum_policy(); + let reader = FilesystemStoreMigrationInventoryReader::open(admission, policy)?; + fs::rename( + sandbox.path().join("segments"), + sandbox.path().join("segments-replaced"), + )?; + fs::create_dir(sandbox.path().join("segments"))?; + + let error = reader + .read() + .err() + .ok_or("replaced segment pool unexpectedly passed")?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::NamespaceChanged { + pool: MigrationInventoryPool::Segments + } + )); + drop(reader); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments.rs b/src/adapters/store_migration/filesystem_inventory_segments.rs new file mode 100644 index 0000000..1964c22 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments.rs @@ -0,0 +1,183 @@ +//! This module owns complete filesystem migration segment-pool admission. + +use std::collections::TryReserveError; + +use cap_std::fs::Dir; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_file::{ + self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, +}; +use super::filesystem_inventory_names; +use crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH; +use crate::adapters::{ + AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, + RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, recovery_pool_name, +}; + +const POOL: MigrationInventoryPool = MigrationInventoryPool::Segments; +pub(super) struct FilesystemMigrationSegmentInventory { + entries: Vec, + digests: Vec, + names: Vec, + remaining: u32, +} + +impl FilesystemMigrationSegmentInventory { + pub(super) fn entries(&self) -> &[StoreMigrationInventoryEntry] { + &self.entries + } + + pub(super) fn contains(&self, digest: SegmentDigest) -> bool { + self.digests.binary_search(&digest).is_ok() + } + + pub(super) const fn len(&self) -> usize { + self.entries.len() + } + + pub(super) fn verify_names( + &self, + directory: &Dir, + ) -> Result<(), FilesystemMigrationInventoryError> { + filesystem_inventory_names::verify(directory, POOL, self.remaining, &self.names) + } +} + +pub(super) fn read( + directory: &Dir, + remaining: u32, + policy: SegmentReadPolicy, +) -> Result { + let names = filesystem_inventory_names::read(directory, POOL, remaining)?; + let capacity = names.len(); + let entry_count = u64::try_from(capacity) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool: POOL })?; + let mut entries = reserve(capacity, entry_count)?; + let mut digests = reserve(capacity, entry_count)?; + for name in &names { + let (entry, digest) = admit(directory, name, policy)?; + entries.push(entry); + digests.push(digest); + } + entries.sort_unstable(); + digests.sort_unstable(); + Ok(FilesystemMigrationSegmentInventory { + entries, + digests, + names, + remaining, + }) +} + +fn reserve( + capacity: usize, + entry_count: u64, +) -> Result, FilesystemMigrationInventoryError> { + let mut values = Vec::new(); + values + .try_reserve_exact(capacity) + .map_err(|source| allocation(entry_count, source))?; + Ok(values) +} + +const fn allocation( + entry_count: u64, + source: TryReserveError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Allocation { + pool: POOL, + entry_count, + source, + } +} + +fn admit( + directory: &Dir, + name: &RecoveryEntryName, + policy: SegmentReadPolicy, +) -> Result<(StoreMigrationInventoryEntry, SegmentDigest), FilesystemMigrationInventoryError> { + let expected = parse_name(name)?; + let encoded = read_encoded(directory, name, expected)?; + let segment = AdmittedSegment::decode(&encoded, policy).map_err(|source| { + artifact_error( + name, + CatalogRestartError::Segment { + expected, + source: Box::new(source), + }, + ) + })?; + if segment.digest() != expected { + return Err(artifact_error( + name, + CatalogRestartError::SegmentCoordinate { + expected, + observed: segment.digest(), + }, + )); + } + Ok(( + StoreMigrationInventoryEntry::from_segment(&segment), + segment.digest(), + )) +} + +fn parse_name( + name: &RecoveryEntryName, +) -> Result { + recovery_pool_name::segment(name).map_err(|source| FilesystemMigrationInventoryError::Name { + pool: POOL, + name: name.clone(), + source, + }) +} + +fn read_encoded( + directory: &Dir, + name: &RecoveryEntryName, + expected: SegmentDigest, +) -> Result, FilesystemMigrationInventoryError> { + let canonical_name = physical_pool_name::segment(expected); + let artifact = CatalogRestartArtifact::Segment { digest: expected }; + filesystem_inventory_file::read( + directory, + &canonical_name, + FilesystemInventoryFilePolicy::new( + artifact, + CatalogRestartPhase::OpenSegment, + CatalogRestartPhase::ReadSegment, + MAXIMUM_SEGMENT_LENGTH, + ), + ) + .map_err(|source| file_error(name, source)) +} + +fn artifact_error( + name: &RecoveryEntryName, + source: CatalogRestartError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Artifact { + pool: POOL, + name: name.clone(), + source: Box::new(source), + } +} + +fn file_error( + name: &RecoveryEntryName, + source: FilesystemInventoryFileError, +) -> FilesystemMigrationInventoryError { + match source { + FilesystemInventoryFileError::Artifact(source) => artifact_error(name, *source), + FilesystemInventoryFileError::Changed => { + FilesystemMigrationInventoryError::ArtifactChanged { + pool: POOL, + name: name.clone(), + } + } + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs b/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs new file mode 100644 index 0000000..1339844 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs @@ -0,0 +1,156 @@ +//! Filesystem migration segment-pool refusal laws. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_segments; +use super::filesystem_inventory_segments_test_fixture::{ + SegmentPoolFixture, maximum_policy, one_zero_bytes, +}; +use crate::adapters::{ + AdmittedSegment, CatalogRestartError, CatalogRestartPhase, RecoveryPoolNameError, + physical_pool_name, +}; + +#[test] +fn noncanonical_segment_name_refuses_inventory() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-name-refusal")?; + let bytes = one_zero_bytes()?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + let canonical = physical_pool_name::segment(segment.digest()); + let stem = canonical + .strip_suffix(".seg") + .ok_or_else(|| io::Error::other("canonical segment name lost its suffix"))?; + fixture.write_named(&format!("{}.seg", stem.to_uppercase()), &bytes)?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 1, + maximum_policy(), + ))?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::Name { + pool: MigrationInventoryPool::Segments, + source: RecoveryPoolNameError::UppercaseDigest, + .. + } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +#[test] +fn corrupt_segment_bytes_refuse_inventory() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-corruption-refusal")?; + let mut bytes = one_zero_bytes()?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + let name = physical_pool_name::segment(segment.digest()); + let first = bytes + .first_mut() + .ok_or_else(|| io::Error::other("segment fixture is empty"))?; + *first ^= u8::MAX; + fixture.write_named(&name, &bytes)?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::Artifact { + pool: observed_pool, + source, + .. + } = error + else { + return Err(io::Error::other("corrupt segment returned wrong refusal").into()); + }; + assert_eq!(observed_pool, MigrationInventoryPool::Segments); + assert!(matches!( + source.as_ref(), + CatalogRestartError::Segment { .. } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn linked_segment_entry_refuses_inventory() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = SegmentPoolFixture::create("migration-segment-link-refusal")?; + let bytes = one_zero_bytes()?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + let name = physical_pool_name::segment(segment.digest()); + fs::write(fixture.path().join("linked-target"), &bytes)?; + symlink("../linked-target", fixture.pool_path().join(name))?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::Artifact { + pool: observed_pool, + source, + .. + } = error + else { + return Err(io::Error::other("linked segment returned wrong refusal").into()); + }; + assert_eq!(observed_pool, MigrationInventoryPool::Segments); + assert!(matches!( + source.as_ref(), + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenSegment, + .. + } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +#[test] +fn segment_pool_above_remaining_limit_refuses_before_names() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-limit-refusal")?; + fixture.write_named("unknown", b"not admitted")?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 0, + maximum_policy(), + ))?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::EntryLimitExceeded { + pool: MigrationInventoryPool::Segments, + maximum: 0, + observed_at_least: 1, + } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +fn require_error( + result: Result, +) -> Result { + result.map_or_else(Ok, |_value| { + Err(io::Error::other( + "filesystem migration segment inventory unexpectedly succeeded", + )) + }) +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs b/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs new file mode 100644 index 0000000..7a39904 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs @@ -0,0 +1,79 @@ +//! Deterministic filesystem migration segment-pool fixture. + +use std::error::Error; +use std::fs; +use std::path::Path; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use crate::LayoutEntryLimit; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedSegment, SegmentDigest, SegmentReadPolicy, SegmentRecordLimit, physical_pool_name, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); + +pub(super) struct SegmentPoolFixture { + sandbox: TestDirectory, +} + +impl SegmentPoolFixture { + pub(super) fn create(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::create_dir(sandbox.path().join("segments"))?; + Ok(Self { sandbox }) + } + + pub(super) fn path(&self) -> &Path { + self.sandbox.path() + } + + pub(super) fn pool_path(&self) -> std::path::PathBuf { + self.path().join("segments") + } + + pub(super) fn open(&self) -> Result> { + Ok(Dir::open_ambient_dir( + self.pool_path(), + ambient_authority(), + )?) + } + + pub(super) fn write_segment(&self, bytes: &[u8]) -> Result> { + let segment = AdmittedSegment::decode(bytes, maximum_policy())?; + fs::write( + self.pool_path() + .join(physical_pool_name::segment(segment.digest())), + bytes, + )?; + Ok(segment.digest()) + } + + pub(super) fn write_named(&self, name: &str, bytes: &[u8]) -> Result<(), Box> { + fs::write(self.pool_path().join(name), bytes)?; + Ok(()) + } + + pub(super) fn remove(self) -> Result<(), Box> { + self.sandbox.remove()?; + Ok(()) + } +} + +pub(super) fn one_zero_bytes() -> Result, Box> { + Ok(decode_hex(SEGMENT_HEX.trim())?) +} + +pub(super) fn empty_bytes() -> Result, Box> { + Ok(decode_hex(EMPTY_SEGMENT_HEX.trim())?) +} + +pub(super) const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_tests.rs b/src/adapters/store_migration/filesystem_inventory_segments_tests.rs new file mode 100644 index 0000000..caa92ac --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments_tests.rs @@ -0,0 +1,36 @@ +//! Filesystem migration segment-pool admission laws. + +use std::error::Error; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_segments; +use super::filesystem_inventory_segments_test_fixture::{ + SegmentPoolFixture, empty_bytes, maximum_policy, one_zero_bytes, +}; +use crate::adapters::AdmittedSegment; + +#[test] +fn every_canonical_segment_is_admitted_into_migration_inventory() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-inventory")?; + let first_bytes = one_zero_bytes()?; + let second_bytes = empty_bytes()?; + let first = AdmittedSegment::decode(&first_bytes, maximum_policy())?; + let second = AdmittedSegment::decode(&second_bytes, maximum_policy())?; + let _first_digest = fixture.write_segment(&first_bytes)?; + let _second_digest = fixture.write_segment(&second_bytes)?; + + let pool = fixture.open()?; + let inventory = filesystem_inventory_segments::read(&pool, 2, maximum_policy())?; + let mut expected = [ + StoreMigrationInventoryEntry::from_segment(&first), + StoreMigrationInventoryEntry::from_segment(&second), + ]; + expected.sort_unstable(); + + assert_eq!(inventory.entries(), expected.as_slice()); + assert!(inventory.contains(first.digest())); + assert!(inventory.contains(second.digest())); + drop(pool); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/migration_catalog_admission.rs b/src/adapters/store_migration/migration_catalog_admission.rs new file mode 100644 index 0000000..bcb97b2 --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_admission.rs @@ -0,0 +1,106 @@ +//! This module owns bounded catalog admission for migration pool inventory. + +use crate::adapters::{ + AdmittedSegment, CatalogAdmissionError, ChecksummedCatalog, SegmentDigest, SegmentReadPolicy, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +use super::{migration_catalog_plan, migration_catalog_records}; + +pub(super) struct AdmittedMigrationCatalog<'a> { + catalog: ChecksummedCatalog<'a>, +} + +impl AdmittedMigrationCatalog<'_> { + pub(super) const fn generation(&self) -> CatalogGeneration { + self.catalog.generation() + } + + pub(super) const fn length(&self) -> CatalogLength { + self.catalog.length() + } + + pub(super) const fn digest(&self) -> CatalogDigest { + self.catalog.digest() + } +} + +pub(super) enum MigrationCatalogAdmissionError { + Catalog(Box), + SegmentSource { + digest: SegmentDigest, + source: E, + }, + SegmentCoordinate { + expected: SegmentDigest, + observed: SegmentDigest, + }, +} + +pub(super) enum MigrationSegmentLoadError { + Missing, + Source(E), +} + +pub(super) fn admit( + catalog: ChecksummedCatalog<'_>, + policy: SegmentReadPolicy, + mut load: F, +) -> Result, MigrationCatalogAdmissionError> +where + F: FnMut(SegmentDigest) -> Result, MigrationSegmentLoadError>, +{ + let mut plan = migration_catalog_plan::plan(catalog)?; + plan.sort_unstable_by_key(|entry| entry.physical_order()); + for entries in + plan.chunk_by(|first, second| first.entry.segment_digest() == second.entry.segment_digest()) + { + admit_segment(entries, policy, &mut load)?; + } + Ok(AdmittedMigrationCatalog { catalog }) +} + +fn admit_segment( + entries: &[migration_catalog_plan::PlannedEntry], + policy: SegmentReadPolicy, + load: &mut F, +) -> Result<(), MigrationCatalogAdmissionError> +where + F: FnMut(SegmentDigest) -> Result, MigrationSegmentLoadError>, +{ + let Some(first) = entries.first() else { + return Ok(()); + }; + let expected = first.entry.segment_digest(); + let encoded = match load(expected) { + Ok(encoded) => encoded, + Err(MigrationSegmentLoadError::Missing) => { + return Err(catalog_error(CatalogAdmissionError::MissingSegment { + digest: expected, + })); + } + Err(MigrationSegmentLoadError::Source(source)) => { + return Err(MigrationCatalogAdmissionError::SegmentSource { + digest: expected, + source, + }); + } + }; + let segment = AdmittedSegment::decode(&encoded, policy).map_err(|source| { + catalog_error(CatalogAdmissionError::Segment { + digest: expected, + source: Box::new(source), + }) + })?; + if segment.digest() != expected { + return Err(MigrationCatalogAdmissionError::SegmentCoordinate { + expected, + observed: segment.digest(), + }); + } + migration_catalog_records::validate(entries, &segment).map_err(catalog_error) +} + +pub(super) fn catalog_error(source: CatalogAdmissionError) -> MigrationCatalogAdmissionError { + MigrationCatalogAdmissionError::Catalog(Box::new(source)) +} diff --git a/src/adapters/store_migration/migration_catalog_plan.rs b/src/adapters/store_migration/migration_catalog_plan.rs new file mode 100644 index 0000000..deefce8 --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_plan.rs @@ -0,0 +1,51 @@ +//! This module owns bounded physical catalog-entry planning for migration. + +use crate::adapters::{ + CatalogAdmissionError, CatalogAllocationPhase, ChecksummedCatalog, DecodedCatalogEntry, + SegmentDigest, +}; + +use super::migration_catalog_admission::{MigrationCatalogAdmissionError, catalog_error}; + +#[derive(Clone, Copy)] +pub(super) struct PlannedEntry { + pub(super) ordinal: usize, + pub(super) entry: DecodedCatalogEntry, +} + +impl PlannedEntry { + pub(super) const fn physical_order(self) -> (SegmentDigest, u64, usize) { + ( + self.entry.segment_digest(), + self.entry.record_offset(), + self.ordinal, + ) + } +} + +pub(super) fn plan( + catalog: ChecksummedCatalog<'_>, +) -> Result, MigrationCatalogAdmissionError> { + let requested = usize::try_from(catalog.entry_count()).map_err(|_source| { + catalog_error(CatalogAdmissionError::EntryCountHostWidth { + observed: catalog.entry_count(), + }) + })?; + let mut plan = Vec::new(); + plan.try_reserve_exact(requested).map_err(|source| { + catalog_error(CatalogAdmissionError::Allocation { + phase: CatalogAllocationPhase::EntryPlan, + requested, + source, + }) + })?; + let entries = catalog + .entries() + .map_err(|source| catalog_error(CatalogAdmissionError::Catalog { source }))?; + for (ordinal, entry) in entries.enumerate() { + let entry = + entry.map_err(|source| catalog_error(CatalogAdmissionError::Catalog { source }))?; + plan.push(PlannedEntry { ordinal, entry }); + } + Ok(plan) +} diff --git a/src/adapters/store_migration/migration_catalog_records.rs b/src/adapters/store_migration/migration_catalog_records.rs new file mode 100644 index 0000000..0156c87 --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_records.rs @@ -0,0 +1,93 @@ +//! This module owns exact migration catalog-to-segment record binding. + +use crate::adapters::{ + AdmittedSegment, AdmittedSegmentRecord, CatalogAdmissionError, DecodedCatalogEntry, +}; + +use super::migration_catalog_plan::PlannedEntry; + +pub(super) fn validate( + entries: &[PlannedEntry], + segment: &AdmittedSegment<'_>, +) -> Result<(), CatalogAdmissionError> { + let digest = segment.digest(); + let mut pending = entries.iter().peekable(); + let mut cursor = segment.record_cursor(); + while let Some(located) = + cursor + .next_record() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })? + { + refuse_preceding(&mut pending, located.offset)?; + validate_at_offset(&mut pending, located.offset, located.record)?; + } + cursor + .finish() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })?; + pending + .next() + .map_or(Ok(()), |entry| Err(location_error(entry.entry))) +} + +fn refuse_preceding( + pending: &mut std::iter::Peekable>, + record_offset: u64, +) -> Result<(), CatalogAdmissionError> { + match pending.peek() { + Some(entry) if entry.entry.record_offset() < record_offset => { + Err(location_error(entry.entry)) + } + Some(_) | None => Ok(()), + } +} + +fn validate_at_offset( + pending: &mut std::iter::Peekable>, + record_offset: u64, + record: AdmittedSegmentRecord<'_>, +) -> Result<(), CatalogAdmissionError> { + while matches!(pending.peek(), Some(entry) if entry.entry.record_offset() == record_offset) { + let Some(entry) = pending.next() else { + break; + }; + validate_record(entry.entry, record)?; + } + Ok(()) +} + +fn validate_record( + entry: DecodedCatalogEntry, + record: AdmittedSegmentRecord<'_>, +) -> Result<(), CatalogAdmissionError> { + if record.header().record_length() != entry.record_length() { + return Err(location_error(entry)); + } + if record.identity() != entry.identity() { + return Err(CatalogAdmissionError::RecordIdentityMismatch { + expected: entry.identity(), + observed: record.identity(), + }); + } + if record.checksum() != entry.checksum() { + return Err(CatalogAdmissionError::RecordChecksumMismatch { + expected: entry.checksum(), + observed: record.checksum(), + }); + } + Ok(()) +} + +const fn location_error(entry: DecodedCatalogEntry) -> CatalogAdmissionError { + CatalogAdmissionError::LocationNotTopLevel { + identity: entry.identity(), + segment_digest: entry.segment_digest(), + record_offset: entry.record_offset(), + record_length: entry.record_length().get(), + } +} diff --git a/src/adapters/store_migration/migration_inventory_entry.rs b/src/adapters/store_migration/migration_inventory_entry.rs index 1ce8bc0..e04b89f 100644 --- a/src/adapters/store_migration/migration_inventory_entry.rs +++ b/src/adapters/store_migration/migration_inventory_entry.rs @@ -2,6 +2,8 @@ use crate::{AdmittedCatalog, AdmittedSegment}; +use super::migration_catalog_admission::AdmittedMigrationCatalog; + const SEGMENT_KIND: u8 = 1; const CATALOG_KIND: u8 = 2; const ENCODED_LENGTH: usize = 56; @@ -32,6 +34,15 @@ impl StoreMigrationInventoryEntry { )) } + pub(super) const fn from_migration_catalog(catalog: &AdmittedMigrationCatalog<'_>) -> Self { + Self(encode( + CATALOG_KIND, + catalog.generation().get(), + catalog.length().get(), + catalog.digest().as_bytes(), + )) + } + /// Returns the exact 56 canonical bytes. pub const fn encoded(&self) -> &[u8; ENCODED_LENGTH] { &self.0 diff --git a/src/lib.rs b/src/lib.rs index 2c0f2ac..f82f81d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,16 +68,18 @@ pub use adapters::{ CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, - ImmutablePoolInventoryDigest, InitialGcStateDigest, InitialRetentionStateDigest, - LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, - LayoutIdTextParseError, MigrationSynchronizationMask, OpenedReusableSegment, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationInventoryReader, + FilesystemWriterLock, ImmutablePoolInventoryDigest, InitialGcStateDigest, + InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, MigrationInventoryNamespace, + MigrationInventoryPool, MigrationSynchronizationMask, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, From 991264f4b03342bc8110416e0ad2a773061ee776 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 06:47:41 -0700 Subject: [PATCH 42/50] Add: Observe filesystem migration authority --- CHANGELOG.md | 4 +- .../segment-store-v2/migration-inventory.md | 9 +- docs/formats/segment-store-v2/recovery.md | 50 +++--- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/filesystem_catalog_publisher.rs | 4 +- src/adapters/filesystem_platform_admission.rs | 33 +++- src/adapters/filesystem_platform_profile.rs | 114 +++++-------- .../filesystem_platform_profile_tests.rs | 84 +++++++++ src/adapters/filesystem_root_identity.rs | 30 ++++ src/adapters/filesystem_store_initializer.rs | 19 ++- src/adapters/mod.rs | 1 + src/adapters/store_migration.rs | 12 ++ .../canonical_migration_intent.rs | 20 ++- .../filesystem_inventory_reader.rs | 17 +- .../filesystem_migration_authority.rs | 160 ++++++++++++++++++ .../filesystem_migration_authority_error.rs | 117 +++++++++++++ ...ystem_migration_authority_error_display.rs | 112 ++++++++++++ .../filesystem_migration_authority_tests.rs | 109 ++++++++++++ ...lesystem_migration_authority_validation.rs | 80 +++++++++ .../migration_catalog_coordinates.rs | 52 ++++++ .../migration_intent_encoder.rs | 46 ++--- .../store_migration/store_root_identity.rs | 43 +++++ src/lib.rs | 51 +++--- 23 files changed, 997 insertions(+), 174 deletions(-) create mode 100644 src/adapters/filesystem_platform_profile_tests.rs create mode 100644 src/adapters/filesystem_root_identity.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_error.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_error_display.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_tests.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_validation.rs create mode 100644 src/adapters/store_migration/migration_catalog_coordinates.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index adf1891..72be8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ after its public API and format compatibility policies are established. exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, writer-locked filesystem - inventory completely admits every immutable-pool artifact, and + inventory completely admits every immutable-pool artifact, filesystem + migration authority derives and revalidates one canonical intent from exact + Linux root, namespace, head, catalog, and inventory coordinates, and `StoreMigrationPhase` freezes 21 transitions with explicit storage and verification-first execution. Retention preflight combines expected-generation planning with deterministic diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md index 09c233c..2eed8dc 100644 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -57,5 +57,10 @@ out-of-order evidence, and reproduces the frozen digest. and pinned capabilities for both immutable pools. It inventories every regular entry, including artifacts not reachable from the current publication head, and reproduces the frozen digest without retaining every artifact body at -once. Migration-session integration that revalidates this inventory -immediately before the first namespace mutation remains in progress. +once. `FilesystemStoreMigrationAuthority` combines that digest with an +identity-stable fixed-width `HEAD`, its selected admitted catalog, the exact +version-1 root namespace, and the admitted physical root coordinates. Its +`verify_current` operation repeats the complete observation and refuses any +different canonical intent before mutation. Filesystem migration storage that +invokes this verification immediately before its first namespace mutation +remains in progress. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index b526f9b..9e56d4a 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -1,13 +1,10 @@ # Migration and Recovery -This page owns the version-2 filesystem namespace, format marker, reader fence, -one-way migration, fixed-stage recovery, GC reservation, and -recovery-disposition reservation. +This page owns version-2 filesystem migration and recovery. ## Exact filesystem namespace -Version 2 preserves the version-1 files and directories and admits these new -coordinates: +Version 2 preserves version-1 files and directories; it adds: ```text reader.lock @@ -50,8 +47,7 @@ Operations are capability-relative and never follow links. | 60 | 4 | reserved | zero | | 64 | 32 | checksum | BLAKE3-256 over bytes `0..64` | -The definition and checksum domains are -`keep.segment-store-definition/v2\0` and +The definition and checksum domains are `keep.segment-store-definition/v2\0` and `keep.segment-store-marker-checksum/v2\0`. A missing marker is version 1 only when the exact version-1 namespace admits. An unsupported, corrupt, substituted, or same-name/different-digest marker refuses. @@ -84,7 +80,6 @@ published segment. ## Migration records Migration is a one-way explicit migration under exclusive writer authority. -Version 1 is never extended in place without durable migration evidence. `migration.intent` is exactly 256 bytes: @@ -101,9 +96,9 @@ Version 1 is never extended in place without durable migration evidence. | 40 | 32 | catalog digest named by version-1 `HEAD` | exact admitted digest | | 72 | 32 | predecessor catalog digest | zero for generation 1 | | 104 | 32 | immutable-pool inventory digest | canonical complete inventory | -| 136 | 8 | root device identity | admitted platform value | -| 144 | 8 | root mount identity | admitted platform value | -| 152 | 8 | root file identity | admitted platform value | +| 136 | 8 | root device identity | admitted Linux `dev_t` | +| 144 | 8 | root mount identity | admitted Linux `statx.stx_mnt_id` | +| 152 | 8 | root file identity | admitted Linux `statx.stx_ino` | | 160 | 32 | target format-definition digest | exact registered v2 digest | | 192 | 32 | new store identifier | deterministic derivation below | | 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | @@ -119,6 +114,10 @@ each migration inventory entry is exactly 56 bytes, and the fixed maximum is 2,097,152 entries. The intent therefore binds the exact catalog generation, length, and digest named by the admitted version-1 `HEAD`. +On Linux, the root device coordinate is `dev_t`, reconstructed from +`statx.stx_dev_major` and `statx.stx_dev_minor`; mount and file use +`statx.stx_mnt_id` and `statx.stx_ino`. Each is big-endian `u64`. + The deterministically derived store identifier is: ```text @@ -191,15 +190,18 @@ Migration performs these ordered steps: 9. Publish `migration.receipt` from `migration.receipt.next` through the fixed-stage protocol. -The [migration crash-point specification](migration-crash.md) owns that -protocol and spans `KEEP-CRASH-053` through `KEEP-CRASH-073`. - +The [migration crash-point specification](migration-crash.md) owns +`KEEP-CRASH-053` through `KEEP-CRASH-073`. Migration never rewrites or deletes admitted version-1 immutable bytes and provides no automatic downgrade. -Version-1 admission refuses once any migration stage, `migration.intent`, -`reader.lock`, `FORMAT`, or version-2 directory is present. Once the canonical -intent is durable, only version-2 migration recovery may continue. +`FilesystemStoreMigrationAuthority` retains the writer lock and pinned root +and pools. It admits the version-1 namespace, Linux root identity, `HEAD`, +complete immutable-pool inventory, and selected catalog. Before mutation, it +requires the same canonical intent. +Version-1 admission refuses after a migration stage, `migration.intent`, +`reader.lock`, `FORMAT`, or version-2 directory exists. After durable intent, +only version-2 migration recovery may continue. ## Partial migration recovery @@ -219,15 +221,13 @@ The migration recovery boundary admits only these ordered prefixes: -A partial migration retry revalidates the intent and every existing byte, -continues idempotently at the first absent canonical step, and never replaces -an existing entry. A missing predecessor, changed version-1 coordinate, -out-of-order name, wrong file kind, substituted byte, conflicting receipt, -unknown entry, or changed root identity is unrecoverable ambiguity. +A partial migration retry revalidates intent and existing bytes, resumes at the +first absent canonical step, and never replaces an entry. A missing predecessor, +changed version-1 coordinate, out-of-order name, wrong kind or bytes, conflicting +receipt, unknown entry, or changed root identity is unrecoverable ambiguity. -Process death before durable canonical intent leaves version 1 plus at most its -non-authoritative stage. Process death after durable intent leaves -recovery-required version-2 migration state. +Death before durable intent leaves v1 plus at most its non-authoritative stage. +Death after durable intent leaves recovery-required v2 migration state. ## Retention publication recovery diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 3ea5b2c..960656f 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,12 +30,12 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; verification-first execution in `tests/store_migration_execution.rs`; mutation-time integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; exact authority observation and drift refusal in `filesystem_migration_authority_tests`; verification-first execution in `tests/store_migration_execution.rs`; filesystem storage integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; remaining compatibility and fuzz matrix | In progress in #19 | diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index b4c191b..16999a2 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -90,7 +90,7 @@ impl FilesystemCatalogPublisher { policy: CatalogRestartPolicy, ) -> io::Result { Self::open( - FilesystemPlatformAdmission::unchecked_for_repository_tasks(lock), + FilesystemPlatformAdmission::unchecked_for_repository_tasks(lock)?, policy, ) } @@ -101,7 +101,7 @@ impl FilesystemCatalogPublisher { policy: CatalogRestartPolicy, ) -> io::Result { Self::open( - FilesystemPlatformAdmission::unchecked_for_tests(lock), + FilesystemPlatformAdmission::unchecked_for_tests(lock)?, policy, ) } diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index 626dab3..8d31009 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -1,6 +1,7 @@ //! This module owns proof that a filesystem root passed platform admission. use super::FilesystemWriterLock; +use super::filesystem_root_identity::FilesystemRootIdentity; /// Exclusive writer authority over a platform-admitted filesystem root. /// @@ -9,24 +10,44 @@ use super::FilesystemWriterLock; #[must_use] pub struct FilesystemPlatformAdmission { lock: FilesystemWriterLock, + root_identity: FilesystemRootIdentity, } impl FilesystemPlatformAdmission { - pub(super) const fn initialized(lock: FilesystemWriterLock) -> Self { - Self { lock } + pub(super) const fn initialized( + lock: FilesystemWriterLock, + root_identity: FilesystemRootIdentity, + ) -> Self { + Self { + lock, + root_identity, + } } #[cfg(test)] - pub(super) const fn unchecked_for_tests(lock: FilesystemWriterLock) -> Self { - Self { lock } + pub(super) fn unchecked_for_tests(lock: FilesystemWriterLock) -> std::io::Result { + Self::unchecked(lock) } #[cfg(feature = "repository-tasks")] - pub(super) const fn unchecked_for_repository_tasks(lock: FilesystemWriterLock) -> Self { - Self { lock } + pub(super) fn unchecked_for_repository_tasks( + lock: FilesystemWriterLock, + ) -> std::io::Result { + Self::unchecked(lock) } pub(super) fn into_lock(self) -> FilesystemWriterLock { self.lock } + + pub(super) fn into_parts(self) -> (FilesystemWriterLock, FilesystemRootIdentity) { + (self.lock, self.root_identity) + } + + #[cfg(any(test, feature = "repository-tasks"))] + fn unchecked(lock: FilesystemWriterLock) -> std::io::Result { + let directory = lock.clone_directory()?; + let root_identity = super::filesystem_platform_profile::root_identity(&directory)?; + Ok(Self::initialized(lock, root_identity)) + } } diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 62e561d..1b6bbda 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -5,6 +5,8 @@ use std::path::Path; use cap_std::fs::Dir; +use super::filesystem_root_identity::FilesystemRootIdentity; + #[cfg(target_os = "linux")] const PROTOCOL_DIRECTORIES: [&str; 3] = ["staging", "segments", "catalogs"]; @@ -17,6 +19,7 @@ struct LinuxDirectoryProperties { device_major: u32, device_minor: u32, mount_id: u64, + inode: u64, } #[cfg(target_os = "linux")] @@ -82,9 +85,43 @@ fn linux_directory_properties(file: &std::fs::File) -> io::Result io::Result { + let file = directory.try_clone()?.into_std_file(); + let properties = linux_directory_properties(&file)?; + Ok(linux_root_identity(properties)) +} + +#[cfg(target_os = "linux")] +fn linux_root_identity(properties: LinuxDirectoryProperties) -> FilesystemRootIdentity { + let device = rustix::fs::makedev(properties.device_major, properties.device_minor); + FilesystemRootIdentity::new(device, properties.mount_id, properties.inode) +} + +#[cfg(all(not(target_os = "linux"), any(test, feature = "repository-tasks")))] +pub(super) fn root_identity(directory: &Dir) -> io::Result { + use cap_fs_ext::MetadataExt; + + let metadata = directory.dir_metadata()?; + Ok(FilesystemRootIdentity::new( + metadata.dev(), + metadata.dev(), + metadata.ino(), + )) +} + +#[cfg(all(not(target_os = "linux"), not(any(test, feature = "repository-tasks"))))] +pub(super) fn root_identity(_directory: &Dir) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "filesystem root identity currently requires the admitted Linux ext4 profile", + )) +} + #[cfg(target_os = "linux")] fn admit_linux_properties( filesystem_type: rustix::fs::FsWord, @@ -130,78 +167,5 @@ fn unsupported_linux_profile() -> io::Error { } #[cfg(all(test, target_os = "linux"))] -mod tests { - use super::{ - LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, - admit_linux_properties, - }; - - use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; - - const EXT4_SUPER_MAGIC: rustix::fs::FsWord = 0x0000_ef53; - const EXT4_CASEFOLD_FLAG: u32 = 0x4000_0000; - - #[test] - fn only_writable_case_sensitive_ext4_is_admitted() { - assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); - assert_unsupported(&admit_linux_properties( - EXT4_SUPER_MAGIC, - StatVfsMountFlags::empty(), - EXT4_CASEFOLD_FLAG, - )); - assert_unsupported(&admit_linux_properties( - EXT4_SUPER_MAGIC, - StatVfsMountFlags::RDONLY, - 0, - )); - assert_unsupported(&admit_linux_properties( - NFS_SUPER_MAGIC, - StatVfsMountFlags::empty(), - 0, - )); - } - - #[test] - fn every_protocol_child_must_share_the_root_filesystem_and_mount() { - assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); - let root = properties(8, 1, 41); - let mut casefolded = root; - casefolded.inode_flags = EXT4_CASEFOLD_FLAG; - let mut read_only = root; - read_only.mount_flags = StatVfsMountFlags::RDONLY; - let mut foreign_format = root; - foreign_format.filesystem_type = NFS_SUPER_MAGIC; - - assert!(admit_linux_child_properties(root, root).is_ok()); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41))); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42))); - assert_unsupported(&admit_linux_child_properties(root, casefolded)); - assert_unsupported(&admit_linux_child_properties(root, read_only)); - assert_unsupported(&admit_linux_child_properties(root, foreign_format)); - } - - fn assert_unsupported(result: &std::io::Result<()>) { - assert!(matches!( - result, - Err(error) - if error.kind() == std::io::ErrorKind::Unsupported - && error.to_string() - == "store namespace does not satisfy one local writable case-sensitive ext4 profile" - )); - } - - const fn properties( - device_major: u32, - device_minor: u32, - mount_id: u64, - ) -> LinuxDirectoryProperties { - LinuxDirectoryProperties { - filesystem_type: EXT4_SUPER_MAGIC, - mount_flags: StatVfsMountFlags::empty(), - inode_flags: 0, - device_major, - device_minor, - mount_id, - } - } -} +#[path = "filesystem_platform_profile_tests.rs"] +mod tests; diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs new file mode 100644 index 0000000..dd9d63e --- /dev/null +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -0,0 +1,84 @@ +//! Linux filesystem platform-profile laws. + +use super::{ + LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, + admit_linux_properties, linux_root_identity, +}; +use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; + +const EXT4_SUPER_MAGIC: rustix::fs::FsWord = 0x0000_ef53; +const EXT4_CASEFOLD_FLAG: u32 = 0x4000_0000; + +#[test] +fn only_writable_case_sensitive_ext4_is_admitted() { + assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); + assert_unsupported(&admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::empty(), + EXT4_CASEFOLD_FLAG, + )); + assert_unsupported(&admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::RDONLY, + 0, + )); + assert_unsupported(&admit_linux_properties( + NFS_SUPER_MAGIC, + StatVfsMountFlags::empty(), + 0, + )); +} + +#[test] +fn every_protocol_child_must_share_the_root_filesystem_and_mount() { + assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); + let root = properties(8, 1, 41, 1); + let mut casefolded = root; + casefolded.inode_flags = EXT4_CASEFOLD_FLAG; + let mut read_only = root; + read_only.mount_flags = StatVfsMountFlags::RDONLY; + let mut foreign_format = root; + foreign_format.filesystem_type = NFS_SUPER_MAGIC; + + assert!(admit_linux_child_properties(root, root).is_ok()); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41, 1))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42, 1))); + assert_unsupported(&admit_linux_child_properties(root, casefolded)); + assert_unsupported(&admit_linux_child_properties(root, read_only)); + assert_unsupported(&admit_linux_child_properties(root, foreign_format)); +} + +#[test] +fn root_identity_uses_linux_device_mount_and_inode_coordinates() { + let identity = linux_root_identity(properties(8, 1, 41, 73)); + assert_eq!(identity.device(), rustix::fs::makedev(8, 1)); + assert_eq!(identity.mount(), 41); + assert_eq!(identity.file(), 73); +} + +fn assert_unsupported(result: &std::io::Result<()>) { + assert!(matches!( + result, + Err(error) + if error.kind() == std::io::ErrorKind::Unsupported + && error.to_string() + == "store namespace does not satisfy one local writable case-sensitive ext4 profile" + )); +} + +const fn properties( + device_major: u32, + device_minor: u32, + mount_id: u64, + inode: u64, +) -> LinuxDirectoryProperties { + LinuxDirectoryProperties { + filesystem_type: EXT4_SUPER_MAGIC, + mount_flags: StatVfsMountFlags::empty(), + inode_flags: 0, + device_major, + device_minor, + mount_id, + inode, + } +} diff --git a/src/adapters/filesystem_root_identity.rs b/src/adapters/filesystem_root_identity.rs new file mode 100644 index 0000000..e63619f --- /dev/null +++ b/src/adapters/filesystem_root_identity.rs @@ -0,0 +1,30 @@ +//! This module owns one admitted physical filesystem-root coordinate. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct FilesystemRootIdentity { + device: u64, + mount: u64, + file: u64, +} + +impl FilesystemRootIdentity { + pub(super) const fn new(device: u64, mount: u64, file: u64) -> Self { + Self { + device, + mount, + file, + } + } + + pub(super) const fn device(self) -> u64 { + self.device + } + + pub(super) const fn mount(self) -> u64 { + self.mount + } + + pub(super) const fn file(self) -> u64 { + self.file + } +} diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index bbcc392..80456bb 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -83,7 +83,17 @@ fn initialize_storage( let lock = storage.into_lock().map_err(|source| { StoreInitializationError::io(StoreInitializationPhase::OpenAndLockWriterFile, source) })?; - Ok(FilesystemPlatformAdmission::initialized(lock)) + let directory = lock.clone_directory().map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) + })?; + let root_identity = + filesystem_platform_profile::root_identity(&directory).map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) + })?; + Ok(FilesystemPlatformAdmission::initialized( + lock, + root_identity, + )) } fn reopen_root( @@ -96,5 +106,10 @@ fn reopen_root( .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; filesystem_initialization_namespace::admit_published(&directory) .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - Ok(FilesystemPlatformAdmission::initialized(lock)) + let root_identity = filesystem_platform_profile::root_identity(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Ok(FilesystemPlatformAdmission::initialized( + lock, + root_identity, + )) } diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 20952e2..3c783ca 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -117,6 +117,7 @@ mod filesystem_recovery_stage_materialization; mod filesystem_recovery_stage_sync; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; +mod filesystem_root_identity; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 93cdfc9..0083394 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -34,6 +34,12 @@ mod filesystem_inventory_segments_refusal_tests; mod filesystem_inventory_segments_test_fixture; #[cfg(test)] mod filesystem_inventory_segments_tests; +mod filesystem_migration_authority; +mod filesystem_migration_authority_error; +mod filesystem_migration_authority_error_display; +#[cfg(test)] +mod filesystem_migration_authority_tests; +mod filesystem_migration_authority_validation; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; @@ -44,6 +50,7 @@ mod immutable_pool_inventory_digest; mod initial_gc_state_digest; mod initial_retention_state_digest; mod migration_catalog_admission; +mod migration_catalog_coordinates; mod migration_catalog_plan; mod migration_catalog_records; mod migration_error; @@ -84,6 +91,11 @@ pub use filesystem_inventory_error::{ MigrationInventoryNamespace, MigrationInventoryPool, }; pub use filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; +pub use filesystem_migration_authority::FilesystemStoreMigrationAuthority; +pub use filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, + StoreRootIdentityCoordinate, +}; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs index 28a95ca..3478a4f 100644 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -1,6 +1,8 @@ //! This boundary module owns canonical owned migration-intent bytes. use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; +use super::store_root_identity::StoreRootIdentities; use super::{ ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, @@ -53,14 +55,24 @@ impl CanonicalStoreMigrationIntent { root_file_identity: StoreRootFileIdentity, ) -> Self { migration_intent_encoder::encode( - snapshot, + MigrationCatalogCoordinates::from_snapshot(snapshot), inventory_digest, - root_device_identity, - root_mount_identity, - root_file_identity, + StoreRootIdentities::new( + root_device_identity, + root_mount_identity, + root_file_identity, + ), ) } + pub(super) fn from_coordinates( + catalog: MigrationCatalogCoordinates, + inventory_digest: ImmutablePoolInventoryDigest, + roots: StoreRootIdentities, + ) -> Self { + migration_intent_encoder::encode(catalog, inventory_digest, roots) + } + /// Returns the exact canonical intent bytes. pub const fn encoded(&self) -> &[u8] { &self.encoded diff --git a/src/adapters/store_migration/filesystem_inventory_reader.rs b/src/adapters/store_migration/filesystem_inventory_reader.rs index 06b00ea..7d989ab 100644 --- a/src/adapters/store_migration/filesystem_inventory_reader.rs +++ b/src/adapters/store_migration/filesystem_inventory_reader.rs @@ -14,6 +14,7 @@ use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; use super::{ ImmutablePoolInventoryDigest, StoreMigrationInventoryEntryCount, StoreMigrationInventoryHasher, }; +use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock, SegmentReadPolicy}; const SEGMENTS_NAME: &str = "segments"; @@ -29,6 +30,7 @@ pub struct FilesystemStoreMigrationInventoryReader { segments: PinnedMigrationPoolDirectory, catalogs: PinnedMigrationPoolDirectory, policy: SegmentReadPolicy, + root_identity: FilesystemRootIdentity, _lock: FilesystemWriterLock, } @@ -47,7 +49,7 @@ impl FilesystemStoreMigrationInventoryReader { admission: FilesystemPlatformAdmission, policy: SegmentReadPolicy, ) -> Result { - let lock = admission.into_lock(); + let (lock, root_identity) = admission.into_parts(); let root = lock.clone_directory() .map_err(|source| FilesystemMigrationInventoryError::Io { @@ -70,6 +72,7 @@ impl FilesystemStoreMigrationInventoryReader { segments, catalogs, policy, + root_identity, _lock: lock, }) } @@ -131,6 +134,18 @@ impl FilesystemStoreMigrationInventoryReader { self.segments.verify(&self.root)?; self.catalogs.verify(&self.root) } + + pub(super) const fn root(&self) -> &Dir { + &self.root + } + + pub(super) const fn catalogs(&self) -> &Dir { + self.catalogs.directory() + } + + pub(super) const fn root_identity(&self) -> FilesystemRootIdentity { + self.root_identity + } } fn hash_inventory( diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs new file mode 100644 index 0000000..e263f18 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority.rs @@ -0,0 +1,160 @@ +//! This module owns exact writer-locked filesystem migration authority. + +use super::filesystem_inventory_file::{self, FilesystemInventoryFilePolicy}; +use super::filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, + StoreRootIdentityCoordinate as RootCoordinate, +}; +use super::filesystem_migration_authority_validation::{ + artifact_error, require_root, verify_catalog, +}; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; +use super::store_root_identity::StoreRootIdentities; +use super::{CanonicalStoreMigrationIntent, FilesystemStoreMigrationInventoryReader}; +use crate::adapters::{ + CatalogRestartArtifact, CatalogRestartPhase, ChecksummedCatalog, ChecksummedPublicationHead, + FilesystemPlatformAdmission, SegmentReadPolicy, filesystem_initialization_namespace, + filesystem_platform_profile, physical_pool_name, +}; + +const HEAD_NAME: &str = "HEAD"; +const HEAD_LENGTH: u64 = 128; + +/// Exclusive authority to observe and migrate one pinned version-1 filesystem root. +/// +/// The authority retains the admitted writer lock and pinned root and immutable +/// pool capabilities for its entire lifetime. Its synchronous, +/// capability-relative filesystem I/O performs no protocol mutation and uses +/// neither a network nor an asynchronous runtime. +#[must_use] +pub struct FilesystemStoreMigrationAuthority { + inventory: FilesystemStoreMigrationInventoryReader, +} + +impl FilesystemStoreMigrationAuthority { + /// Pins one admitted filesystem root for migration observation. + /// + /// This synchronous constructor opens pinned directory capabilities but + /// materializes no artifact bodies and performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) + /// when the root capability cannot be cloned or either immutable pool + /// cannot be pinned without following links. + pub fn open( + admission: FilesystemPlatformAdmission, + policy: SegmentReadPolicy, + ) -> Result { + let inventory = FilesystemStoreMigrationInventoryReader::open(admission, policy) + .map_err(|source| Error::Inventory { source })?; + Ok(Self { inventory }) + } + + /// Observes one canonical intent from exact current version-1 authority. + /// + /// The synchronous call admits the exact published root namespace, physical + /// root coordinate, fixed-width head, complete immutable-pool inventory, + /// and head-selected catalog. Peak content allocation is bounded by one + /// catalog and one segment in addition to the bounded semantic inventory. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) + /// at the exact namespace, root, artifact, coordinate, or inventory refusal. + pub fn observe_intent(&self) -> Result { + self.verify_namespace()?; + let roots = self.verify_root_identity()?; + let head_bytes = self.read_head()?; + let head = ChecksummedPublicationHead::decode(&head_bytes) + .map_err(|source| Error::Head { source })?; + let inventory_digest = self + .inventory + .read() + .map_err(|source| Error::Inventory { source })?; + let coordinates = self.read_catalog(head)?; + if self.read_head()? != head_bytes { + return Err(Error::HeadChanged); + } + self.verify_namespace()?; + let _current_roots = self.verify_root_identity()?; + Ok(CanonicalStoreMigrationIntent::from_coordinates( + coordinates, + inventory_digest, + roots, + )) + } + + /// Re-observes and compares every coordinate in one canonical intent. + /// + /// This has the same synchronous I/O and bounded-allocation behavior as + /// [`Self::observe_intent`] and performs no protocol mutation. + /// + /// # Errors + /// + /// Returns the exact observation refusal or + /// [`FilesystemMigrationAuthorityError::IntentChanged`] with both intent + /// digests when current authority no longer reproduces `expected`. + pub fn verify_current(&self, expected: &CanonicalStoreMigrationIntent) -> Result<(), Error> { + let observed = self.observe_intent()?; + if &observed == expected { + Ok(()) + } else { + Err(Error::IntentChanged { + expected: expected.digest(), + observed: observed.digest(), + }) + } + } + + fn verify_namespace(&self) -> Result<(), Error> { + filesystem_initialization_namespace::admit_published(self.inventory.root()) + .map_err(|source| Error::Namespace { source }) + } + + fn verify_root_identity(&self) -> Result { + let expected = self.inventory.root_identity(); + let observed = filesystem_platform_profile::root_identity(self.inventory.root()) + .map_err(|source| Error::RootIdentity { source })?; + require_root(RootCoordinate::Device, expected.device(), observed.device())?; + require_root(RootCoordinate::Mount, expected.mount(), observed.mount())?; + require_root(RootCoordinate::File, expected.file(), observed.file())?; + Ok(StoreRootIdentities::from_filesystem(observed)) + } + + fn read_head(&self) -> Result, Error> { + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::OpenHead, + CatalogRestartPhase::ReadHead, + HEAD_LENGTH, + ); + filesystem_inventory_file::read(self.inventory.root(), HEAD_NAME, policy) + .map_err(|source| artifact_error(Artifact::Head, source)) + } + + fn read_catalog( + &self, + head: ChecksummedPublicationHead<'_>, + ) -> Result { + let artifact = Artifact::Catalog { + generation: head.generation(), + digest: head.catalog_digest(), + }; + let name = physical_pool_name::catalog(head.generation(), head.catalog_digest()); + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + CatalogRestartPhase::ReadCatalog, + head.catalog_length().get(), + ); + let bytes = filesystem_inventory_file::read(self.inventory.catalogs(), &name, policy) + .map_err(|source| artifact_error(artifact, source))?; + let catalog = ChecksummedCatalog::decode(&bytes).map_err(|source| Error::Catalog { + generation: head.generation(), + digest: head.catalog_digest(), + source: Box::new(source), + })?; + verify_catalog(head, catalog) + } +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_error.rs b/src/adapters/store_migration/filesystem_migration_authority_error.rs new file mode 100644 index 0000000..4e98601 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_error.rs @@ -0,0 +1,117 @@ +//! This boundary module owns filesystem migration-authority failures. + +use std::io; + +use super::{FilesystemMigrationInventoryError, StoreMigrationIntentDigest}; +use crate::adapters::{CatalogDecodeError, CatalogRestartError, PublicationHeadDecodeError}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Published version-1 artifact observed while establishing migration authority. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FilesystemMigrationAuthorityArtifact { + /// The mutable published `HEAD`. + Head, + /// The immutable catalog selected by `HEAD`. + Catalog { + /// Catalog generation named by `HEAD`. + generation: CatalogGeneration, + /// Catalog digest named by `HEAD`. + digest: CatalogDigest, + }, +} + +/// Physical store-root coordinate compared during migration admission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreRootIdentityCoordinate { + /// Platform device coordinate. + Device, + /// Platform mount coordinate. + Mount, + /// Platform file coordinate. + File, +} + +/// Failure to observe or revalidate exact filesystem migration authority. +#[derive(Debug)] +pub enum FilesystemMigrationAuthorityError { + /// Complete immutable-pool inventory could not be admitted. + Inventory { + /// Preserved inventory refusal. + source: FilesystemMigrationInventoryError, + }, + /// The exact published version-1 root namespace could not be admitted. + Namespace { + /// Preserved capability-relative filesystem source. + source: io::Error, + }, + /// The physical root identity could not be observed. + RootIdentity { + /// Preserved platform source. + source: io::Error, + }, + /// One physical root coordinate changed under retained authority. + RootIdentityChanged { + /// Coordinate that changed. + coordinate: StoreRootIdentityCoordinate, + /// Coordinate retained by platform admission. + expected: u64, + /// Coordinate observed immediately before migration. + observed: u64, + }, + /// One selected artifact could not be read completely. + Artifact { + /// Exact artifact being observed. + artifact: FilesystemMigrationAuthorityArtifact, + /// Preserved bounded-read refusal. + source: Box, + }, + /// One selected artifact changed physical identity while being read. + ArtifactChanged { + /// Exact artifact that changed. + artifact: FilesystemMigrationAuthorityArtifact, + }, + /// The published head bytes were malformed. + Head { + /// Preserved head-decoding refusal. + source: PublicationHeadDecodeError, + }, + /// The selected catalog bytes were malformed. + Catalog { + /// Catalog generation selected by the head. + generation: CatalogGeneration, + /// Catalog digest selected by the head. + digest: CatalogDigest, + /// Preserved catalog-decoding refusal. + source: Box, + }, + /// Head and selected catalog generation coordinates disagreed. + CatalogGeneration { + /// Generation required by the head. + expected: CatalogGeneration, + /// Generation observed in the catalog. + observed: CatalogGeneration, + }, + /// Head and selected catalog length coordinates disagreed. + CatalogLength { + /// Length required by the head. + expected: CatalogLength, + /// Length observed in the catalog. + observed: CatalogLength, + }, + /// Head and selected catalog digest coordinates disagreed. + CatalogDigest { + /// Digest required by the head. + expected: CatalogDigest, + /// Digest observed in the catalog. + observed: CatalogDigest, + }, + /// The mutable head changed during authority observation. + HeadChanged, + /// Re-observation did not reproduce the supplied canonical intent. + IntentChanged { + /// Intent authorized by the caller. + expected: StoreMigrationIntentDigest, + /// Intent derived from current filesystem authority. + observed: StoreMigrationIntentDigest, + }, +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_error_display.rs b/src/adapters/store_migration/filesystem_migration_authority_error_display.rs new file mode 100644 index 0000000..9fc8c14 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_error_display.rs @@ -0,0 +1,112 @@ +//! This module owns filesystem migration-authority error presentation. + +use std::error::Error; +use std::fmt; + +use super::filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, + StoreRootIdentityCoordinate, +}; + +impl fmt::Display for FilesystemMigrationAuthorityArtifact { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Head => formatter.write_str("HEAD"), + Self::Catalog { generation, digest } => { + write!(formatter, "catalog {generation:?}/{digest:?}") + } + } + } +} + +impl fmt::Display for StoreRootIdentityCoordinate { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Device => "device", + Self::Mount => "mount", + Self::File => "file", + }) + } +} + +impl fmt::Display for FilesystemMigrationAuthorityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Inventory { .. } => { + formatter.write_str("filesystem migration inventory was refused") + } + Self::Namespace { .. } => { + formatter.write_str("filesystem migration root namespace was refused") + } + Self::RootIdentity { .. } => { + formatter.write_str("filesystem migration root identity could not be observed") + } + Self::RootIdentityChanged { + coordinate, + expected, + observed, + } => write!( + formatter, + "filesystem migration root {coordinate} changed: expected {expected}, observed \ + {observed}" + ), + Self::Artifact { artifact, .. } => { + write!(formatter, "filesystem migration could not read {artifact}") + } + Self::ArtifactChanged { artifact } => { + write!( + formatter, + "filesystem migration {artifact} changed identity" + ) + } + Self::Head { .. } => formatter.write_str("filesystem migration HEAD was malformed"), + Self::Catalog { + generation, digest, .. + } => write!( + formatter, + "filesystem migration catalog {generation:?}/{digest:?} was malformed" + ), + Self::CatalogGeneration { expected, observed } => write!( + formatter, + "filesystem migration catalog generation disagreed: expected {expected:?}, \ + observed {observed:?}" + ), + Self::CatalogLength { expected, observed } => write!( + formatter, + "filesystem migration catalog length disagreed: expected {expected:?}, observed \ + {observed:?}" + ), + Self::CatalogDigest { expected, observed } => write!( + formatter, + "filesystem migration catalog digest disagreed: expected {expected:?}, observed \ + {observed:?}" + ), + Self::HeadChanged => { + formatter.write_str("filesystem migration HEAD changed during observation") + } + Self::IntentChanged { expected, observed } => write!( + formatter, + "filesystem migration intent changed: expected {expected:?}, observed {observed:?}" + ), + } + } +} + +impl Error for FilesystemMigrationAuthorityError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Inventory { source } => Some(source), + Self::Namespace { source } | Self::RootIdentity { source } => Some(source), + Self::Artifact { source, .. } => Some(source), + Self::Head { source } => Some(source), + Self::Catalog { source, .. } => Some(source), + Self::RootIdentityChanged { .. } + | Self::ArtifactChanged { .. } + | Self::CatalogGeneration { .. } + | Self::CatalogLength { .. } + | Self::CatalogDigest { .. } + | Self::HeadChanged + | Self::IntentChanged { .. } => None, + } + } +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_tests.rs b/src/adapters/store_migration/filesystem_migration_authority_tests.rs new file mode 100644 index 0000000..5afbc84 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_tests.rs @@ -0,0 +1,109 @@ +//! Writer-locked filesystem migration authority laws. + +use std::error::Error; +use std::fs; + +use super::FilesystemMigrationAuthorityError; +use super::filesystem_migration_authority::FilesystemStoreMigrationAuthority; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{AdmittedSegment, FilesystemPlatformAdmission, physical_pool_name}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); +const CATALOG_NAME: &str = + "0000000000000001-04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320.cat"; +const SEGMENT_NAME: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; +const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; + +#[test] +fn exact_published_v1_authority_constructs_and_revalidates_one_intent() -> Result<(), Box> +{ + let (sandbox, authority) = open_authority("migration-authority-current")?; + let intent = authority.observe_intent()?; + authority.verify_current(&intent)?; + + assert_eq!(intent.catalog_generation().get(), 1); + assert_eq!( + intent.inventory_digest().as_bytes().as_slice(), + decode_hex(INVENTORY_DIGEST)? + ); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn immutable_pool_drift_refuses_the_retained_intent() -> Result<(), Box> { + let (sandbox, authority) = open_authority("migration-authority-inventory-drift")?; + let intent = authority.observe_intent()?; + let bytes = decode_hex(EMPTY_SEGMENT_HEX.trim())?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + fs::write( + sandbox + .path() + .join("segments") + .join(physical_pool_name::segment(segment.digest())), + bytes, + )?; + + let error = authority + .verify_current(&intent) + .err() + .ok_or("changed inventory unexpectedly retained authority")?; + assert!(matches!( + error, + FilesystemMigrationAuthorityError::IntentChanged { expected, observed } + if expected == intent.digest() && observed != expected + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn version_two_namespace_evidence_refuses_before_mutation() -> Result<(), Box> { + let (sandbox, authority) = open_authority("migration-authority-v2-evidence")?; + let intent = authority.observe_intent()?; + fs::write(sandbox.path().join("FORMAT"), [])?; + + let error = authority + .verify_current(&intent) + .err() + .ok_or("version-two evidence unexpectedly retained authority")?; + assert!(matches!( + error, + FilesystemMigrationAuthorityError::Namespace { source } + if source.kind() == std::io::ErrorKind::InvalidData + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +fn open_authority( + name: &str, +) -> Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { + let sandbox = TestDirectory::create(name)?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + fs::write( + sandbox.path().join("segments").join(SEGMENT_NAME), + decode_hex(SEGMENT_HEX.trim())?, + )?; + fs::write( + sandbox.path().join("catalogs").join(CATALOG_NAME), + decode_hex(CATALOG_HEX.trim())?, + )?; + fs::write(sandbox.path().join("HEAD"), decode_hex(HEAD_HEX.trim())?)?; + let authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; + Ok((sandbox, authority)) +} + +const fn maximum_policy() -> crate::adapters::SegmentReadPolicy { + super::filesystem_inventory_catalogs_test_fixture::maximum_policy() +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_validation.rs b/src/adapters/store_migration/filesystem_migration_authority_validation.rs new file mode 100644 index 0000000..fe68a46 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_validation.rs @@ -0,0 +1,80 @@ +//! This module owns migration-authority coordinate validation. + +use super::filesystem_inventory_file::FilesystemInventoryFileError; +use super::filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, + StoreRootIdentityCoordinate as RootCoordinate, +}; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; +use crate::adapters::{ChecksummedCatalog, ChecksummedPublicationHead}; + +pub(super) const fn require_root( + coordinate: RootCoordinate, + expected: u64, + observed: u64, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::RootIdentityChanged { + coordinate, + expected, + observed, + }) + } +} + +pub(super) fn artifact_error(artifact: Artifact, source: FilesystemInventoryFileError) -> Error { + match source { + FilesystemInventoryFileError::Artifact(source) => Error::Artifact { artifact, source }, + FilesystemInventoryFileError::Changed => Error::ArtifactChanged { artifact }, + } +} + +pub(super) fn verify_catalog( + head: ChecksummedPublicationHead<'_>, + catalog: ChecksummedCatalog<'_>, +) -> Result { + require_generation(head.generation(), catalog.generation())?; + require_length(head.catalog_length(), catalog.length())?; + require_digest(head.catalog_digest(), catalog.digest())?; + Ok(MigrationCatalogCoordinates::new( + catalog.generation(), + catalog.length(), + catalog.digest(), + catalog.previous_catalog_digest(), + )) +} + +fn require_generation( + expected: crate::CatalogGeneration, + observed: crate::CatalogGeneration, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::CatalogGeneration { expected, observed }) + } +} + +fn require_length( + expected: crate::CatalogLength, + observed: crate::CatalogLength, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::CatalogLength { expected, observed }) + } +} + +fn require_digest( + expected: crate::CatalogDigest, + observed: crate::CatalogDigest, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::CatalogDigest { expected, observed }) + } +} diff --git a/src/adapters/store_migration/migration_catalog_coordinates.rs b/src/adapters/store_migration/migration_catalog_coordinates.rs new file mode 100644 index 0000000..d3baaaf --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_coordinates.rs @@ -0,0 +1,52 @@ +//! This module owns admitted catalog coordinates for intent encoding. + +use crate::{CatalogDigest, CatalogGeneration, CatalogLength, CatalogSnapshot}; + +#[derive(Clone, Copy)] +pub(super) struct MigrationCatalogCoordinates { + generation: CatalogGeneration, + length: CatalogLength, + digest: CatalogDigest, + predecessor: Option, +} + +impl MigrationCatalogCoordinates { + pub(super) const fn new( + generation: CatalogGeneration, + length: CatalogLength, + digest: CatalogDigest, + predecessor: Option, + ) -> Self { + Self { + generation, + length, + digest, + predecessor, + } + } + + pub(super) const fn from_snapshot(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Self { + Self::new( + snapshot.generation(), + snapshot.catalog_length(), + snapshot.catalog_digest(), + snapshot.previous_catalog_digest(), + ) + } + + pub(super) const fn generation(self) -> CatalogGeneration { + self.generation + } + + pub(super) const fn length(self) -> CatalogLength { + self.length + } + + pub(super) const fn digest(self) -> CatalogDigest { + self.digest + } + + pub(super) const fn predecessor(self) -> Option { + self.predecessor + } +} diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs index faa7572..1aebbf1 100644 --- a/src/adapters/store_migration/migration_intent_encoder.rs +++ b/src/adapters/store_migration/migration_intent_encoder.rs @@ -1,41 +1,27 @@ //! This boundary module owns canonical migration-intent encoding. use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; use super::migration_intent_format::StoreIdentifierFields; +use super::store_root_identity::StoreRootIdentities; use super::{ CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, - StoreIdentifier, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, - migration_intent_format as format, + StoreIdentifier, migration_intent_format as format, }; -use crate::CatalogSnapshot; - -#[derive(Clone, Copy)] -struct RootIdentities { - device: StoreRootDeviceIdentity, - mount: StoreRootMountIdentity, - file: StoreRootFileIdentity, -} pub(super) fn encode( - snapshot: &CatalogSnapshot<'_, '_, '_>, + catalog: MigrationCatalogCoordinates, inventory_digest: ImmutablePoolInventoryDigest, - root_device_identity: StoreRootDeviceIdentity, - root_mount_identity: StoreRootMountIdentity, - root_file_identity: StoreRootFileIdentity, + roots: StoreRootIdentities, ) -> CanonicalStoreMigrationIntent { let fields = StoreIdentifierFields { - catalog_generation: snapshot.generation(), - catalog_length: snapshot.catalog_length(), - catalog_digest: snapshot.catalog_digest(), - predecessor_catalog_digest: snapshot.previous_catalog_digest(), + catalog_generation: catalog.generation(), + catalog_length: catalog.length(), + catalog_digest: catalog.digest(), + predecessor_catalog_digest: catalog.predecessor(), inventory_digest, target_definition_digest: StoreFormatDefinitionDigest::VERSION_TWO, }; - let roots = RootIdentities { - device: root_device_identity, - mount: root_mount_identity, - file: root_file_identity, - }; let store_identifier = format::store_identifier(&fields); let mut encoded = [0_u8; format::ENCODED_LENGTH]; let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); @@ -50,9 +36,9 @@ pub(super) fn encode( catalog_digest: fields.catalog_digest, predecessor_catalog_digest: fields.predecessor_catalog_digest, inventory_digest: fields.inventory_digest, - root_device_identity: roots.device, - root_mount_identity: roots.mount, - root_file_identity: roots.file, + root_device_identity: roots.device(), + root_mount_identity: roots.mount(), + root_file_identity: roots.file(), target_definition_digest: fields.target_definition_digest, store_identifier, }, @@ -63,7 +49,7 @@ pub(super) fn encode( fn write_preimage( output: &mut [u8], fields: &StoreIdentifierFields, - roots: RootIdentities, + roots: StoreRootIdentities, store_identifier: StoreIdentifier, ) { let (magic, output) = output.split_at_mut(16); @@ -89,11 +75,11 @@ fn write_preimage( let (inventory_digest, output) = output.split_at_mut(32); inventory_digest.copy_from_slice(fields.inventory_digest.as_bytes()); let (device_identity, output) = output.split_at_mut(8); - device_identity.copy_from_slice(&roots.device.get().to_be_bytes()); + device_identity.copy_from_slice(&roots.device().get().to_be_bytes()); let (mount_identity, output) = output.split_at_mut(8); - mount_identity.copy_from_slice(&roots.mount.get().to_be_bytes()); + mount_identity.copy_from_slice(&roots.mount().get().to_be_bytes()); let (file_identity, output) = output.split_at_mut(8); - file_identity.copy_from_slice(&roots.file.get().to_be_bytes()); + file_identity.copy_from_slice(&roots.file().get().to_be_bytes()); let (definition_digest, store_identifier_slot) = output.split_at_mut(32); definition_digest.copy_from_slice(fields.target_definition_digest.as_bytes()); store_identifier_slot.copy_from_slice(store_identifier.as_bytes()); diff --git a/src/adapters/store_migration/store_root_identity.rs b/src/adapters/store_migration/store_root_identity.rs index 39bc4bf..3ddce7b 100644 --- a/src/adapters/store_migration/store_root_identity.rs +++ b/src/adapters/store_migration/store_root_identity.rs @@ -1,5 +1,7 @@ //! This module owns physical store-root recovery coordinates. +use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; + macro_rules! root_identity { ($name:ident, $documentation:literal) => { #[doc = $documentation] @@ -36,3 +38,44 @@ root_identity!( StoreRootFileIdentity, "Platform file identity bound into a migration intent." ); + +#[derive(Clone, Copy)] +pub(super) struct StoreRootIdentities { + device: StoreRootDeviceIdentity, + mount: StoreRootMountIdentity, + file: StoreRootFileIdentity, +} + +impl StoreRootIdentities { + pub(super) const fn new( + device: StoreRootDeviceIdentity, + mount: StoreRootMountIdentity, + file: StoreRootFileIdentity, + ) -> Self { + Self { + device, + mount, + file, + } + } + + pub(super) const fn from_filesystem(identity: FilesystemRootIdentity) -> Self { + Self::new( + StoreRootDeviceIdentity::from_admitted(identity.device()), + StoreRootMountIdentity::from_admitted(identity.mount()), + StoreRootFileIdentity::from_admitted(identity.file()), + ) + } + + pub(super) const fn device(self) -> StoreRootDeviceIdentity { + self.device + } + + pub(super) const fn mount(self) -> StoreRootMountIdentity { + self.mount + } + + pub(super) const fn file(self) -> StoreRootFileIdentity { + self.file + } +} diff --git a/src/lib.rs b/src/lib.rs index f82f81d..18cd737 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,6 +68,7 @@ pub use adapters::{ CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, @@ -75,27 +76,28 @@ pub use adapters::{ FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationInventoryReader, - FilesystemWriterLock, ImmutablePoolInventoryDigest, InitialGcStateDigest, - InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, MigrationInventoryNamespace, - MigrationInventoryPool, MigrationSynchronizationMask, OpenedReusableSegment, - PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, - RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, - RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, - RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, - RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, - RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, - RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationAuthority, + FilesystemStoreMigrationInventoryReader, FilesystemWriterLock, ImmutablePoolInventoryDigest, + InitialGcStateDigest, InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + MigrationInventoryNamespace, MigrationInventoryPool, MigrationSynchronizationMask, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -117,9 +119,10 @@ pub use adapters::{ StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, StoreMigrationStorage, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, + StoreRootFileIdentity, StoreRootIdentityCoordinate, StoreRootMountIdentity, + WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, + assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, + classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, execute_recovery_stage_completion, execute_recovery_stage_discard, execute_store_migration, fingerprint_recovery_stage, initialize_store, plan_recovery_next_head_finalization, From 5b26a1dbc9095f853bd4fda63e69195b74353228 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:17:37 -0700 Subject: [PATCH 43/50] Stream recovery reads and reject trailing artifacts --- src/adapters/catalog_restart_io.rs | 153 ++++++++++++- ...lesystem_recovery_stage_materialization.rs | 213 +++++++++++++++++- 2 files changed, 351 insertions(+), 15 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 0a62c94..41513c6 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -9,6 +9,8 @@ use cap_std::fs::{Dir, File, OpenOptions}; use super::{CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase}; +const CATALOG_RESTART_READ_BUFFER_LENGTH: usize = 8_192; + pub(super) fn open_root(root: &Path) -> Result { Dir::open_ambient_dir(root, ambient_authority()) .map_err(|source| CatalogRestartError::io(CatalogRestartPhase::OpenRoot, source)) @@ -46,6 +48,7 @@ pub(super) fn read_exact( byte_count: expected, source: None, })?; + let mut encoded = Vec::new(); encoded .try_reserve_exact(host_length) @@ -54,22 +57,79 @@ pub(super) fn read_exact( byte_count: expected, source: Some(source), })?; - encoded.resize(host_length, 0); - file.read_exact(&mut encoded) - .map_err(|source| CatalogRestartError::io(phase, source))?; - reject_trailing_bytes(&mut file, artifact, phase, expected)?; + read_exact_to(&mut file, artifact, phase, expected, |chunk| { + encoded.extend_from_slice(chunk); + Ok(()) + })?; Ok(encoded) } -fn reject_trailing_bytes( - file: &mut File, +pub(super) fn read_exact_to( + source: &mut R, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, + mut on_chunk: F, +) -> Result<(), CatalogRestartError> +where + R: Read, + F: FnMut(&[u8]) -> Result<(), CatalogRestartError>, +{ + let mut observed = 0_u64; + let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; + let chunk_length = u64::try_from(buffer.len()) + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + + while observed < expected { + let remaining = expected + .checked_sub(observed) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + let offered = remaining + .min(chunk_length) + .try_into() + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + let read_buffer = buffer + .get_mut(..offered) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + match source.read(read_buffer) { + Ok(0) => { + return Err(CatalogRestartError::io( + phase, + io::Error::new( + io::ErrorKind::UnexpectedEof, + "restart artifact ended before the expected boundary", + ), + )); + } + Ok(count) => { + let bytes = read_buffer + .get(..count) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + on_chunk(bytes)?; + let increment = u64::try_from(count).map_err(|_source| { + CatalogRestartError::LengthArithmetic { artifact, expected } + })?; + observed = observed + .checked_add(increment) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} + Err(source) => return Err(CatalogRestartError::io(phase, source)), + } + } + reject_trailing_bytes(source, artifact, phase, expected)?; + Ok(()) +} + +fn reject_trailing_bytes( + source: &mut R, artifact: CatalogRestartArtifact, phase: CatalogRestartPhase, expected: u64, ) -> Result<(), CatalogRestartError> { let mut trailing = [0_u8; 1]; loop { - match file.read(&mut trailing) { + match source.read(&mut trailing) { Ok(0) => return Ok(()), Ok(observed) => { let increment = u64::try_from(observed).map_err(|_source| { @@ -90,3 +150,82 @@ fn reject_trailing_bytes( } } } + +#[cfg(test)] +mod tests { + use std::io::{Cursor, ErrorKind}; + + use super::*; + + #[test] + fn read_exact_to_streams_chunks() { + let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); + let mut observed = Vec::>::new(); + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 7, + |chunk| { + observed.push(chunk.to_vec()); + Ok(()) + }, + ); + + assert!(result.is_ok()); + assert_eq!(observed.concat(), b"abcdefg"); + } + + #[test] + fn read_exact_to_rejects_short_artifacts() { + let mut source = Cursor::new(vec![b'a', b'b']); + let mut seen = 0_u8; + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + |_chunk| { + seen = seen.checked_add(1).expect("unexpected chunk overflow"); + Ok(()) + }, + ); + + let error = result.unwrap_err(); + assert_eq!(seen, 1); + assert!(matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::ReadCatalog, + ref source, + } if source.kind() == ErrorKind::UnexpectedEof + )); + } + + #[test] + fn read_exact_to_rejects_trailing_bytes() { + let mut source = Cursor::new(vec![b'a', b'b', b'c']); + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + |_| Ok(()), + ); + + let error = result.unwrap_err(); + let expected = 2_u64; + assert!(matches!( + error, + CatalogRestartError::Length { + artifact: CatalogRestartArtifact::Head, + minimum, + maximum, + observed: 3 + } if minimum == expected && maximum == expected + )); + } +} diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index 0f769a2..d005b48 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -1,6 +1,6 @@ //! This module owns exact writable recovery-stage materialization. -use std::io::{Read, Seek, SeekFrom}; +use std::io::{self, Read, Seek, SeekFrom}; use cap_std::fs::File; @@ -14,12 +14,7 @@ pub(super) fn read_and_position( let mut encoded = allocate(stage, length)?; file.seek(SeekFrom::Start(0)) .map_err(|source| FilesystemRecoveryStageError::Position { stage, source })?; - file.read_exact(&mut encoded) - .map_err(|source| FilesystemRecoveryStageError::Materialize { - stage, - expected: length, - source, - })?; + read_exact(file, stage, length, &mut encoded)?; verify_position(file, stage, length)?; Ok(encoded.into_boxed_slice()) } @@ -42,10 +37,87 @@ fn allocate( source, } })?; - encoded.resize(host_length, 0); Ok(encoded) } +fn read_exact( + file: &mut File, + stage: RecoveryStage, + length: RecoveryStageLength, + encoded: &mut Vec, +) -> Result<(), FilesystemRecoveryStageError> { + let expected = length.get(); + let observed = file + .by_ref() + .take(expected) + .read_to_end(encoded) + .map_err(|source| FilesystemRecoveryStageError::Materialize { + stage, + expected: length, + source, + })?; + let observed = + u64::try_from(observed).map_err(|_source| FilesystemRecoveryStageError::LengthChanged { + stage, + expected: length, + observed: expected, + })?; + if observed < expected { + return Err(FilesystemRecoveryStageError::Materialize { + stage, + expected: length, + source: io::Error::new( + io::ErrorKind::UnexpectedEof, + "recovery stage ended before the expected boundary", + ), + }); + } + reject_trailing_bytes(file, stage, length)?; + Ok(()) +} + +fn reject_trailing_bytes( + file: &mut File, + stage: RecoveryStage, + expected: RecoveryStageLength, +) -> Result<(), FilesystemRecoveryStageError> { + let mut trailing = [0_u8; 1]; + loop { + match file.read(&mut trailing) { + Ok(0) => return Ok(()), + Ok(read_bytes) => { + let increment = u64::try_from(read_bytes).map_err(|_source| { + FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed: expected.get(), + } + })?; + let observed = expected.get().checked_add(increment).ok_or( + FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed: expected.get(), + }, + )?; + return Err(FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed, + }); + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + Err(source) => { + return Err(FilesystemRecoveryStageError::Materialize { + stage, + expected, + source, + }); + } + } + } +} + pub(super) fn verify_position( file: &mut File, stage: RecoveryStage, @@ -64,3 +136,128 @@ pub(super) fn verify_position( }) } } + +#[cfg(test)] +mod tests { + use std::error::Error; + use std::fs; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + + use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; + use cap_std::fs::OpenOptions; + use cap_std::{ambient_authority, fs::Dir}; + + use super::*; + + static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + + #[test] + fn read_exact_reads_expected_bytes_without_trailing() -> Result<(), Box> { + let sandbox = TestDirectory::create("stage-materialization-exact")?; + let path = sandbox.path().join("stage.bin"); + fs::write(&path, b"abcdef")?; + let mut file = open_for_tests(&path)?; + let encoded = super::read_and_position( + &mut file, + RecoveryStage::Segment, + RecoveryStageLength::from_validated(6), + )?; + assert_eq!(encoded.as_ref(), b"abcdef"); + drop(file); + sandbox.remove()?; + Ok(()) + } + + #[test] + fn read_and_position_rejects_short_stage() -> Result<(), Box> { + let sandbox = TestDirectory::create("stage-materialization-short")?; + let path = sandbox.path().join("stage.bin"); + fs::write(&path, b"abc")?; + let mut file = open_for_tests(&path)?; + let error = super::read_and_position( + &mut file, + RecoveryStage::Segment, + RecoveryStageLength::from_validated(5), + ) + .expect_err("short stage materialization was admitted"); + + assert!(matches!( + error, + FilesystemRecoveryStageError::Materialize { + stage: RecoveryStage::Segment, + expected, + source, + } if expected.get() == 5 && source.kind() == std::io::ErrorKind::UnexpectedEof + )); + drop(file); + sandbox.remove()?; + Ok(()) + } + + #[test] + fn read_and_position_rejects_trailing_bytes() -> Result<(), Box> { + let sandbox = TestDirectory::create("stage-materialization-trailing")?; + let path = sandbox.path().join("stage.bin"); + fs::write(&path, b"abcdef")?; + let mut file = open_for_tests(&path)?; + let error = super::read_and_position( + &mut file, + RecoveryStage::Segment, + RecoveryStageLength::from_validated(3), + ) + .expect_err("trailing-stage materialization was admitted"); + + assert!(matches!( + error, + FilesystemRecoveryStageError::LengthChanged { + stage: RecoveryStage::Segment, + expected, + observed: 4, + } if expected.get() == 3 + )); + drop(file); + sandbox.remove()?; + Ok(()) + } + + fn open_for_tests(path: &PathBuf) -> Result> { + let directory = Dir::open_ambient_dir( + path.parent() + .expect("directory parent exists for stage fixture"), + ambient_authority(), + )?; + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + let file = directory.open_with( + path.file_name() + .expect("file path has file name for fixture") + .to_str() + .expect("file name is UTF-8 for fixture"), + &options, + )?; + Ok(file) + } + + struct TestDirectory { + path: PathBuf, + } + + impl TestDirectory { + fn create(name: &str) -> std::io::Result { + let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("keep-{name}-{}-{sequence}", std::process::id())); + fs::create_dir(&path)?; + Ok(Self { path }) + } + + fn path(&self) -> &std::path::Path { + &self.path + } + + fn remove(self) -> std::io::Result<()> { + fs::remove_dir_all(self.path) + } + } +} From 2fffbdca475b15d7daa2c4a5df42ec61ba0f5f86 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:27:42 -0700 Subject: [PATCH 44/50] Test: add streaming large-input callback memory harness --- src/adapters/catalog_restart_io.rs | 158 ++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 41513c6..140d5e1 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -153,10 +153,52 @@ fn reject_trailing_bytes( #[cfg(test)] mod tests { - use std::io::{Cursor, ErrorKind}; + use std::io; + use std::io::{Cursor, ErrorKind, Read}; + use std::mem::size_of; use super::*; + #[test] + fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() { + const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; + const READER_STRIDE: u64 = 2_u64 * 1024; + const CALLBACK_BUDGET_BYTES: usize = 16 * 1024; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + |chunk| budget.consume(chunk), + ); + + assert!(result.is_ok(), "{result:?}"); + assert!(budget.observed_bytes() > 0); + assert_eq!(budget.observed_bytes(), TOTAL_BYTES); + assert!( + budget.max_chunk() >= READER_STRIDE as usize, + "reader stride should be observed" + ); + assert!( + budget.max_chunk() <= budget.callback_limit(), + "callback should remain in budget" + ); + assert!( + budget.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH, + "read buffer bounds should hold" + ); + assert!(budget.total_chunks() > 0); + assert!( + size_of::() < 128, + "callback state should stay compact" + ); + assert_eq!(budget.observed_bytes(), TOTAL_BYTES); + } + #[test] fn read_exact_to_streams_chunks() { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); @@ -228,4 +270,118 @@ mod tests { } if minimum == expected && maximum == expected )); } + + struct SyntheticStreamingReader { + remaining: u64, + emit_stride: u64, + } + + impl SyntheticStreamingReader { + fn new(total: u64, emit_stride: u64) -> Self { + Self { + remaining: total, + emit_stride, + } + } + } + + impl Read for SyntheticStreamingReader { + fn read(&mut self, sink: &mut [u8]) -> io::Result { + if self.remaining == 0 { + return Ok(0); + } + + let sink_capacity = match u64::try_from(sink.len()) { + Ok(capacity) => capacity, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink capacity exceeds supported range", + )); + } + }; + let emitted: usize = match self + .emit_stride + .min(self.remaining) + .min(sink_capacity) + .try_into() + { + Ok(size) => size, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "requested read size exceeds supported range", + )); + } + }; + + sink[..emitted].fill(0x5a); + self.remaining -= u64::try_from(emitted) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; + Ok(emitted) + } + } + + struct StreamingCallbackBudget { + observed_bytes: u64, + total_chunks: u64, + max_chunk: usize, + callback_limit: usize, + expected_total: u64, + } + + impl StreamingCallbackBudget { + fn new(expected_total: u64, callback_limit: usize) -> Self { + Self { + observed_bytes: 0, + total_chunks: 0, + max_chunk: 0, + callback_limit, + expected_total, + } + } + + fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { + self.total_chunks = + self.total_chunks + .checked_add(1) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + self.max_chunk = self.max_chunk.max(chunk.len()); + + self.observed_bytes = self + .observed_bytes + .checked_add(u64::try_from(chunk.len()).map_err(|_source| { + CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + } + })?) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + Ok(()) + } + + fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + fn max_chunk(&self) -> usize { + self.max_chunk + } + + fn callback_limit(&self) -> usize { + self.callback_limit + } + + fn total_chunks(&self) -> u64 { + self.total_chunks + } + } } From ee876cf6a4ea911597ba4461b7005f62b6afd0f2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:33:24 -0700 Subject: [PATCH 45/50] Test: add streaming write_exact_to regression coverage --- src/adapters/catalog_restart_io.rs | 198 ++++++++++++++++++++++++++++- 1 file changed, 196 insertions(+), 2 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 140d5e1..9bbccfa 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,4 +1,5 @@ -//! This module owns exact capability-relative restart artifact reads. +//! This module owns exact capability-relative restart artifact reads and +//! bounded streaming writes. use std::io::{self, Read}; use std::path::Path; @@ -64,6 +65,66 @@ pub(super) fn read_exact( Ok(encoded) } +#[cfg(test)] +pub(super) fn write_exact_to( + source: &mut R, + destination: &mut W, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, +) -> Result<(), CatalogRestartError> +where + R: Read, + W: io::Write, +{ + let mut observed = 0_u64; + let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; + let chunk_length = u64::try_from(buffer.len()) + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + + while observed < expected { + let remaining = expected + .checked_sub(observed) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + let offered = remaining + .min(chunk_length) + .try_into() + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + let read_buffer = buffer + .get_mut(..offered) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + match source.read(read_buffer) { + Ok(0) => { + return Err(CatalogRestartError::io( + phase, + io::Error::new( + io::ErrorKind::UnexpectedEof, + "restart artifact ended before the expected boundary", + ), + )); + } + Ok(count) => { + let bytes = read_buffer + .get(..count) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + destination + .write_all(bytes) + .map_err(|source| CatalogRestartError::io(phase, source))?; + let increment = u64::try_from(count).map_err(|_source| { + CatalogRestartError::LengthArithmetic { artifact, expected } + })?; + observed = observed + .checked_add(increment) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} + Err(source) => return Err(CatalogRestartError::io(phase, source)), + } + } + reject_trailing_bytes(source, artifact, phase, expected)?; + Ok(()) +} + pub(super) fn read_exact_to( source: &mut R, artifact: CatalogRestartArtifact, @@ -154,7 +215,7 @@ fn reject_trailing_bytes( #[cfg(test)] mod tests { use std::io; - use std::io::{Cursor, ErrorKind, Read}; + use std::io::{Cursor, ErrorKind, Read, Write}; use std::mem::size_of; use super::*; @@ -199,6 +260,32 @@ mod tests { assert_eq!(budget.observed_bytes(), TOTAL_BYTES); } + #[test] + fn write_exact_to_streams_large_virtual_file_with_small_writer_state() { + const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; + const READER_STRIDE: u64 = 2_u64 * 1024; + const WRITER_BUDGET_BYTES: usize = 4 * 1024; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); + + let result = write_exact_to( + &mut source, + &mut sink, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + ); + + assert!(result.is_ok(), "{result:?}"); + assert_eq!(sink.observed_bytes(), TOTAL_BYTES); + assert!(sink.total_chunks() > 0); + assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); + assert!(sink.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH); + assert!(sink.max_chunk() >= READER_STRIDE as usize); + assert!(size_of::() < 64); + } + #[test] fn read_exact_to_streams_chunks() { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); @@ -271,6 +358,57 @@ mod tests { )); } + #[test] + fn write_exact_to_rejects_short_artifacts() { + let mut source = Cursor::new(vec![b'a', b'b']); + let mut sink = StreamingWriteSink::new(16 * 1024); + + let result = write_exact_to( + &mut source, + &mut sink, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + ); + + let error = result.unwrap_err(); + assert_eq!(sink.observed_bytes(), 2); + assert!(matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::ReadCatalog, + ref source, + } if source.kind() == ErrorKind::UnexpectedEof + )); + } + + #[test] + fn write_exact_to_rejects_trailing_bytes() { + let mut source = Cursor::new(vec![b'a', b'b', b'c']); + let mut sink = StreamingWriteSink::new(16 * 1024); + + let result = write_exact_to( + &mut source, + &mut sink, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + ); + + let error = result.unwrap_err(); + let expected = 2_u64; + assert_eq!(sink.observed_bytes(), 2); + assert!(matches!( + error, + CatalogRestartError::Length { + artifact: CatalogRestartArtifact::Head, + minimum, + maximum, + observed: 3 + } if minimum == expected && maximum == expected + )); + } + struct SyntheticStreamingReader { remaining: u64, emit_stride: u64, @@ -384,4 +522,60 @@ mod tests { self.total_chunks } } + + struct StreamingWriteSink { + observed_bytes: u64, + observed_chunks: u64, + max_chunk: usize, + writer_memory_limit: usize, + } + + impl StreamingWriteSink { + fn new(writer_memory_limit: usize) -> Self { + Self { + observed_bytes: 0, + observed_chunks: 0, + max_chunk: 0, + writer_memory_limit, + } + } + + fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + fn total_chunks(&self) -> u64 { + self.observed_chunks + } + + fn max_chunk(&self) -> usize { + self.max_chunk + } + } + + impl Write for StreamingWriteSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.observed_chunks = self.observed_chunks.checked_add(1).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow") + })?; + let observed = u64::try_from(bytes.len()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow") + })?; + self.observed_bytes = self.observed_bytes.checked_add(observed).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "write count overflow") + })?; + self.max_chunk = self.max_chunk.max(bytes.len()); + if bytes.len() > self.writer_memory_limit { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink memory budget exceeded", + )); + } + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } } From a0f60f63549b66141cd8870bb7fe11903bd74e7c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:39:52 -0700 Subject: [PATCH 46/50] refactor: shape catalog restart streaming transfer API --- src/adapters/catalog_restart_io.rs | 206 +++++++++++++++-------------- 1 file changed, 108 insertions(+), 98 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 9bbccfa..a54e532 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,5 +1,5 @@ //! This module owns exact capability-relative restart artifact reads and -//! bounded streaming writes. +//! bounded, exact-transfer streaming. use std::io::{self, Read}; use std::path::Path; @@ -12,6 +12,39 @@ use super::{CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase}; const CATALOG_RESTART_READ_BUFFER_LENGTH: usize = 8_192; +#[derive(Clone, Copy, Debug)] +pub(super) struct ExactTransfer { + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, +} + +impl ExactTransfer { + const fn new( + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, + ) -> Self { + Self { + artifact, + phase, + expected, + } + } + + fn artifact(&self) -> CatalogRestartArtifact { + self.artifact + } + + fn phase(&self) -> CatalogRestartPhase { + self.phase + } + + fn expected(&self) -> u64 { + self.expected + } +} + pub(super) fn open_root(root: &Path) -> Result { Dir::open_ambient_dir(root, ambient_authority()) .map_err(|source| CatalogRestartError::io(CatalogRestartPhase::OpenRoot, source)) @@ -43,99 +76,59 @@ pub(super) fn read_exact( phase: CatalogRestartPhase, expected: u64, ) -> Result, CatalogRestartError> { - let host_length = - usize::try_from(expected).map_err(|_source| CatalogRestartError::Allocation { - artifact, - byte_count: expected, + let transfer = ExactTransfer::new(artifact, phase, expected); + let host_length = usize::try_from(transfer.expected()).map_err(|_source| { + CatalogRestartError::Allocation { + artifact: transfer.artifact(), + byte_count: transfer.expected(), source: None, - })?; + } + })?; let mut encoded = Vec::new(); encoded .try_reserve_exact(host_length) .map_err(|source| CatalogRestartError::Allocation { - artifact, + artifact: transfer.artifact(), byte_count: expected, source: Some(source), })?; - read_exact_to(&mut file, artifact, phase, expected, |chunk| { + copy_exact_to_chunks(&mut file, transfer, |chunk| { encoded.extend_from_slice(chunk); Ok(()) })?; Ok(encoded) } -#[cfg(test)] -pub(super) fn write_exact_to( +pub(super) fn copy_exact( source: &mut R, destination: &mut W, - artifact: CatalogRestartArtifact, - phase: CatalogRestartPhase, - expected: u64, -) -> Result<(), CatalogRestartError> + transfer: ExactTransfer, +) -> Result where R: Read, W: io::Write, { - let mut observed = 0_u64; - let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; - let chunk_length = u64::try_from(buffer.len()) - .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; - - while observed < expected { - let remaining = expected - .checked_sub(observed) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - let offered = remaining - .min(chunk_length) - .try_into() - .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; - let read_buffer = buffer - .get_mut(..offered) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - match source.read(read_buffer) { - Ok(0) => { - return Err(CatalogRestartError::io( - phase, - io::Error::new( - io::ErrorKind::UnexpectedEof, - "restart artifact ended before the expected boundary", - ), - )); - } - Ok(count) => { - let bytes = read_buffer - .get(..count) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - destination - .write_all(bytes) - .map_err(|source| CatalogRestartError::io(phase, source))?; - let increment = u64::try_from(count).map_err(|_source| { - CatalogRestartError::LengthArithmetic { artifact, expected } - })?; - observed = observed - .checked_add(increment) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - } - Err(source) if source.kind() == io::ErrorKind::Interrupted => {} - Err(source) => return Err(CatalogRestartError::io(phase, source)), - } - } - reject_trailing_bytes(source, artifact, phase, expected)?; - Ok(()) + copy_exact_to_chunks(source, transfer, |chunk| { + destination + .write_all(chunk) + .map_err(|source| CatalogRestartError::io(transfer.phase(), source)) + }) } -pub(super) fn read_exact_to( +pub(super) fn copy_exact_to_chunks( source: &mut R, - artifact: CatalogRestartArtifact, - phase: CatalogRestartPhase, - expected: u64, + transfer: ExactTransfer, mut on_chunk: F, -) -> Result<(), CatalogRestartError> +) -> Result where R: Read, F: FnMut(&[u8]) -> Result<(), CatalogRestartError>, { + let artifact = transfer.artifact(); + let phase = transfer.phase(); + let expected = transfer.expected(); + let mut observed = 0_u64; let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; let chunk_length = u64::try_from(buffer.len()) @@ -179,7 +172,7 @@ where } } reject_trailing_bytes(source, artifact, phase, expected)?; - Ok(()) + Ok(observed) } fn reject_trailing_bytes( @@ -229,11 +222,13 @@ mod tests { let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - TOTAL_BYTES, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + ), |chunk| budget.consume(chunk), ); @@ -261,7 +256,7 @@ mod tests { } #[test] - fn write_exact_to_streams_large_virtual_file_with_small_writer_state() { + fn copy_exact_streams_large_virtual_file_with_small_writer_state() { const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; const READER_STRIDE: u64 = 2_u64 * 1024; const WRITER_BUDGET_BYTES: usize = 4 * 1024; @@ -269,15 +264,20 @@ mod tests { let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); - let result = write_exact_to( + let result = copy_exact( &mut source, &mut sink, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - TOTAL_BYTES, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + ), ); - assert!(result.is_ok(), "{result:?}"); + let observed = result.unwrap_or_else(|error| { + panic!("copy should succeed for expected length: {error:?}"); + }); + assert_eq!(observed, TOTAL_BYTES); assert_eq!(sink.observed_bytes(), TOTAL_BYTES); assert!(sink.total_chunks() > 0); assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); @@ -291,11 +291,13 @@ mod tests { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); let mut observed = Vec::>::new(); - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 7, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 7, + ), |chunk| { observed.push(chunk.to_vec()); Ok(()) @@ -311,11 +313,13 @@ mod tests { let mut source = Cursor::new(vec![b'a', b'b']); let mut seen = 0_u8; - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 4, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + ), |_chunk| { seen = seen.checked_add(1).expect("unexpected chunk overflow"); Ok(()) @@ -337,11 +341,13 @@ mod tests { fn read_exact_to_rejects_trailing_bytes() { let mut source = Cursor::new(vec![b'a', b'b', b'c']); - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 2, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + ), |_| Ok(()), ); @@ -359,16 +365,18 @@ mod tests { } #[test] - fn write_exact_to_rejects_short_artifacts() { + fn copy_exact_rejects_short_artifacts() { let mut source = Cursor::new(vec![b'a', b'b']); let mut sink = StreamingWriteSink::new(16 * 1024); - let result = write_exact_to( + let result = copy_exact( &mut source, &mut sink, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 4, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + ), ); let error = result.unwrap_err(); @@ -383,16 +391,18 @@ mod tests { } #[test] - fn write_exact_to_rejects_trailing_bytes() { + fn copy_exact_rejects_trailing_bytes() { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let mut sink = StreamingWriteSink::new(16 * 1024); - let result = write_exact_to( + let result = copy_exact( &mut source, &mut sink, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 2, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + ), ); let error = result.unwrap_err(); From 593e448b8c71d0d128bfb699a20c1c96e69f544e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:42:09 -0700 Subject: [PATCH 47/50] feat: expose transfer-specific restart IO copy API --- src/adapters/catalog_restart_io.rs | 39 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index a54e532..bca44ce 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,7 +1,7 @@ //! This module owns exact capability-relative restart artifact reads and //! bounded, exact-transfer streaming. -use std::io::{self, Read}; +use std::io::{self, Read, Write}; use std::path::Path; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; @@ -20,7 +20,7 @@ pub(super) struct ExactTransfer { } impl ExactTransfer { - const fn new( + pub(super) const fn new( artifact: CatalogRestartArtifact, phase: CatalogRestartPhase, expected: u64, @@ -32,15 +32,15 @@ impl ExactTransfer { } } - fn artifact(&self) -> CatalogRestartArtifact { + pub(super) const fn artifact(&self) -> CatalogRestartArtifact { self.artifact } - fn phase(&self) -> CatalogRestartPhase { + pub(super) const fn phase(&self) -> CatalogRestartPhase { self.phase } - fn expected(&self) -> u64 { + pub(super) const fn expected(&self) -> u64 { self.expected } } @@ -93,10 +93,10 @@ pub(super) fn read_exact( byte_count: expected, source: Some(source), })?; - copy_exact_to_chunks(&mut file, transfer, |chunk| { - encoded.extend_from_slice(chunk); - Ok(()) - })?; + let mut sink = VecWrite { + encoded: &mut encoded, + }; + copy_exact(&mut file, &mut sink, transfer)?; Ok(encoded) } @@ -205,6 +205,21 @@ fn reject_trailing_bytes( } } +struct VecWrite<'a> { + encoded: &'a mut Vec, +} + +impl Write for VecWrite<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.encoded.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + #[cfg(test)] mod tests { use std::io; @@ -287,7 +302,7 @@ mod tests { } #[test] - fn read_exact_to_streams_chunks() { + fn copy_exact_to_chunks_streams() { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); let mut observed = Vec::>::new(); @@ -309,7 +324,7 @@ mod tests { } #[test] - fn read_exact_to_rejects_short_artifacts() { + fn copy_exact_to_chunks_rejects_short_artifacts() { let mut source = Cursor::new(vec![b'a', b'b']); let mut seen = 0_u8; @@ -338,7 +353,7 @@ mod tests { } #[test] - fn read_exact_to_rejects_trailing_bytes() { + fn copy_exact_to_chunks_rejects_trailing_bytes() { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let result = copy_exact_to_chunks( From 57aaa76be49f1745fa140f24f61089fdeb562935 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:59:00 -0700 Subject: [PATCH 48/50] Fix: enforce checked conversions in streaming IO adapters --- src/adapters/catalog_restart_io.rs | 132 ++++++++++++------ ...lesystem_recovery_stage_materialization.rs | 62 ++++---- src/adapters/filesystem_root_identity.rs | 4 + 3 files changed, 132 insertions(+), 66 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index bca44ce..cfd32ae 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -222,6 +222,7 @@ impl Write for VecWrite<'_> { #[cfg(test)] mod tests { + use std::error::Error; use std::io; use std::io::{Cursor, ErrorKind, Read, Write}; use std::mem::size_of; @@ -229,15 +230,22 @@ mod tests { use super::*; #[test] - fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() { + fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() + -> Result<(), Box> { const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; - const READER_STRIDE: u64 = 2_u64 * 1024; + const READER_STRIDE_BYTES: usize = 2_usize * 1024; const CALLBACK_BUDGET_BYTES: usize = 16 * 1024; - - let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let Ok(reader_stride) = u64::try_from(READER_STRIDE_BYTES) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "reader stride is outside supported range", + ))); + }; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, reader_stride); let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); - let result = copy_exact_to_chunks( + let _observed = copy_exact_to_chunks( &mut source, ExactTransfer::new( CatalogRestartArtifact::Head, @@ -245,13 +253,12 @@ mod tests { TOTAL_BYTES, ), |chunk| budget.consume(chunk), - ); + )?; - assert!(result.is_ok(), "{result:?}"); assert!(budget.observed_bytes() > 0); assert_eq!(budget.observed_bytes(), TOTAL_BYTES); assert!( - budget.max_chunk() >= READER_STRIDE as usize, + budget.max_chunk() >= READER_STRIDE_BYTES, "reader stride should be observed" ); assert!( @@ -268,18 +275,26 @@ mod tests { "callback state should stay compact" ); assert_eq!(budget.observed_bytes(), TOTAL_BYTES); + Ok(()) } #[test] - fn copy_exact_streams_large_virtual_file_with_small_writer_state() { + fn copy_exact_streams_large_virtual_file_with_small_writer_state() -> Result<(), Box> + { const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; - const READER_STRIDE: u64 = 2_u64 * 1024; + const READER_STRIDE_BYTES: usize = 2_usize * 1024; const WRITER_BUDGET_BYTES: usize = 4 * 1024; - - let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let Ok(reader_stride) = u64::try_from(READER_STRIDE_BYTES) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "reader stride is outside supported range", + ))); + }; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, reader_stride); let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); - let result = copy_exact( + let observed = copy_exact( &mut source, &mut sink, ExactTransfer::new( @@ -287,26 +302,23 @@ mod tests { CatalogRestartPhase::ReadCatalog, TOTAL_BYTES, ), - ); - - let observed = result.unwrap_or_else(|error| { - panic!("copy should succeed for expected length: {error:?}"); - }); + )?; assert_eq!(observed, TOTAL_BYTES); assert_eq!(sink.observed_bytes(), TOTAL_BYTES); assert!(sink.total_chunks() > 0); assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); assert!(sink.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH); - assert!(sink.max_chunk() >= READER_STRIDE as usize); + assert!(sink.max_chunk() >= READER_STRIDE_BYTES); assert!(size_of::() < 64); + Ok(()) } #[test] - fn copy_exact_to_chunks_streams() { + fn copy_exact_to_chunks_streams() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); let mut observed = Vec::>::new(); - let result = copy_exact_to_chunks( + let _observed = copy_exact_to_chunks( &mut source, ExactTransfer::new( CatalogRestartArtifact::Head, @@ -317,14 +329,14 @@ mod tests { observed.push(chunk.to_vec()); Ok(()) }, - ); + )?; - assert!(result.is_ok()); assert_eq!(observed.concat(), b"abcdefg"); + Ok(()) } #[test] - fn copy_exact_to_chunks_rejects_short_artifacts() { + fn copy_exact_to_chunks_rejects_short_artifacts() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b']); let mut seen = 0_u8; @@ -336,12 +348,25 @@ mod tests { 4, ), |_chunk| { - seen = seen.checked_add(1).expect("unexpected chunk overflow"); + seen = match seen.checked_add(1) { + Some(total) => total, + None => { + return Err(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: 4, + }); + } + }; Ok(()) }, ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "short artifact should have been rejected", + ))); + }; assert_eq!(seen, 1); assert!(matches!( error, @@ -350,10 +375,11 @@ mod tests { ref source, } if source.kind() == ErrorKind::UnexpectedEof )); + Ok(()) } #[test] - fn copy_exact_to_chunks_rejects_trailing_bytes() { + fn copy_exact_to_chunks_rejects_trailing_bytes() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let result = copy_exact_to_chunks( @@ -366,7 +392,12 @@ mod tests { |_| Ok(()), ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "trailing bytes should have been rejected", + ))); + }; let expected = 2_u64; assert!(matches!( error, @@ -377,10 +408,11 @@ mod tests { observed: 3 } if minimum == expected && maximum == expected )); + Ok(()) } #[test] - fn copy_exact_rejects_short_artifacts() { + fn copy_exact_rejects_short_artifacts() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b']); let mut sink = StreamingWriteSink::new(16 * 1024); @@ -394,7 +426,12 @@ mod tests { ), ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "short artifact should have been rejected", + ))); + }; assert_eq!(sink.observed_bytes(), 2); assert!(matches!( error, @@ -403,10 +440,11 @@ mod tests { ref source, } if source.kind() == ErrorKind::UnexpectedEof )); + Ok(()) } #[test] - fn copy_exact_rejects_trailing_bytes() { + fn copy_exact_rejects_trailing_bytes() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let mut sink = StreamingWriteSink::new(16 * 1024); @@ -420,7 +458,12 @@ mod tests { ), ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "trailing bytes should have been rejected", + ))); + }; let expected = 2_u64; assert_eq!(sink.observed_bytes(), 2); assert!(matches!( @@ -432,6 +475,7 @@ mod tests { observed: 3 } if minimum == expected && maximum == expected )); + Ok(()) } struct SyntheticStreamingReader { @@ -454,14 +498,11 @@ mod tests { return Ok(0); } - let sink_capacity = match u64::try_from(sink.len()) { - Ok(capacity) => capacity, - Err(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink capacity exceeds supported range", - )); - } + let Ok(sink_capacity) = u64::try_from(sink.len()) else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink capacity exceeds supported range", + )); }; let emitted: usize = match self .emit_stride @@ -478,9 +519,16 @@ mod tests { } }; - sink[..emitted].fill(0x5a); - self.remaining -= u64::try_from(emitted) + let read_window = sink.get_mut(..emitted).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "read window overflow") + })?; + read_window.fill(0x5a); + let emitted_u64 = u64::try_from(emitted) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; + self.remaining = self + .remaining + .checked_sub(emitted_u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; Ok(emitted) } } diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index d005b48..c1a98aa 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -93,20 +93,20 @@ fn reject_trailing_bytes( observed: expected.get(), } })?; - let observed = expected.get().checked_add(increment).ok_or( + let observed = expected.get().checked_add(increment).ok_or_else(|| { FilesystemRecoveryStageError::LengthChanged { stage, expected, observed: expected.get(), - }, - )?; + } + })?; return Err(FilesystemRecoveryStageError::LengthChanged { stage, expected, observed, }); } - Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} Err(source) => { return Err(FilesystemRecoveryStageError::Materialize { stage, @@ -141,7 +141,8 @@ pub(super) fn verify_position( mod tests { use std::error::Error; use std::fs; - use std::path::PathBuf; + use std::io; + use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; @@ -175,12 +176,16 @@ mod tests { let path = sandbox.path().join("stage.bin"); fs::write(&path, b"abc")?; let mut file = open_for_tests(&path)?; - let error = super::read_and_position( + let Err(error) = super::read_and_position( &mut file, RecoveryStage::Segment, RecoveryStageLength::from_validated(5), - ) - .expect_err("short stage materialization was admitted"); + ) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "short stage materialization should have failed", + ))); + }; assert!(matches!( error, @@ -201,12 +206,16 @@ mod tests { let path = sandbox.path().join("stage.bin"); fs::write(&path, b"abcdef")?; let mut file = open_for_tests(&path)?; - let error = super::read_and_position( + let Err(error) = super::read_and_position( &mut file, RecoveryStage::Segment, RecoveryStageLength::from_validated(3), - ) - .expect_err("trailing-stage materialization was admitted"); + ) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "trailing stage materialization should have failed", + ))); + }; assert!(matches!( error, @@ -221,21 +230,26 @@ mod tests { Ok(()) } - fn open_for_tests(path: &PathBuf) -> Result> { - let directory = Dir::open_ambient_dir( - path.parent() - .expect("directory parent exists for stage fixture"), - ambient_authority(), - )?; + fn open_for_tests(path: &Path) -> Result> { + let directory_path = path.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "stage fixture path does not have a parent directory", + ) + })?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "stage fixture path has no UTF-8 file name", + ) + })?; + let directory = Dir::open_ambient_dir(directory_path, ambient_authority())?; let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No).nonblock(true); - let file = directory.open_with( - path.file_name() - .expect("file path has file name for fixture") - .to_str() - .expect("file name is UTF-8 for fixture"), - &options, - )?; + let file = directory.open_with(file_name, &options)?; Ok(file) } diff --git a/src/adapters/filesystem_root_identity.rs b/src/adapters/filesystem_root_identity.rs index e63619f..36af8bb 100644 --- a/src/adapters/filesystem_root_identity.rs +++ b/src/adapters/filesystem_root_identity.rs @@ -8,6 +8,10 @@ pub(super) struct FilesystemRootIdentity { } impl FilesystemRootIdentity { + #[cfg(any( + target_os = "linux", + all(not(target_os = "linux"), any(test, feature = "repository-tasks")) + ))] pub(super) const fn new(device: u64, mount: u64, file: u64) -> Self { Self { device, From 877fef013e356d1272c1de7627ecf6fb6d2581f9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 15 Aug 2026 04:02:34 -0700 Subject: [PATCH 49/50] Refactor: isolate catalog restart IO test doubles --- src/adapters/catalog_restart_io.rs | 179 +---------------- .../catalog_restart_io_test_doubles.rs | 180 ++++++++++++++++++ src/adapters/mod.rs | 2 + 3 files changed, 186 insertions(+), 175 deletions(-) create mode 100644 src/adapters/catalog_restart_io_test_doubles.rs diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index cfd32ae..3af6b0f 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -224,9 +224,12 @@ impl Write for VecWrite<'_> { mod tests { use std::error::Error; use std::io; - use std::io::{Cursor, ErrorKind, Read, Write}; + use std::io::{Cursor, ErrorKind}; use std::mem::size_of; + use super::super::catalog_restart_io_test_doubles::{ + StreamingCallbackBudget, StreamingWriteSink, SyntheticStreamingReader, + }; use super::*; #[test] @@ -477,178 +480,4 @@ mod tests { )); Ok(()) } - - struct SyntheticStreamingReader { - remaining: u64, - emit_stride: u64, - } - - impl SyntheticStreamingReader { - fn new(total: u64, emit_stride: u64) -> Self { - Self { - remaining: total, - emit_stride, - } - } - } - - impl Read for SyntheticStreamingReader { - fn read(&mut self, sink: &mut [u8]) -> io::Result { - if self.remaining == 0 { - return Ok(0); - } - - let Ok(sink_capacity) = u64::try_from(sink.len()) else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink capacity exceeds supported range", - )); - }; - let emitted: usize = match self - .emit_stride - .min(self.remaining) - .min(sink_capacity) - .try_into() - { - Ok(size) => size, - Err(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "requested read size exceeds supported range", - )); - } - }; - - let read_window = sink.get_mut(..emitted).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "read window overflow") - })?; - read_window.fill(0x5a); - let emitted_u64 = u64::try_from(emitted) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; - self.remaining = self - .remaining - .checked_sub(emitted_u64) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; - Ok(emitted) - } - } - - struct StreamingCallbackBudget { - observed_bytes: u64, - total_chunks: u64, - max_chunk: usize, - callback_limit: usize, - expected_total: u64, - } - - impl StreamingCallbackBudget { - fn new(expected_total: u64, callback_limit: usize) -> Self { - Self { - observed_bytes: 0, - total_chunks: 0, - max_chunk: 0, - callback_limit, - expected_total, - } - } - - fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { - self.total_chunks = - self.total_chunks - .checked_add(1) - .ok_or(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - })?; - - self.max_chunk = self.max_chunk.max(chunk.len()); - - self.observed_bytes = self - .observed_bytes - .checked_add(u64::try_from(chunk.len()).map_err(|_source| { - CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - } - })?) - .ok_or(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - })?; - - Ok(()) - } - - fn observed_bytes(&self) -> u64 { - self.observed_bytes - } - - fn max_chunk(&self) -> usize { - self.max_chunk - } - - fn callback_limit(&self) -> usize { - self.callback_limit - } - - fn total_chunks(&self) -> u64 { - self.total_chunks - } - } - - struct StreamingWriteSink { - observed_bytes: u64, - observed_chunks: u64, - max_chunk: usize, - writer_memory_limit: usize, - } - - impl StreamingWriteSink { - fn new(writer_memory_limit: usize) -> Self { - Self { - observed_bytes: 0, - observed_chunks: 0, - max_chunk: 0, - writer_memory_limit, - } - } - - fn observed_bytes(&self) -> u64 { - self.observed_bytes - } - - fn total_chunks(&self) -> u64 { - self.observed_chunks - } - - fn max_chunk(&self) -> usize { - self.max_chunk - } - } - - impl Write for StreamingWriteSink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.observed_chunks = self.observed_chunks.checked_add(1).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow") - })?; - let observed = u64::try_from(bytes.len()).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow") - })?; - self.observed_bytes = self.observed_bytes.checked_add(observed).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "write count overflow") - })?; - self.max_chunk = self.max_chunk.max(bytes.len()); - if bytes.len() > self.writer_memory_limit { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink memory budget exceeded", - )); - } - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } } diff --git a/src/adapters/catalog_restart_io_test_doubles.rs b/src/adapters/catalog_restart_io_test_doubles.rs new file mode 100644 index 0000000..ebe5026 --- /dev/null +++ b/src/adapters/catalog_restart_io_test_doubles.rs @@ -0,0 +1,180 @@ +//! This module owns bounded streaming doubles for catalog restart I/O laws. + +use std::io::{self, Read, Write}; + +use super::{CatalogRestartArtifact, CatalogRestartError}; + +pub(super) struct SyntheticStreamingReader { + remaining: u64, + emit_stride: u64, +} + +impl SyntheticStreamingReader { + pub(super) fn new(total: u64, emit_stride: u64) -> Self { + Self { + remaining: total, + emit_stride, + } + } +} + +impl Read for SyntheticStreamingReader { + fn read(&mut self, sink: &mut [u8]) -> io::Result { + if self.remaining == 0 { + return Ok(0); + } + + let Ok(sink_capacity) = u64::try_from(sink.len()) else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink capacity exceeds supported range", + )); + }; + let emitted: usize = match self + .emit_stride + .min(self.remaining) + .min(sink_capacity) + .try_into() + { + Ok(size) => size, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "requested read size exceeds supported range", + )); + } + }; + + let read_window = sink + .get_mut(..emitted) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read window overflow"))?; + read_window.fill(0x5a); + let emitted_u64 = u64::try_from(emitted) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; + self.remaining = self + .remaining + .checked_sub(emitted_u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; + Ok(emitted) + } +} + +pub(super) struct StreamingCallbackBudget { + observed_bytes: u64, + total_chunks: u64, + max_chunk: usize, + callback_limit: usize, + expected_total: u64, +} + +impl StreamingCallbackBudget { + pub(super) fn new(expected_total: u64, callback_limit: usize) -> Self { + Self { + observed_bytes: 0, + total_chunks: 0, + max_chunk: 0, + callback_limit, + expected_total, + } + } + + pub(super) fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { + self.total_chunks = + self.total_chunks + .checked_add(1) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + self.max_chunk = self.max_chunk.max(chunk.len()); + + self.observed_bytes = self + .observed_bytes + .checked_add(u64::try_from(chunk.len()).map_err(|_source| { + CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + } + })?) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + Ok(()) + } + + pub(super) fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + pub(super) fn max_chunk(&self) -> usize { + self.max_chunk + } + + pub(super) fn callback_limit(&self) -> usize { + self.callback_limit + } + + pub(super) fn total_chunks(&self) -> u64 { + self.total_chunks + } +} + +pub(super) struct StreamingWriteSink { + observed_bytes: u64, + observed_chunks: u64, + max_chunk: usize, + writer_memory_limit: usize, +} + +impl StreamingWriteSink { + pub(super) fn new(writer_memory_limit: usize) -> Self { + Self { + observed_bytes: 0, + observed_chunks: 0, + max_chunk: 0, + writer_memory_limit, + } + } + + pub(super) fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + pub(super) fn total_chunks(&self) -> u64 { + self.observed_chunks + } + + pub(super) fn max_chunk(&self) -> usize { + self.max_chunk + } +} + +impl Write for StreamingWriteSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.observed_chunks = self + .observed_chunks + .checked_add(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow"))?; + let observed = u64::try_from(bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow"))?; + self.observed_bytes = self + .observed_bytes + .checked_add(observed) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "write count overflow"))?; + self.max_chunk = self.max_chunk.max(bytes.len()); + if bytes.len() > self.writer_memory_limit { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink memory budget exceeded", + )); + } + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 3c783ca..6d69ae8 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -50,6 +50,8 @@ mod catalog_restart_artifact; mod catalog_restart_byte_limit; mod catalog_restart_error; mod catalog_restart_io; +#[cfg(test)] +mod catalog_restart_io_test_doubles; mod catalog_restart_loader; mod catalog_restart_phase; mod catalog_restart_policy; From c195bb1aa6563c4c1ebfd79b4cf7e3b2ada1d079 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 15 Aug 2026 04:34:55 -0700 Subject: [PATCH 50/50] Fix: isolate root identity from platform admission --- src/adapters/filesystem_platform_profile.rs | 34 +++++++++++++++---- .../filesystem_platform_profile_tests.rs | 10 +++--- .../durability_crash_matrix/error/display.rs | 23 ++++++++++++- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 1b6bbda..ca0423c 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -19,7 +19,6 @@ struct LinuxDirectoryProperties { device_major: u32, device_minor: u32, mount_id: u64, - inode: u64, } #[cfg(target_os = "linux")] @@ -85,21 +84,42 @@ fn linux_directory_properties(file: &std::fs::File) -> io::Result io::Result { let file = directory.try_clone()?.into_std_file(); - let properties = linux_directory_properties(&file)?; - Ok(linux_root_identity(properties)) + linux_file_identity(&file) +} + +#[cfg(target_os = "linux")] +fn linux_file_identity(file: &std::fs::File) -> io::Result { + use rustix::fs::{AtFlags, StatxFlags, statx}; + + let required = StatxFlags::BASIC_STATS | StatxFlags::MNT_ID; + let status = statx(file, ".", AtFlags::empty(), required)?; + let observed = StatxFlags::from_bits_retain(status.stx_mask); + if !observed.contains(required) { + return Err(unsupported_linux_profile()); + } + Ok(linux_root_identity( + status.stx_dev_major, + status.stx_dev_minor, + status.stx_mnt_id, + status.stx_ino, + )) } #[cfg(target_os = "linux")] -fn linux_root_identity(properties: LinuxDirectoryProperties) -> FilesystemRootIdentity { - let device = rustix::fs::makedev(properties.device_major, properties.device_minor); - FilesystemRootIdentity::new(device, properties.mount_id, properties.inode) +fn linux_root_identity( + device_major: u32, + device_minor: u32, + mount_id: u64, + inode: u64, +) -> FilesystemRootIdentity { + let device = rustix::fs::makedev(device_major, device_minor); + FilesystemRootIdentity::new(device, mount_id, inode) } #[cfg(all(not(target_os = "linux"), any(test, feature = "repository-tasks")))] diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs index dd9d63e..4f4776e 100644 --- a/src/adapters/filesystem_platform_profile_tests.rs +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -32,7 +32,7 @@ fn only_writable_case_sensitive_ext4_is_admitted() { #[test] fn every_protocol_child_must_share_the_root_filesystem_and_mount() { assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); - let root = properties(8, 1, 41, 1); + let root = properties(8, 1, 41); let mut casefolded = root; casefolded.inode_flags = EXT4_CASEFOLD_FLAG; let mut read_only = root; @@ -41,8 +41,8 @@ fn every_protocol_child_must_share_the_root_filesystem_and_mount() { foreign_format.filesystem_type = NFS_SUPER_MAGIC; assert!(admit_linux_child_properties(root, root).is_ok()); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41, 1))); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42, 1))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42))); assert_unsupported(&admit_linux_child_properties(root, casefolded)); assert_unsupported(&admit_linux_child_properties(root, read_only)); assert_unsupported(&admit_linux_child_properties(root, foreign_format)); @@ -50,7 +50,7 @@ fn every_protocol_child_must_share_the_root_filesystem_and_mount() { #[test] fn root_identity_uses_linux_device_mount_and_inode_coordinates() { - let identity = linux_root_identity(properties(8, 1, 41, 73)); + let identity = linux_root_identity(8, 1, 41, 73); assert_eq!(identity.device(), rustix::fs::makedev(8, 1)); assert_eq!(identity.mount(), 41); assert_eq!(identity.file(), 73); @@ -70,7 +70,6 @@ const fn properties( device_major: u32, device_minor: u32, mount_id: u64, - inode: u64, ) -> LinuxDirectoryProperties { LinuxDirectoryProperties { filesystem_type: EXT4_SUPER_MAGIC, @@ -79,6 +78,5 @@ const fn properties( device_major, device_minor, mount_id, - inode, } } diff --git a/xtask/src/durability_crash_matrix/error/display.rs b/xtask/src/durability_crash_matrix/error/display.rs index b9a215d..79f24e8 100644 --- a/xtask/src/durability_crash_matrix/error/display.rs +++ b/xtask/src/durability_crash_matrix/error/display.rs @@ -217,7 +217,9 @@ fn format_boundary( formatter: &mut fmt::Formatter<'_>, ) -> fmt::Result { match error { - DurabilityCrashMatrixError::Io { action, .. } => write!(formatter, "cannot {action}"), + DurabilityCrashMatrixError::Io { action, source } => { + write!(formatter, "cannot {action}: {source}") + } DurabilityCrashMatrixError::NonUnicodeStatePath => { formatter.write_str("post-crash store path is not valid Unicode") } @@ -233,3 +235,22 @@ fn format_boundary( _ => Err(fmt::Error), } } + +#[cfg(test)] +mod tests { + use std::io; + + use super::DurabilityCrashMatrixError; + + #[test] + fn io_boundary_diagnostics_preserve_the_exact_source() { + let error = DurabilityCrashMatrixError::io( + "open crash catalog publisher", + io::Error::new(io::ErrorKind::Unsupported, "profile probe escaped bypass"), + ); + assert_eq!( + error.to_string(), + "cannot open crash catalog publisher: profile probe escaped bypass" + ); + } +}