From a48915af401caa5958d68d3db675e8b62e05950b Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 20 Jul 2026 02:02:57 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(wire):=20S3/#108=20=E2=80=94=20Hop-B?= =?UTF-8?q?=20arrow=20shard=20writer=20+=20month=5Fslice=20seam=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_shard: one (target,month) shard from declared primitives via views_frames.io.arrow.save, §2 header embedded, file named from the header's own declarations (§4.1b), SHA-256 of the complete bytes returned for the §4.2 manifest. month_slice joins the frame seam (frame → one month's primitives, missing month fails loud) so the writer never touches the frame API. Byte parity proven against the fixture arrow shard; toolchain pin asserted fail-loud (pyarrow 23.0.1) — drift can never read as conformance. Part of epic #105. Closes #108. Co-Authored-By: Claude Fable 5 --- tests/test_wire_shard.py | 90 +++++++++++++++++++ .../unfao/frame_extraction.py | 17 ++++ views_postprocessing/unfao/wire/shard.py | 35 ++++++++ 3 files changed, 142 insertions(+) create mode 100644 tests/test_wire_shard.py create mode 100644 views_postprocessing/unfao/wire/shard.py diff --git a/tests/test_wire_shard.py b/tests/test_wire_shard.py new file mode 100644 index 0000000..1380375 --- /dev/null +++ b/tests/test_wire_shard.py @@ -0,0 +1,90 @@ +"""Hop-B shard writer (wire/shard.py) — byte parity with the fixture arrow shard. + +The byte tests assert the pinned toolchain (fixture README: pyarrow 23.0.1) and +FAIL LOUD on drift — never skip-silent: a quiet skip would read as conformance. +""" + +import hashlib +import zipfile +from pathlib import Path + +import pyarrow +import pytest + +from views_postprocessing.unfao import track_a_source +from views_postprocessing.unfao.frame_extraction import month_slice +from views_postprocessing.unfao.wire import header as wire_header +from views_postprocessing.unfao.wire import shard as wire_shard + +_FIX = Path(__file__).resolve().parent / "fixtures" / "wire_contract" +_PINNED_PYARROW = "23.0.1" + + +def _fixture_frame_and_header(): + manifest_bytes = (_FIX / "fixture_run_0__lr_ged_sb__manifest.json").read_bytes() + manifest = track_a_source.read_manifest(manifest_bytes) + shard_bytes = (_FIX / manifest["shards"][0]["name"]).read_bytes() + return track_a_source.read_shard( + shard_bytes, expected_sha256=manifest["shards"][0]["sha256"] + ) + + +def test_toolchain_is_pinned(): + assert pyarrow.__version__ == _PINNED_PYARROW, ( + f"pinned toolchain violated: byte-parity oracle requires pyarrow " + f"{_PINNED_PYARROW}, found {pyarrow.__version__} — re-pin the environment " + f"or regenerate the fixture (a contract change, §10)." + ) + + +def test_byte_parity_with_the_fixture_arrow_shard(tmp_path): + frame, track_a_header = _fixture_frame_and_header() + values, time, unit = month_slice(frame, 543) + built_header = wire_header.build_header( + sample_count=track_a_header["sample_count"], + dtype=track_a_header["dtype"], + target=track_a_header["target"], + time_id=track_a_header["time_id"], + run_id=track_a_header["run_id"], + generated_at=track_a_header["generated_at"], + provenance=track_a_header["provenance"], + sharding_index=track_a_header["sharding"]["index"], + sharding_count=track_a_header["sharding"]["count"], + ) + name, sha = wire_shard.write_shard( + values, time, unit, header=built_header, directory=tmp_path + ) + assert name == "fixture_run_0__lr_ged_sb__m000543.arrow.parquet" + canonical = (_FIX / name).read_bytes() + assert (tmp_path / name).read_bytes() == canonical + assert sha == hashlib.sha256(canonical).hexdigest() + + +def test_sha_matches_sha256sums(tmp_path): + frame, track_a_header = _fixture_frame_and_header() + values, time, unit = month_slice(frame, 543) + _, sha = wire_shard.write_shard( + values, time, unit, header=track_a_header, directory=tmp_path + ) + sums = (_FIX / "SHA256SUMS").read_text() + assert f"{sha} fixture_run_0__lr_ged_sb__m000543.arrow.parquet" in sums + + +def test_month_slice_missing_month_fails_loud(): + frame, _ = _fixture_frame_and_header() + with pytest.raises(ValueError, match="month 999"): + month_slice(frame, 999) + + +def test_header_passthrough_survives_roundtrip(tmp_path): + # The embedded metadata equals the Track-A header — one header, both envelopes. + from views_frames.io import arrow as vf_arrow + + frame, track_a_header = _fixture_frame_and_header() + values, time, unit = month_slice(frame, 543) + name, _ = wire_shard.write_shard( + values, time, unit, header=track_a_header, directory=tmp_path + ) + with zipfile.ZipFile(_FIX / "fixture_run_0__lr_ged_sb__m000543.tap.zip") as zf: + assert zf.read("metadata.json") # fixture sanity + assert vf_arrow.load(tmp_path / name)["metadata"] == track_a_header diff --git a/views_postprocessing/unfao/frame_extraction.py b/views_postprocessing/unfao/frame_extraction.py index 9d22843..08da2dd 100644 --- a/views_postprocessing/unfao/frame_extraction.py +++ b/views_postprocessing/unfao/frame_extraction.py @@ -37,3 +37,20 @@ def cells_of(frame: PredictionFrame) -> set[int]: def months_of(frame: PredictionFrame) -> NDArray[np.int64]: """The distinct month ids present in the frame, ascending (its index ``time`` axis).""" return np.unique(np.asarray(frame.index.time, dtype=np.int64)) + + +def month_slice( + frame: PredictionFrame, month_id: int +) -> tuple[NDArray[np.float32], NDArray[np.int64], NDArray[np.int64]]: + """One month's ``(values, time, unit)`` primitives, row order preserved. + + The per-(target, month) sharding cut (ADR-013 §4.1) as a seam concern: the + shard writer stays frame-API-free by receiving primitives. A month absent + from the frame fails loud — the caller declared it, the frame must carry it. + """ + time = np.asarray(frame.index.time, dtype=np.int64) + mask = time == month_id + if not mask.any(): + raise ValueError(f"month {month_id} not present in frame (months: {months_of(frame)}).") + unit = np.asarray(frame.index.unit, dtype=np.int64) + return frame.values[mask], time[mask], unit[mask] diff --git a/views_postprocessing/unfao/wire/shard.py b/views_postprocessing/unfao/wire/shard.py new file mode 100644 index 0000000..4cca97f --- /dev/null +++ b/views_postprocessing/unfao/wire/shard.py @@ -0,0 +1,35 @@ +"""The Hop-B arrow shard writer (ADR-013 §4.1). + +One ``views_frames.io.arrow`` file per (target, month), the §2 header embedded — +written from declared primitives (one month's ``values/time/unit`` slice, cut by +``frame_extraction.month_slice``) so this module never touches the frame API. +The shard's file name derives from the header's own declarations (never invented +here), and the returned SHA-256 of the complete file bytes is what the §4.2 run +manifest will pin. + +Byte determinism: given identical inputs and the pinned toolchain (pyarrow per the +fixture README), the emitted bytes equal the §10 fixture's — the test is byte +parity, not schema similarity. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from views_frames.io import arrow as vf_arrow + +from views_postprocessing.unfao.wire.naming import shard_name + + +def write_shard(values, time, unit, *, header: dict, directory: Path) -> tuple[str, str]: + """Write one (target, month) shard; return ``(file_name, sha256_of_bytes)``. + + ``header`` is the §2 dict from ``wire.header.build_header`` — its ``run_id``, + ``target`` and ``time_id`` name the file (§4.1b); the whole dict is embedded + as the arrow metadata (one header, both envelopes). + """ + name = shard_name(header["run_id"], header["target"], header["time_id"]) + path = Path(directory) / name + vf_arrow.save(path, values=values, time=time, unit=unit, level="pgm", metadata=header) + return name, hashlib.sha256(path.read_bytes()).hexdigest() From e47db5d7ec7d3d18b412ce7115146a3b3e866f85 Mon Sep 17 00:00:00 2001 From: Polichinl Date: Mon, 20 Jul 2026 02:18:29 +0200 Subject: [PATCH 2/2] =?UTF-8?q?fix(fixture):=20re-baseline=20Hop-B=20artif?= =?UTF-8?q?acts=20to=20the=20production=20toolchain=20(pyarrow=2016.1.0)?= =?UTF-8?q?=20=E2=80=94=20=C2=A710=20contract-change=20event,=20maintainer?= =?UTF-8?q?=20Option=20A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delivery repo's locked env can never exceed pyarrow <17 (views-pipeline-core → viewser → views-storage chain); the fixture's Hop-B bytes were generated on an unlocked dev env at 23.0.1. Regenerated arrow shard + sidecar + run manifest + SHA256SUMS under 16.1.0; Track-A bytes verified byte-identical (producer conformance untouched). New root hash 9658a648…; pyarrow now an explicit first-class pin (>=16.1.0,<17); ADR post-adoption entry records the event and the platform debt (lift the ceiling at source: pipeline-core#280); vendors notified on their PR #276. Co-Authored-By: Claude Fable 5 --- .../013_sampled_forecast_wire_contract.md | 15 +++++++++++++++ poetry.lock | 4 ++-- pyproject.toml | 1 + tests/fixtures/wire_contract/README.md | 4 ++-- tests/fixtures/wire_contract/SHA256SUMS | 6 +++--- ...re_run_0__lr_ged_sb__m000543.arrow.parquet | Bin 2919 -> 3230 bytes .../fixture_run_0__manifest.json | 4 ++-- .../fixture_run_0__sidecar.parquet | Bin 3348 -> 4188 bytes tests/test_wire_shard.py | 2 +- 9 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/ADRs/013_sampled_forecast_wire_contract.md b/docs/ADRs/013_sampled_forecast_wire_contract.md index 78c6674..0566c6e 100644 --- a/docs/ADRs/013_sampled_forecast_wire_contract.md +++ b/docs/ADRs/013_sampled_forecast_wire_contract.md @@ -1029,6 +1029,21 @@ record execution progress against it. ADR (§0–§11) has now been independently falsification-audited; every finding fixed same-day; 40 permanent guards enforce the fixes (`tests/test_falsify_adr013_*.py`).** +- **2026-07-20 — §10 fixture RE-BASELINED to the production toolchain (maintainer + decision, Option A; a §10 contract-change event).** The fixture's Hop-B byte + artifacts had been generated under pyarrow 23.0.1 (an unlocked dev + environment); the delivery repo's actual locked toolchain is **pyarrow 16.1.0** + and cannot be newer — `views-pipeline-core → viewser → views-storage` hard-pins + `pyarrow <17`. The three Hop-B artifacts (arrow shard, sidecar, run manifest) + + `SHA256SUMS` were regenerated under 16.1.0; **Track-A bytes are unchanged** + (verified byte-identical — producer-side conformance unaffected). New root + hash: `9658a6484cc9d975412e52624d52f328985f14cf58e3fc9fbdf3e64ab5a0564b`. + pyarrow is now an explicit first-class pin in this repo's pyproject + (`>=16.1.0,<17`). Vendors notified to re-vendor (their §10.1 pinned-hash tests + catch it regardless — the mechanism's second live save). **Platform debt noted + (maintainer: "fix this through the pipeline"): lifting the `<17` ceiling at its + source is filed as pipeline-core#280; when it lifts, the fixture gets one + planned re-baseline, coordinated, never a drive-by.** - **2026-07-19 — retention direction given (maintainer): a configurable retention period with automatic deletion** (e.g. 12 or 36 months — the value to be decided with the owner). This settles the *shape* of the §3.5 policy; the owner diff --git a/poetry.lock b/poetry.lock index faf5403..96069ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -4800,4 +4800,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.15" -content-hash = "57977341097211c8de6d4743c34a97411cdb524494a920ac5175f0eb5299c164" +content-hash = "5f2d192a6a36dd429b473195ca6dc30424b57aed8890fbce99741895015578ef" diff --git a/pyproject.toml b/pyproject.toml index d9aceb5..fba9ffc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ license = "MIT" python = ">=3.11,<3.15" views-pipeline-core = ">=2.1.3,<3.0.0" views-frames = ">=1.0,<2" +pyarrow = ">=16.1.0,<17.0.0" [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] diff --git a/tests/fixtures/wire_contract/README.md b/tests/fixtures/wire_contract/README.md index 73d3d91..64b6299 100644 --- a/tests/fixtures/wire_contract/README.md +++ b/tests/fixtures/wire_contract/README.md @@ -8,7 +8,7 @@ cross-repo contract**; on mismatch, re-vendor from here. **The pinned root hash (SHA-256 of `SHA256SUMS`):** ``` -b1f3878df9ef74b25dce53a070e1711db39dfdf1c6ca3e1f5a716875ceb32f44 +9658a6484cc9d975412e52624d52f328985f14cf58e3fc9fbdf3e64ab5a0564b ``` ## Contents (1 run × 1 target × 1 month × 6 cells × S=4) @@ -29,6 +29,6 @@ injected fixed literals (`run_id="fixture_run_0"`, `generated_at="2026-07-15T00: ## Regeneration `PYTHONPATH=. python3 scripts/build_wire_fixture.py` — byte-reproducible **with the pinned -tool versions** (numpy per lockfile, `pyarrow 23.0.1`, `views_frames 1.0.0`; parquet bytes +tool versions** (numpy per lockfile, `pyarrow 16.1.0`, `views_frames 1.0.0`; parquet bytes vary across pyarrow versions). The committed bytes + `SHA256SUMS` are canonical regardless. **A change to this fixture is a change to the contract (§10)** — do not regenerate casually. diff --git a/tests/fixtures/wire_contract/SHA256SUMS b/tests/fixtures/wire_contract/SHA256SUMS index 98f760f..aaf8128 100644 --- a/tests/fixtures/wire_contract/SHA256SUMS +++ b/tests/fixtures/wire_contract/SHA256SUMS @@ -1,5 +1,5 @@ -e0fc42b615e30aaa11933c69a3efa0eb19285fe36a69b23820a33149dbbf5ac0 fixture_run_0__lr_ged_sb__m000543.arrow.parquet +203650fd6933b2366ed373ad8669d1c8b4c97553e30db9d2f07985a201012c54 fixture_run_0__lr_ged_sb__m000543.arrow.parquet 61a78b9f569dff0cd97e6de673edaf55799649dffbf2c555d3733a475a55f081 fixture_run_0__lr_ged_sb__m000543.tap.zip 29d408ad8673fef692a954d26caa7890d769ad4db1c270369f772f6ebc544834 fixture_run_0__lr_ged_sb__manifest.json -ee33863ed1a67945ca2d0481d64741568fe3205449c5f3fa0791e6ec87c3535f fixture_run_0__manifest.json -3056eeda144bafa6bcc0b6ff6da041addd57d9cc8bc333526387dd2d463f0617 fixture_run_0__sidecar.parquet +bf516f0f2318f1d2d7d21b5adaac07f78ba89bc8418f956494084b7301292990 fixture_run_0__manifest.json +eb959b153a2eafcb5917ee80ed0ac40d88ca9c24243796fefd272afea5c71401 fixture_run_0__sidecar.parquet diff --git a/tests/fixtures/wire_contract/fixture_run_0__lr_ged_sb__m000543.arrow.parquet b/tests/fixtures/wire_contract/fixture_run_0__lr_ged_sb__m000543.arrow.parquet index c5021f6eed05e690bc0582f9ba7a27754d3645be..847b2ead8f678c4f3eeb74b58d5cbc05bc4ac4a0 100644 GIT binary patch delta 1342 zcma)4L5$Nz6dgOuW*4D~1!563s7OUtrB=xH#tXSMnJcY{(t|y_y51b!{U9s zfe>21$ipsioP@-e+`P)4TY1I_FXlEcok7K)3M&Ov;1O)6;SU}4qN@r1 z2Fd6DVUKusZN8BcPX5jbe{kZxy!hw5$d#4>%$wZN{4H1dUi{@$sj#-7aN^Jj;vVDh zH!r;5#s5!=kJjfexNnzYeBd(2d<(@kU!|wD+;liQ^%~t^iCgKi;bK5fnIB7_%mFQde(Np zE~D*~n4OWm*%`S?+25@}kq{@aro>bwD8SBHaiu=?v`OHl3~XwfasuS~wo;MZs7(n< zG@mp&s;Av%gh@C#w5|5EXW!)GcZ zC~o4uW3qeM*lAG}Y1dSb&wLUF=3@s(t&xqpKSrqXa zO>nH6QY=cme-qjTF1wo~aKn0RE7Y0oD|(1yZ$^uSlMFm0qh6yUgY|u25BAH@``cMd zv>+koWaPpf*iF@b*5hj2s+E&n^B7~17h*aW1IL^Mq^57_8rgFjMv##*hBL5#-!au_ zp>%31dx2|az4J7j9xKg;E{KVS%QUuCqeEyMLMtTpZOIkZ`w;If+7ZKi&{=_^s_beB zZCz|BB!HQsBB{GSEPzz7{nK!5={s!}OtfIkz$5ALyLFelAa7lkFH6PU5<;JTXlEUv I8`X~d5v@CUiU0rr delta 1246 zcmah|%a7Yc7`NSIw-plX9^!7)1$r_ zLAZ8Pf)$A(y7)1Z_V+jMi1Owe(P{AN%kNfIY4=ZlY)++%*LdZ+UsdY6g0T1a=1+$Y zpO9ZaIQZ-9&G1UMDI7Lx&5p3M*F2~dNjhpsGLpX+<;NFm^da;&-)__zLPHW@*@K_= z-`r^`0s!X>COM zeO)(nI6grp+`VU~3mT_xZciWFw=*LnCw6X|PJ#X51a~bIoY=SRUY-Q1R>nPN610b7 z_mo>Iu%A3Gv)<+@o{M`b=jYrS4hi4N&N5rM*t(K;qD-2bb!W zF%a+qCm&3RZ{*nTbFfIYim1DwzB)LMy$)gm2PX8VLp1UY$_1x5C#r?|E4%7x86LRZgMA0c-7_BqadA_7-k%0V|0pW@r|!M}k-okS3tUEMvm{b_=C=ow zcorlq?Wm;{MD3u^(q`D?LR%!I6-j0+A;BzSSWI|^*!Dr}nu@*7 zgrozW)M)8YlC2q;lW)P((4U~#cfhe*!gdq^%mNpd8MCPC=AqZ&@iVvo+w=MHR4@Bx z>k^~tb10jWK&3hM?k0B<@CpmL8sAKVJDNUIrjEn(L2Ve~E~%s_k3oR;zh@Cu&M-`<* zk(49SMTnb7hhkMkx`?GXNC}RibSO?DQn%tD-uT4QCw0^h|{=jl9lwV4clgG;Q zARB<6Gx#jy*Z8=Iq9`Zo^}1#RdIi%%Dnn*0&cH=u0ACDHD_OF>fZPFW9>JDPU!fdm z&@K6Y}jpBQs?eE@9ZaUINQ#PGp ewt6|eD9o@HghAC?rd_*W`-;x{CS#ZCQ{gW-C))D> delta 912 zcma)(ziSg=7{}k|eeW*!tM*d3ms|}^4&)^zO~|EMRIK(+6bA?C=+JPQQUYzRW^)Zv z1RYY}j2#^M2UNQ`I#dx62k9i$#SR@h)ZX@PAgjJw3>}p)5g@KJxqV|pK7fjoSYfqjckm>VTtS7 zwBj&i3U`M1yyh4p{0rC-W*eA?TKQ(mo4l=`Vzu>^#^Z-k)JdQpfqnt@R$^U=pFla4 z{S8lDc@DG>?1jSi<@_8jkst_i#P|JC4s^;9=2%b6U77Ohs=^~(){lW60(+BTM|_FQ zq=G}qo!+1Z5kzO~H=$%@l9j+}8 zRhW32Rez`m-SZc{zrGBLT>t<8 diff --git a/tests/test_wire_shard.py b/tests/test_wire_shard.py index 1380375..9bfa69b 100644 --- a/tests/test_wire_shard.py +++ b/tests/test_wire_shard.py @@ -17,7 +17,7 @@ from views_postprocessing.unfao.wire import shard as wire_shard _FIX = Path(__file__).resolve().parent / "fixtures" / "wire_contract" -_PINNED_PYARROW = "23.0.1" +_PINNED_PYARROW = "16.1.0" def _fixture_frame_and_header():