diff --git a/docker/README.md b/docker/README.md index fc1f612..0adb203 100644 --- a/docker/README.md +++ b/docker/README.md @@ -211,6 +211,7 @@ reversible, no key required: "digest": "sha256:", "provenance": { "engine": { "version_string": "...", "major": 0, "minor": 10, "patch": 2, "commit_sha": "..." }, + "feed": { "canonicalization": "pf-ohlcv-barc-le-v1", "source_values_sha256": "..." }, "codegen": { "version": "0.6.4", "generated_cpp_sha256": "...", "transpiled_from_pine": true }, "strategy": { "initial_capital": 1000000.0, "pyramiding": 1, "commission_type": "percent", "...": "all strategy() params, effective" }, "inputs": { "Fast Length": { "type": "int", "default": 9, "value": "8" }, "...": "all input()s, effective" }, @@ -225,6 +226,13 @@ reversible, no key required: when no override was passed. `value` is the applied override if one was given, otherwise the default. `digest` is a stable id for a run under a given harness and its runtime settings (same inputs + same settings ⇒ same digest). +`feed.source_values_sha256` identifies the primary OHLCV source by hashing a +versioned domain prefix followed by every parsed source row, in original order, +as little-endian `<5dq>` records (`open`, `high`, `low`, `close`, `volume`, +`timestamp`). The source identity is computed before validation-only start/end +slicing, so slice bounds remain a separate runtime concern. Paths, CSV newline +style, header order, and equivalent numeric spellings do not affect the hash. + Decode the token to recover the provenance: ```bash diff --git a/docker/run_json.py b/docker/run_json.py index 5f7d8eb..9569c6c 100755 --- a/docker/run_json.py +++ b/docker/run_json.py @@ -60,6 +60,7 @@ "digest": "sha256:", # stable run id over canonical JSON "provenance": { "engine": { version_string, major, minor, patch, commit_sha }, + "feed": { canonicalization, source_values_sha256 }, "codegen": { version, generated_cpp_sha256, transpiled_from_pine }, "strategy": { ...all strategy() params, effective... }, "inputs": { "": { type, default, value }, ... }, @@ -83,6 +84,7 @@ import json import math import re +import struct import sys import time from datetime import datetime, timezone @@ -135,6 +137,23 @@ _INPUT_RE = re.compile( r'get_input_(\w+)\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*((?:[^();]|\([^()]*\))*?)\s*\)') +# Canonical primary-feed identity. Hash the numeric BarC values in source-row +# order, before any validation-only start/end slicing. The domain prefix makes +# the byte contract versioned and prevents cross-domain hash reuse. +SOURCE_FEED_CANONICALIZATION = "pf-ohlcv-barc-le-v1" +_SOURCE_FEED_HASH_PREFIX = b"pineforge:ohlcv:barc-le:v1\0" +_SOURCE_FEED_RECORD = struct.Struct("<5dq") + + +def _new_source_feed_hasher(): + h = hashlib.sha256() + h.update(_SOURCE_FEED_HASH_PREFIX) + return h + + +def _update_source_feed_hash(h, row) -> None: + h.update(_SOURCE_FEED_RECORD.pack(*row)) + def _ctor_body(cpp_text: str) -> str: """Return the GeneratedStrategy constructor body, or '' if not found. @@ -264,7 +283,10 @@ def _codegen_version() -> str: def build_provenance(engine: dict, cpp_path, transpiled: bool, inputs_applied: dict, overrides_applied: dict, - runtime: dict | None) -> dict: + runtime: dict | None, *, source_feed_sha256: str) -> dict: + if not isinstance(source_feed_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", source_feed_sha256): + raise ValueError("source_feed_sha256 must be a lowercase SHA-256 hex digest") cpp_text = "" cpp_sha = None if cpp_path: @@ -276,6 +298,10 @@ def build_provenance(engine: dict, cpp_path, transpiled: bool, cpp_text = "" return { "engine": engine, + "feed": { + "canonicalization": SOURCE_FEED_CANONICALIZATION, + "source_values_sha256": source_feed_sha256, + }, "codegen": { "version": _codegen_version(), "generated_cpp_sha256": cpp_sha, @@ -475,19 +501,23 @@ def check_abi(lib: ctypes.CDLL) -> None: # --- helpers ---------------------------------------------------------- -def load_bars(csv_path: Path) -> tuple[ctypes.Array, int]: +def load_bars(csv_path: Path) -> tuple[ctypes.Array, int, str]: + """Load the source tape once and return bars, count, and canonical hash.""" rows: list[tuple[float, float, float, float, float, int]] = [] + feed_hasher = _new_source_feed_hasher() with csv_path.open(newline="", encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: - rows.append(( + parsed = ( float(row["open"]), float(row["high"]), float(row["low"]), float(row["close"]), float(row["volume"]), int(row["timestamp"]), - )) + ) + _update_source_feed_hash(feed_hasher, parsed) + rows.append(parsed) n = len(rows) bars = (BarC * n)() for i, (o, h, l, c, v, ts) in enumerate(rows): @@ -497,7 +527,7 @@ def load_bars(csv_path: Path) -> tuple[ctypes.Array, int]: bars[i].close = c bars[i].volume = v bars[i].timestamp = ts - return bars, n + return bars, n, feed_hasher.hexdigest() def load_strategy(so_path: Path) -> ctypes.CDLL: @@ -823,7 +853,7 @@ def main() -> int: magnifier_samples = max(2, int(args.magnifier_samples)) magnifier_dist = parse_magnifier_dist(args.magnifier_dist) - bars, n = load_bars(args.ohlcv) + bars, n, source_feed_sha256 = load_bars(args.ohlcv) first_ts, last_ts = bars[0].timestamp, bars[n - 1].timestamp lib = load_strategy(args.so) @@ -931,6 +961,7 @@ def _run(st, rep): inputs, overrides, applied_runtime, + source_feed_sha256=source_feed_sha256, )) except Exception: out["fingerprint"] = None diff --git a/scripts/fingerprint_self_test.py b/scripts/fingerprint_self_test.py index e274117..b0b7944 100644 --- a/scripts/fingerprint_self_test.py +++ b/scripts/fingerprint_self_test.py @@ -15,9 +15,12 @@ import ast import base64 +import hashlib import importlib.util import json +import struct import sys +import tempfile from pathlib import Path REPO = Path(__file__).resolve().parent.parent @@ -141,7 +144,6 @@ def check(name: str, cond: bool) -> None: check(f"{label}: token decodes to provenance", decoded == prov) check(f"{label}: digest prefixed sha256:", fp["digest"].startswith("sha256:")) canonical = json.dumps(prov, sort_keys=True, separators=(",", ":")) - import hashlib check(f"{label}: digest matches canonical sha256", fp["digest"] == "sha256:" + hashlib.sha256(canonical.encode()).hexdigest()) fp2 = m.build_fingerprint(prov) @@ -190,6 +192,148 @@ def check(name: str, cond: bool) -> None: check("run_strategy: main fingerprint path uses runtime helper", main_uses_helper) + # --- canonical source-feed identity --------------------------------- + # Seven test classes pin the feed contract independently of any strategy + # binary. Rows use the exact BarC field order: O,H,L,C,V,timestamp. + rows = [ + (100.0, 101.0, 99.0, 100.5, 10.0, 1_000), + (100.5, 103.0, 100.0, 102.0, 20.0, 2_000), + (102.0, 104.0, 101.0, 103.5, 30.0, 3_000), + (103.5, 105.0, 102.5, 104.0, 40.0, 4_000), + ] + + def expected_feed_sha(feed_rows) -> str: + h = hashlib.sha256() + h.update(b"pineforge:ohlcv:barc-le:v1\0") + record = struct.Struct("<5dq") + for row in feed_rows: + h.update(record.pack(*row)) + return h.hexdigest() + + def loader_fails(loader, path: Path) -> bool: + try: + loader(path) + except (OSError, KeyError, ValueError, IndexError): + return True + return False + + def provenance_rejects(module, digest) -> bool: + try: + module.build_provenance({}, None, False, {}, {}, {}, + source_feed_sha256=digest) + except ValueError: + return True + return False + + with tempfile.TemporaryDirectory(prefix="pf-fingerprint-feed-") as td: + tmp = Path(td) + feed_a = tmp / "a.csv" + feed_b = tmp / "b.csv" + feed_changed = tmp / "changed.csv" + feed_reordered = tmp / "reordered.csv" + malformed = tmp / "malformed.csv" + + # A: canonical bytes are domain-prefixed <5dq> records. + feed_a.write_text( + "timestamp,open,high,low,close,volume\n" + "".join( + f"{ts},{o:g},{h:g},{lo:g},{c:g},{v:g}\n" + for o, h, lo, c, v, ts in rows), + encoding="utf-8") + expected = expected_feed_sha(rows) + rs_bars, rs_n, rs_sha = rs._load_bars(feed_a) + check("feed A: local hash pins domain-prefixed <5dq> contract", + rs_n == len(rows) and rs_sha == expected + and len(rs_bars) == len(rows)) + + # B: Docker and local loaders must agree on one source tape. + rj_bars, rj_n, rj_sha = rj.load_bars(feed_a) + check("feed B: local/docker loaders agree", + rj_n == rs_n and len(rj_bars) == len(rows) + and rj_sha == rs_sha) + + # C: path, header order, newline style and numeric spelling are not + # semantic; the parsed values stay identical. + feed_b.write_bytes(( + "open,high,low,close,volume,timestamp\r\n" + "".join( + f"{o:.1f},{h:.1f},{lo:.1f},{c:.1f},{v:.1f},{ts}\r\n" + for o, h, lo, c, v, ts in rows) + ).encode("utf-8")) + _, _, rs_sha_b = rs._load_bars(feed_b) + _, _, rj_sha_b = rj.load_bars(feed_b) + check("feed C: equivalent parsed values ignore path/text formatting", + rs_sha_b == expected and rj_sha_b == expected) + + # D: content and original row order are both identity-bearing, even + # when count plus first/last timestamps are unchanged. + changed_rows = list(rows) + changed_rows[1] = (*changed_rows[1][:3], 102.25, + *changed_rows[1][4:]) + feed_changed.write_text( + "timestamp,open,high,low,close,volume\n" + "".join( + f"{ts},{o},{h},{lo},{c},{v}\n" + for o, h, lo, c, v, ts in changed_rows), + encoding="utf-8") + reordered_rows = [rows[0], rows[2], rows[1], rows[3]] + feed_reordered.write_text( + "timestamp,open,high,low,close,volume\n" + "".join( + f"{ts},{o},{h},{lo},{c},{v}\n" + for o, h, lo, c, v, ts in reordered_rows), + encoding="utf-8") + _, _, changed_sha = rs._load_bars(feed_changed) + _, _, reordered_sha = rs._load_bars(feed_reordered) + check("feed D: middle value and row-order changes alter identity", + changed_sha != expected and reordered_sha != expected + and changed_sha != reordered_sha) + + # E: source identity is computed before local start/end slicing. + _, filtered_n, filtered_sha = rs._load_bars( + feed_a, ohlcv_start_ms=3_000, ohlcv_end_ms=4_000) + check("feed E: validation slicing preserves source identity", + filtered_n == 2 and filtered_sha == expected) + + # F: feed identity is mandatory provenance and changes the digest. + prov_rs = rs.build_provenance( + {}, None, False, {}, {}, {}, source_feed_sha256=expected) + prov_rj = rj.build_provenance( + {}, None, False, {}, {}, {}, source_feed_sha256=expected) + changed_prov = rs.build_provenance( + {}, None, False, {}, {}, {}, source_feed_sha256=changed_sha) + check("feed F: provenance records feed and digest discriminates tapes", + prov_rs == prov_rj + and prov_rs["feed"] == { + "canonicalization": "pf-ohlcv-barc-le-v1", + "source_values_sha256": expected, + } + and rs.build_fingerprint(prov_rs)["digest"] + != rs.build_fingerprint(changed_prov)["digest"]) + check("feed F: null/invalid identities fail closed", + provenance_rejects(rs, None) + and provenance_rejects(rj, None) + and provenance_rejects(rs, expected.upper())) + + # G: absent/malformed sources fail in both loaders before provenance. + malformed.write_text( + "timestamp,open,high,low,close\n1000,1,2,0,1.5\n", + encoding="utf-8") + missing = tmp / "missing.csv" + check("feed G: missing/malformed feeds produce no identity", + loader_fails(rs._load_bars, missing) + and loader_fails(rj.load_bars, missing) + and loader_fails(rs._load_bars, malformed) + and loader_fails(rj.load_bars, malformed)) + + # Pin production call-site plumbing so a future refactor cannot compute a + # digest and then silently omit it from one harness's provenance. + docker_tree = ast.parse(RUN_JSON.read_text(encoding="utf-8")) + callsite_has_feed = lambda tree: any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "build_provenance" + and any(kw.arg == "source_feed_sha256" for kw in node.keywords) + for node in ast.walk(tree)) + check("feed call sites: local and docker require source identity", + callsite_has_feed(source_tree) and callsite_has_feed(docker_tree)) + # --- the two copies must agree byte-for-byte on the fixture ---------- check("copies agree: parse_strategy_params", rj.parse_strategy_params(FIXTURE_CPP) == rs.parse_strategy_params(FIXTURE_CPP)) diff --git a/scripts/run_strategy.py b/scripts/run_strategy.py index 9dcf27a..af6f3b1 100644 --- a/scripts/run_strategy.py +++ b/scripts/run_strategy.py @@ -46,6 +46,7 @@ import json import os import re +import struct import sys import time from datetime import datetime, timezone @@ -125,6 +126,23 @@ _INPUT_RE = re.compile( r'get_input_(\w+)\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*((?:[^();]|\([^()]*\))*?)\s*\)') +# Canonical primary-feed identity. Hash the numeric BarC values in source-row +# order, before any validation-only start/end slicing. The domain prefix makes +# the byte contract versioned and prevents cross-domain hash reuse. +SOURCE_FEED_CANONICALIZATION = "pf-ohlcv-barc-le-v1" +_SOURCE_FEED_HASH_PREFIX = b"pineforge:ohlcv:barc-le:v1\0" +_SOURCE_FEED_RECORD = struct.Struct("<5dq") + + +def _new_source_feed_hasher(): + h = hashlib.sha256() + h.update(_SOURCE_FEED_HASH_PREFIX) + return h + + +def _update_source_feed_hash(h, row) -> None: + h.update(_SOURCE_FEED_RECORD.pack(*row)) + def _ctor_body(cpp_text: str) -> str: """Return the GeneratedStrategy constructor body, or '' if not found. @@ -254,7 +272,10 @@ def _codegen_version() -> str: def build_provenance(engine: dict, cpp_path, transpiled: bool, inputs_applied: dict, overrides_applied: dict, - runtime: dict | None) -> dict: + runtime: dict | None, *, source_feed_sha256: str) -> dict: + if not isinstance(source_feed_sha256, str) or not re.fullmatch( + r"[0-9a-f]{64}", source_feed_sha256): + raise ValueError("source_feed_sha256 must be a lowercase SHA-256 hex digest") cpp_text = "" cpp_sha = None if cpp_path: @@ -266,6 +287,10 @@ def build_provenance(engine: dict, cpp_path, transpiled: bool, cpp_text = "" return { "engine": engine, + "feed": { + "canonicalization": SOURCE_FEED_CANONICALIZATION, + "source_values_sha256": source_feed_sha256, + }, "codegen": { "version": _codegen_version(), "generated_cpp_sha256": cpp_sha, @@ -738,16 +763,27 @@ def run(self, bars_csv: Path, params: dict | None = None, (inclusive bound), letting validation tooling test alternate spans without a trimmed CSV copy. + CSV-backed runs also return ``source_feed_sha256``: the canonical + identity of the full parsed source tape before either bound is applied. + Preloaded-bar callers retain their established return shape and do not + receive an inferred identity for an already-sliced buffer. + ``on_report`` (when given) is invoked with the live ``ReportC`` after the engine-error check and BEFORE ``report_free``, so callers can read report fields the summary dict does not carry (``metrics.equity``, the raw ``equity_curve``, ...). """ + source_feed_sha256 = None if preloaded_bars is not None: + # Preserve the established (bars, n) contract. A future + # fingerprinting caller using preloaded bars must supply the + # source-tape identity through an explicit API rather than hashing + # an already-sliced subset here. bars, n = preloaded_bars else: - bars, n = _load_bars(bars_csv, ohlcv_start_ms=ohlcv_start_ms, - ohlcv_end_ms=ohlcv_end_ms) + bars, n, source_feed_sha256 = _load_bars( + bars_csv, ohlcv_start_ms=ohlcv_start_ms, + ohlcv_end_ms=ohlcv_end_ms) params = params or {} params_json = json.dumps(params).encode() @@ -849,14 +885,17 @@ def run(self, bars_csv: Path, params: dict | None = None, ) if on_report is not None: on_report(report) - return _report_to_dict(report) + result = _report_to_dict(report) + if source_feed_sha256 is not None: + result["source_feed_sha256"] = source_feed_sha256 + return result finally: self.lib.report_free(ctypes.byref(report)) self.lib.strategy_free(state) def _load_bars(csv_path: Path, *, ohlcv_start_ms: int | None = None, - ohlcv_end_ms: int | None = None) -> tuple[ctypes.Array, int]: + ohlcv_end_ms: int | None = None) -> tuple[ctypes.Array, int, str]: """Read OHLCV CSV (timestamp, open, high, low, close, volume) into BarC[]. When ``ohlcv_start_ms`` is given, drop bars whose timestamp is below @@ -864,6 +903,9 @@ def _load_bars(csv_path: Path, *, ohlcv_start_ms: int | None = None, chart history (per-probe ``inputs.json::ohlcv_start_ms`` metadata). ``ohlcv_end_ms`` symmetrically drops bars whose timestamp is above that bound (inclusive keep). + + Returns ``(effective_bars, effective_count, source_feed_sha256)``. The hash + covers every parsed source row before either bound is applied. """ # Vectorized load: parsing the CSV and building the BarC[] with a per-bar # Python/ctypes loop is ~99% of a run's wall time (the C++ backtest itself is @@ -882,35 +924,47 @@ def _load_bars(csv_path: Path, *, ohlcv_start_ms: int | None = None, col = {name: header.index(name) for name in ("open", "high", "low", "close", "volume", "timestamp")} a = _np.loadtxt(csv_path, delimiter=",", skiprows=1, ndmin=2) - if a.size: + source_n = len(a) + dt = _np.dtype([("open", "<f8"), ("high", "<f8"), ("low", "<f8"), + ("close", "<f8"), ("volume", "<f8"), ("timestamp", "<i8")]) + if dt.itemsize != _SOURCE_FEED_RECORD.size: # pragma: no cover - invariant + raise RuntimeError("canonical OHLCV dtype does not match <5dq>") + source_rows = _np.empty(source_n, dtype=dt) + for name in ("open", "high", "low", "close", "volume"): + source_rows[name] = a[:, col[name]] if source_n else [] + source_rows["timestamp"] = ( + a[:, col["timestamp"]].astype("<i8") if source_n else []) + feed_hasher = _new_source_feed_hasher() + feed_hasher.update(memoryview(source_rows)) + source_feed_sha256 = feed_hasher.hexdigest() + + rows = source_rows + if a.size and (ohlcv_start_ms is not None or ohlcv_end_ms is not None): ts = a[:, col["timestamp"]] keep = _np.ones(len(a), dtype=bool) if ohlcv_start_ms is not None: keep &= ts >= ohlcv_start_ms if ohlcv_end_ms is not None: keep &= ts <= ohlcv_end_ms - a = a[keep] - n = len(a) - dt = _np.dtype([("open", "<f8"), ("high", "<f8"), ("low", "<f8"), - ("close", "<f8"), ("volume", "<f8"), ("timestamp", "<i8")]) - s = _np.empty(n, dtype=dt) - for name in ("open", "high", "low", "close", "volume"): - s[name] = a[:, col[name]] if n else a[:0] - s["timestamp"] = a[:, col["timestamp"]].astype("<i8") if n else a[:0] - bars = (BarC * n).from_buffer_copy(s.tobytes()) if n else (BarC * 0)() - return bars, n + rows = source_rows[keep] + n = len(rows) + bars = (BarC * n).from_buffer_copy(rows.tobytes()) if n else (BarC * 0)() + return bars, n, source_feed_sha256 # Fallback (no numpy): explicit parse + per-bar build. rows: list[tuple[float, float, float, float, float, int]] = [] + feed_hasher = _new_source_feed_hasher() with csv_path.open(newline="", encoding="utf-8") as f: reader = csv.DictReader(f) for row in reader: ts = int(row["timestamp"]) + parsed = (float(row["open"]), float(row["high"]), float(row["low"]), + float(row["close"]), float(row["volume"]), ts) + _update_source_feed_hash(feed_hasher, parsed) if ohlcv_start_ms is not None and ts < ohlcv_start_ms: continue if ohlcv_end_ms is not None and ts > ohlcv_end_ms: continue - rows.append((float(row["open"]), float(row["high"]), float(row["low"]), - float(row["close"]), float(row["volume"]), ts)) + rows.append(parsed) n = len(rows) bars = (BarC * n)() for i, (o, h, l, c, v, ts) in enumerate(rows): @@ -920,7 +974,7 @@ def _load_bars(csv_path: Path, *, ohlcv_start_ms: int | None = None, bars[i].close = c bars[i].volume = v bars[i].timestamp = ts - return bars, n + return bars, n, feed_hasher.hexdigest() def _report_to_dict(r: ReportC) -> dict: @@ -1346,6 +1400,7 @@ def main() -> int: inputs_applied, overrides_applied, runtime, + source_feed_sha256=report.get("source_feed_sha256"), )) args.fingerprint_json.parent.mkdir(parents=True, exist_ok=True) with args.fingerprint_json.open("w", encoding="utf-8") as f: