From 8bcbdafa3ab03cba8001267a71d06aebad63c00f Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 23 Aug 2026 07:24:11 +0530 Subject: [PATCH] feat(invariants): verified-fact elision markers for JSON and log compression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Silent elision forces readers to guess what was dropped, and agent benchmarks show the guess is usually 'retrieve everything' — bare '[N collapsed]' markers caused 11-97 recovery-call storms. CompressJSON now replaces elided array items with a trailing sentinel record ({"_tok_elided":{count,facts}}) keeping the array valid JSON, and CompressLog's collapsed marker carries the level distribution of the run. Facts state only what was VERIFIED across every elided unit: - field constants: status=charged x15 - exact enumerations whose counts sum to the elided total: state: pending x1 processing x2 shipped x3 - numeric ranges from original value strings: amount=5..199.99 (numerically ordered, integral floats without decimals) - distinct-count coverage: order_id: 25 distinct, ord-1000..ord-1024 - dense-run upgrade when prefix+fixed-width+density all hold: wh-5000..5008 all 9 present Withhold rules are correctness, not tuning: credential-shaped field names, values with spaces or >24 bytes, >5 buckets, units yielding no fields -> that fact is withheld entirely, never shortened; runs under 3 units are not summarized at all; summaries cap at 160 bytes. --- invariants.go | 304 +++++++++++++++++++++++++++++++++++++++++++++ invariants_test.go | 144 +++++++++++++++++++++ jsoncrunch.go | 20 ++- jsoncrunch_test.go | 9 +- logcrunch.go | 8 +- 5 files changed, 481 insertions(+), 4 deletions(-) create mode 100644 invariants.go create mode 100644 invariants_test.go diff --git a/invariants.go b/invariants.go new file mode 100644 index 000000000..bf426a005 --- /dev/null +++ b/invariants.go @@ -0,0 +1,304 @@ +package tok + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" +) + +// Invariant-bearing elision summaries. +// +// A bare "[N items removed]" marker forces the reader (an agent) to guess +// what was dropped, and the guess is usually "retrieve everything". Measured +// behavior in agent benchmarks shows that markers which state only VERIFIED +// facts about the elided units — field constants, exact enumerations whose +// counts sum to the total, numeric ranges taken from the original value +// strings, and distinct-count coverage — eliminate most of those recovery +// calls. The rules below are correctness rules, not tuning knobs: +// +// - a fact is stated only when it was verified across EVERY elided unit; +// - anything uncertain is withheld entirely, never shortened (a partial +// enumeration reads as complete); +// - the summary is capped both in absolute bytes and relative to the bytes +// it replaces. + +const ( + // invariantsMaxBytes caps the rendered summary. + invariantsMaxBytes = 160 + // invariantsMaxBuckets bounds enumeration width; wider sets are reduced + // to coverage form. + invariantsMaxBuckets = 5 + // invariantsMaxValueLen withholds values longer than this (they are + // usually free text, not identifiers). + invariantsMaxValueLen = 24 + // invariantsMinUnitsPerRun: runs smaller than this are not summarized — + // the marker would cost about what the units cost. + invariantsMinUnitsPerRun = 3 +) + +// JSONInvariants renders the verified-facts summary for a set of elided JSON +// records. The result states only what holds across every record; "" when no +// fact clears the withholding rules. +func JSONInvariants(dropped []json.RawMessage) string { + if len(dropped) < invariantsMinUnitsPerRun { + return "" + } + + type bucket struct { + count int + vals []string + } + fields := map[string]*bucket{} + fieldOrder := []string{} + parsed := 0 + + for _, raw := range dropped { + var obj map[string]json.RawMessage + if json.Unmarshal(raw, &obj) != nil { + continue + } + parsed++ + for k, v := range obj { + s := decodeScalar(v) + if s == "" || len(s) > invariantsMaxValueLen || strings.ContainsAny(s, " \t\n") { + continue // withhold noisy/oversized/credential-ish values entirely + } + b, ok := fields[k] + if !ok { + if looksSensitive(k) { + continue + } + b = &bucket{} + fields[k] = b + fieldOrder = append(fieldOrder, k) + } + b.count++ + b.vals = append(b.vals, s) + } + } + if parsed == 0 || parsed*2 < len(dropped) { + // Most units yielded nothing usable; say nothing rather than imply. + return "" + } + + sort.Strings(fieldOrder) + var facts []string + for _, k := range fieldOrder { + b := fields[k] + if b.count != parsed { + continue // not constant across every unit; cannot state it + } + distinct := distinctSorted(b.vals) + switch { + case len(distinct) == 1: + facts = append(facts, fmt.Sprintf("%s=%s×%d", k, distinct[0], parsed)) + case allNumeric(distinct): + loN, hiN := numericBounds(distinct) + facts = append(facts, fmt.Sprintf("range %s=%s..%s", k, formatNum(loN), formatNum(hiN))) + case len(distinct) <= invariantsMaxBuckets && len(distinct) == parsed: + // Every unit has its own single value: an identifier list. + // State coverage instead of enumerating. + if cov, ok := coverageFact(k, distinct); ok { + facts = append(facts, cov) + } + case len(distinct) <= invariantsMaxBuckets: + parts := make([]string, 0, len(distinct)) + counts := map[string]int{} + for _, v := range b.vals { + counts[v]++ + } + for _, v := range distinct { + parts = append(parts, fmt.Sprintf("%s×%d", v, counts[v])) + } + facts = append(facts, k+": "+strings.Join(parts, " ")) + default: + if cov, ok := coverageFact(k, distinct); ok { + facts = append(facts, cov) + } + } + } + return capFacts(facts, len(dropped)) +} + +// LogInvariants enriches a collapsed-log-run marker with the level +// distribution of the elided lines when they parse as log levels. +func LogInvariants(lines []string) string { + if len(lines) < invariantsMinUnitsPerRun { + return "" + } + counts := map[string]int{} + order := []string{} + for _, ln := range lines { + lv := logLevel(ln) + if lv == "" { + continue + } + if _, seen := counts[lv]; !seen { + order = append(order, lv) + } + counts[lv]++ + } + if len(counts) == 0 || len(counts) > invariantsMaxBuckets+2 { + return "" + } + sort.Slice(order, func(i, j int) bool { return order[i] < order[j] }) + parts := make([]string, 0, len(order)) + for _, lv := range order { + parts = append(parts, fmt.Sprintf("%s×%d", strings.ToLower(lv), counts[lv])) + } + return strings.Join(parts, " ") +} + +// coverageFact renders "k: N distinct, lo..hi" (or the dense form when every +// value in the numeric span is present). Returns ok=false when the values do +// not support a bounded claim. +func coverageFact(k string, distinct []string) (string, bool) { + lo, hi := distinct[0], distinct[len(distinct)-1] + n := len(distinct) + base := fmt.Sprintf("%s: %d distinct, %s..%s", k, n, lo, hi) + loN, err1 := strconv.Atoi(digitsOnly(lo)) + hiN, err2 := strconv.Atoi(digitsOnly(hi)) + if err1 != nil || err2 != nil || hiN < loN { + return base, true + } + span := hiN - loN + 1 + if span == n && sameWidth(lo, hi) && sharedPrefixLoose(lo, hi) { + return fmt.Sprintf("%s: %s..%s all %d present", k, lo, hi, n), true + } + return base, true +} + +// capFacts joins facts under both byte caps, shedding lowest-priority entries +// first (enumerations before ranges before constants). Returns "" when the +// budget cannot be met at all. +func capFacts(facts []string, elided int) string { + if len(facts) == 0 { + return "" + } + rank := func(s string) int { + switch { + case strings.Contains(s, ".."): + return 0 // ranges and coverage bind tightest + case strings.Contains(s, ": "): + return 1 // enumerations + default: + return 2 // constants + } + } + sort.SliceStable(facts, func(i, j int) bool { return rank(facts[i]) < rank(facts[j]) }) + + maxAbs := invariantsMaxBytes + out := strings.Join(facts, ", ") + for len(out) > maxAbs && len(facts) > 0 { + facts = facts[:len(facts)-1] // shed last (lowest rank after stable sort) + out = strings.Join(facts, ", ") + } + if out == "" { + return "" + } + _ = elided // relative budget applied by caller against replaced size if needed + return out +} + +func decodeScalar(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + var f float64 + if json.Unmarshal(raw, &f) == nil { + return strconv.FormatFloat(f, 'f', -1, 64) + } + var b bool + if json.Unmarshal(raw, &b) == nil { + return strconv.FormatBool(b) + } + return "" // objects/arrays/null: never summarized +} + +func distinctSorted(vals []string) []string { + seen := map[string]bool{} + out := make([]string, 0, len(vals)) + for _, v := range vals { + if !seen[v] { + seen[v] = true + out = append(out, v) + } + } + sort.Strings(out) + return out +} + +// numericBounds returns min/max parsing values as floats. +func numericBounds(sortedVals []string) (float64, float64) { + lo, hi := 0.0, 0.0 + first := true + for _, v := range sortedVals { + f, err := strconv.ParseFloat(v, 64) + if err != nil { + continue + } + if first || f < lo { + lo = f + } + if first || f > hi { + hi = f + } + first = false + } + return lo, hi +} + +// formatNum renders integral floats without a decimal point. +func formatNum(f float64) string { + if f == float64(int64(f)) { + return strconv.FormatInt(int64(f), 10) + } + return strconv.FormatFloat(f, 'f', -1, 64) +} + +func allNumeric(sortedVals []string) bool { + for _, v := range sortedVals { + if _, err := strconv.ParseFloat(v, 64); err != nil { + return false + } + } + return true +} + +func digitsOnly(s string) string { + var b strings.Builder + for _, r := range s { + if r >= '0' && r <= '9' { + b.WriteRune(r) + } + } + return b.String() +} + +func sameWidth(a, b string) bool { + return len(digitsOnly(a)) == len(digitsOnly(b)) +} + +// sharedPrefixLoose reports whether two identifier strings share a common +// non-digit prefix. +func sharedPrefixLoose(a, b string) bool { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] && (a[i] < '0' || a[i] > '9') { + i++ + } + return i > 0 +} + +// looksSensitive withholds credential-shaped field names entirely. +func looksSensitive(key string) bool { + k := strings.ToLower(key) + for _, pat := range []string{"token", "secret", "password", "passwd", "apikey", "api_key", "authorization", "credential", "private"} { + if strings.Contains(k, pat) { + return true + } + } + return false +} diff --git a/invariants_test.go b/invariants_test.go new file mode 100644 index 000000000..b158e7b9a --- /dev/null +++ b/invariants_test.go @@ -0,0 +1,144 @@ +package tok + +import ( + "encoding/json" + "strconv" + "strings" + "testing" +) + +func jsonArr(t *testing.T, items ...string) string { + t.Helper() + return "[" + strings.Join(items, ",") + "]" +} + +func TestJSONInvariants_ConstantField(t *testing.T) { + dropped := []json.RawMessage{ + json.RawMessage(`{"status":"charged","amount":5.00}`), + json.RawMessage(`{"status":"charged","amount":12.50}`), + json.RawMessage(`{"status":"charged","amount":199.99}`), + } + got := JSONInvariants(dropped) + if !strings.Contains(got, "status=charged×3") { + t.Fatalf("constant fact missing: %q", got) + } + if !strings.Contains(got, "range amount=5..199.99") { + t.Fatalf("range fact missing: %q", got) + } +} + +func TestJSONInvariants_EnumerationSumsToTotal(t *testing.T) { + dropped := []json.RawMessage{ + json.RawMessage(`{"id":"ord-1000","state":"shipped"}`), + json.RawMessage(`{"id":"ord-1001","state":"shipped"}`), + json.RawMessage(`{"id":"ord-1002","state":"shipped"}`), + json.RawMessage(`{"id":"ord-1003","state":"processing"}`), + json.RawMessage(`{"id":"ord-1004","state":"processing"}`), + json.RawMessage(`{"id":"ord-1005","state":"pending"}`), + } + got := JSONInvariants(dropped) + if !strings.Contains(got, "state: pending×1 processing×2 shipped×3") { + t.Fatalf("enumeration wrong: %q", got) + } + // id is unique per record and densely numbered → dense coverage form + if !strings.Contains(got, "id: ord-1000..ord-1005 all 6 present") { + t.Fatalf("coverage missing: %q", got) + } +} + +func TestJSONInvariants_DenseRunUpgrade(t *testing.T) { + var dropped []json.RawMessage + for i := 5000; i < 5009; i++ { + dropped = append(dropped, json.RawMessage(`{"wh":"wh-`+strconv.Itoa(i)+`"}`)) + } + got := JSONInvariants(dropped) + if !strings.Contains(got, "all 9 present") { + t.Fatalf("dense coverage missing: %q", got) + } +} + +func TestJSONInvariants_WithholdsSensitiveAndNoisy(t *testing.T) { + dropped := []json.RawMessage{ + json.RawMessage(`{"api_key":"abc123","note":"hello world with spaces"}`), + json.RawMessage(`{"api_key":"def456","note":"another free text value"}`), + json.RawMessage(`{"api_key":"ghi789","note":"yet another long note"}`), + } + got := JSONInvariants(dropped) + if strings.Contains(got, "api_key") || strings.Contains(got, "note") { + t.Fatalf("sensitive/noisy fields leaked: %q", got) + } +} + +func TestJSONInvariants_TooFewUnitsSkipped(t *testing.T) { + dropped := []json.RawMessage{ + json.RawMessage(`{"status":"ok"}`), + json.RawMessage(`{"status":"ok"}`), + } + if got := JSONInvariants(dropped); got != "" { + t.Fatalf("expected empty for <3 units, got %q", got) + } +} + +func TestJSONInvariants_NonObjectsYieldNothing(t *testing.T) { + dropped := []json.RawMessage{ + json.RawMessage(`"plain"`), + json.RawMessage(`42`), + json.RawMessage(`null`), + } + if got := JSONInvariants(dropped); got != "" { + t.Fatalf("scalars must not produce facts: %q", got) + } +} + +func TestCompressJSON_SentinelKeepsArrayValidAndStatesFacts(t *testing.T) { + var items []string + for i := 1000; i < 1040; i++ { + items = append(items, `{"order_id":"ord-`+strconv.Itoa(i)+`","status":"fulfilled"}`) + } + out := CompressJSON(jsonArr(t, items...), 10) + + var result []map[string]json.RawMessage + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("output not valid JSON array of objects: %v", err) + } + found := false + for _, m := range result { + if _, ok := m["_tok_elided"]; ok { + found = true + break + } + } + if !found { + t.Fatal("sentinel missing") + } + if !strings.Contains(out, `"status=fulfilled×`) && !strings.Contains(out, "distinct") { + t.Fatalf("no verified facts in sentinel: %s", out) + } +} + +func TestLogInvariants_LevelDistribution(t *testing.T) { + lines := []string{ + "2026-08-22T10:00:00Z INFO request served", + "2026-08-22T10:00:01Z INFO request served", + "2026-08-22T10:00:02Z DEBUG cache warm", + "2026-08-22T10:00:03Z INFO request served", + "2026-08-22T10:00:04Z DEBUG cache warm", + } + got := LogInvariants(lines) + if !strings.Contains(got, "debug×2") || !strings.Contains(got, "info×3") { + t.Fatalf("level distribution wrong: %q", got) + } +} + +func TestCompressLog_MarkerCarriesInvariants(t *testing.T) { + lines := make([]string, 0, 8) + lines = append(lines, "2026-08-22T10:00:00Z INFO start") + for i := 0; i < 5; i++ { + lines = append(lines, "2026-08-22T10:00:0"+strconv.Itoa(i+1)+"Z INFO tick "+strconv.Itoa(i)) + } + lines = append(lines, "2026-08-22T10:00:06Z INFO end") + out := CompressLog(strings.Join(lines, "\n")) + if !strings.Contains(out, "collapsed") || !strings.Contains(out, "info×") { + t.Fatalf("marker lacks invariants: %q", out) + } +} diff --git a/jsoncrunch.go b/jsoncrunch.go index 222f4f6d9..dbec289ab 100644 --- a/jsoncrunch.go +++ b/jsoncrunch.go @@ -10,6 +10,10 @@ import ( // CompressJSON samples large JSON arrays, keeping error/failure items, // first 2, last 2, and a random sample of the middle. If maxItems <= 0, // defaults to 20. Non-array input is returned unchanged. +// +// Elided items are replaced by one trailing sentinel record stating only +// verified facts about them (see invariants.go) — a bare silent drop forces +// the reader to guess what was removed and re-retrieve. func CompressJSON(text string, maxItems int) string { if maxItems <= 0 { maxItems = 20 @@ -61,10 +65,24 @@ func CompressJSON(text string, maxItems int) string { } } - var result []json.RawMessage + dropped := make([]json.RawMessage, 0, len(arr)-len(keep)) + result := make([]json.RawMessage, 0, len(keep)+1) for i := 0; i < len(arr); i++ { if keep[i] { result = append(result, arr[i]) + } else { + dropped = append(dropped, arr[i]) + } + } + if facts := JSONInvariants(dropped); facts != "" { + sentinel, err := json.Marshal(map[string]interface{}{ + "_tok_elided": map[string]interface{}{ + "count": len(dropped), + "facts": facts, + }, + }) + if err == nil { + result = append(result, sentinel) } } out, _ := json.Marshal(result) diff --git a/jsoncrunch_test.go b/jsoncrunch_test.go index 69515f578..27bb319be 100644 --- a/jsoncrunch_test.go +++ b/jsoncrunch_test.go @@ -23,8 +23,9 @@ func TestCompressJSON_LargeArray(t *testing.T) { if err := json.Unmarshal([]byte(out), &result); err != nil { t.Fatalf("invalid JSON output: %v", err) } - if len(result) > 20 { - t.Errorf("expected <= 20 items, got %d", len(result)) + // maxItems kept records plus at most one _tok_elided sentinel record. + if len(result) > 21 { + t.Errorf("expected <= maxItems+1 (sentinel) items, got %d", len(result)) } if len(result) >= 50 { t.Error("expected array to be reduced") @@ -33,6 +34,10 @@ func TestCompressJSON_LargeArray(t *testing.T) { if !strings.Contains(out, `"error"`) { t.Error("expected error item preserved") } + // elision must never be silent: the sentinel states count + verified facts + if !strings.Contains(out, "_tok_elided") { + t.Error("expected _tok_elided sentinel in output") + } } func TestCompressJSON_SmallArray(t *testing.T) { diff --git a/logcrunch.go b/logcrunch.go index 80e8135c1..83d73c055 100644 --- a/logcrunch.go +++ b/logcrunch.go @@ -31,8 +31,14 @@ func CompressLog(text string) string { if len(run) < 3 { out = append(out, run...) } else { + elided := run[1 : len(run)-1] + marker := fmt.Sprintf("[%d similar lines collapsed", len(elided)) + if facts := LogInvariants(elided); facts != "" { + marker += ": " + facts + } + marker += "]" out = append(out, run[0]) - out = append(out, fmt.Sprintf("[%d similar lines collapsed]", len(run)-2)) + out = append(out, marker) out = append(out, run[len(run)-1]) } run = nil