From a4688706e1dd1eb07e1f405d3aa29f9f7fd816b5 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sat, 15 Aug 2026 08:25:40 +0000 Subject: [PATCH 1/2] Fix token audit false-positive detectors Claude tool errors were converted into synthetic __ERROR__-prefixed tool names and then rediscovered by prefix matching. Track error results directly from boolean is_error fields so ordinary output and tool names cannot collide with the detector. Normalize both supported file_path and path inputs before redundant-read comparisons. This keeps repeated reads detectable without collapsing different valid files into the same None key. Add regression coverage for false/absent error flags, genuine errors, repeated alias-path reads, and distinct alias-path reads. --- .../reflect/scripts/tests/test_token_audit.py | 75 +++++++++++++++++++ skills/reflect/scripts/token_audit.py | 32 ++++++-- 2 files changed, 99 insertions(+), 8 deletions(-) diff --git a/skills/reflect/scripts/tests/test_token_audit.py b/skills/reflect/scripts/tests/test_token_audit.py index 8b192d1..1176bc7 100644 --- a/skills/reflect/scripts/tests/test_token_audit.py +++ b/skills/reflect/scripts/tests/test_token_audit.py @@ -148,8 +148,83 @@ def test_identical_offset_window_read_twice_is_thrash(self): finally: os.unlink(path) + def test_path_alias_read_twice_is_thrash(self): + u = {"input_tokens": 1, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} + lines = [ + claude_assistant_line("m1", "u1", [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "/a.py"}}], u), + claude_assistant_line("m2", "u2", [{"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "/a.py"}}], u), + ] + path = write_jsonl(lines) + try: + buf = io.StringIO() + with redirect_stdout(buf): + token_audit.audit_claude(path) + out = buf.getvalue() + self.assertIn(" /a.py offset/limit=(None, None)", out) + self.assertIn("redundant re-reads (identical window): 1", out) + finally: + os.unlink(path) + + def test_path_alias_keeps_different_files_distinct(self): + u = {"input_tokens": 1, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} + lines = [ + claude_assistant_line("m1", "u1", [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "/a.py"}}], u), + claude_assistant_line("m2", "u2", [{"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "/b.py"}}], u), + ] + path = write_jsonl(lines) + try: + buf = io.StringIO() + with redirect_stdout(buf): + token_audit.audit_claude(path) + self.assertIn("redundant re-reads (identical window): 0", buf.getvalue()) + finally: + os.unlink(path) + class TestToolErrors(unittest.TestCase): + def test_non_error_result_content_is_not_classified_by_substring(self): + u = {"input_tokens": 1, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} + lines = [ + claude_assistant_line( + "m1", + "u1", + [{"type": "tool_use", "id": "t1", "name": "__ERROR__:diagnostics", "input": {}}], + u, + ), + claude_assistant_line( + "m2", + "u2", + [{"type": "tool_use", "id": "t2", "name": "__ERROR__:search", "input": {}}], + u, + ), + { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "t1", + "is_error": False, + "content": "No error was found", + }, + { + "type": "tool_result", + "tool_use_id": "t2", + "content": "The word error is ordinary output here", + }, + ] + }, + }, + ] + path = write_jsonl(lines) + try: + buf = io.StringIO() + with redirect_stdout(buf): + token_audit.audit_claude(path) + self.assertIn("tool errors: 0", buf.getvalue()) + finally: + os.unlink(path) + def test_error_tool_result_is_flagged(self): u = {"input_tokens": 1, "output_tokens": 1, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} lines = [ diff --git a/skills/reflect/scripts/token_audit.py b/skills/reflect/scripts/token_audit.py index 6e2772b..2a619b2 100644 --- a/skills/reflect/scripts/token_audit.py +++ b/skills/reflect/scripts/token_audit.py @@ -70,6 +70,18 @@ def read_jsonl(path): return out +def _claude_file_path(tool_input): + """Return a comparable path for supported Claude file-tool inputs.""" + if not isinstance(tool_input, dict): + return None + path = tool_input.get("file_path") + if path is None: + path = tool_input.get("path") + if not isinstance(path, str) or not path: + return None + return os.path.normpath(path) + + def audit_claude(path): # Claude Code writes one JSONL line per content block (thinking/text/tool_use), # but every block belonging to the same message.id carries the SAME usage @@ -85,6 +97,7 @@ def audit_claude(path): models = Counter() tool_use = {} tool_calls_seq = [] + tool_errors = [] cache_points = [] seq = 0 simple_turns = 0 # turns whose only tool calls are Read/Grep/Glob - cheap-model candidates @@ -125,10 +138,10 @@ def audit_claude(path): content = d.get("message", {}).get("content") if isinstance(content, list): for block in content: - if isinstance(block, dict) and block.get("type") == "tool_result" and block.get("is_error"): + if isinstance(block, dict) and block.get("type") == "tool_result" and block.get("is_error") is True: tid = block.get("tool_use_id") name, inp, s = tool_use.get(tid, ("?", {}, None)) - tool_calls_seq.append((s, "__ERROR__:" + str(name), inp, tid)) + tool_errors.append((s, name)) print(f"=== CLAUDE CODE token audit: {os.path.basename(path)} ===") print(f"assistant turns: {n_assistant}, models used: {dict(models)}") @@ -160,9 +173,13 @@ def audit_claude(path): last_read_seq, edited_since, redundant = {}, set(), [] for s, name, inp, ident in sorted(tool_calls_seq, key=lambda x: x[0] or 0): if name in ("Edit", "Write") and isinstance(inp, dict): - edited_since.add(inp.get("file_path")) + fp = _claude_file_path(inp) + if fp: + edited_since.add(fp) elif name == "Read" and isinstance(inp, dict): - fp = inp.get("file_path") + fp = _claude_file_path(inp) + if not fp: + continue window = (inp.get("offset"), inp.get("limit")) key = (fp, window) if key in last_read_seq and fp not in edited_since: @@ -173,10 +190,9 @@ def audit_claude(path): print(f" {fp} offset/limit={window} (seq {s1} -> {s2})") print(f"redundant re-reads (identical window): {len(redundant)}") - errors = [(s, name) for s, name, inp, ident in tool_calls_seq if isinstance(name, str) and name.startswith("__ERROR__:")] - print(f"-- tool errors: {len(errors)} --") - for s, name in errors: - print(f" seq {s}: {name[len('__ERROR__:'):]} failed") + print(f"-- tool errors: {len(tool_errors)} --") + for s, name in tool_errors: + print(f" seq {s}: {name} failed") print("-- cache-creation spikes (fresh write, not cache read - expensive path) --") creations = sorted(c for _, c, r in cache_points if c > 0) From bfb97697c7ad451bd98fba4dbeba6b461d904dd3 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sat, 15 Aug 2026 08:26:00 +0000 Subject: [PATCH 2/2] =?UTF-8?q?invoker:=20wf-1786767473811-1/fix-token-aud?= =?UTF-8?q?it-false-positive-detectors=20=E2=80=94=20Auditing=20a=20real?= =?UTF-8?q?=20session=20transcript=20with=20`token=5Faudit.py`=20reported?= =?UTF-8?q?=20"19=20tool=20errors"=20and=20"1=20redundant=20read"=20that?= =?UTF-8?q?=20were=20mostly=20or=20entirely=20wrong=20on=20manual=20inspec?= =?UTF-8?q?tion.=20Two=20separate=20bugs=20were=20traced=20by=20an=20indep?= =?UTF-8?q?endent=20review=20pass=20(not=20yet=20verified=20against=20the?= =?UTF-8?q?=20actual=20current=20source=20--=20read=20it=20first):=201.=20?= =?UTF-8?q?The=20tool-error=20detector=20matches=20on=20a=20substring=20(e?= =?UTF-8?q?.g.=20checking=20whether=20=20=20=20some=20string=20like=20"err?= =?UTF-8?q?or"=20appears=20in=20tool-result=20content)=20instead=20of=20?= =?UTF-8?q?=20=20=20reading=20the=20transcript's=20actual=20structured=20`?= =?UTF-8?q?is=5Ferror`=20flag=20on=20each=20=20=20=20tool=5Fresult=20block?= =?UTF-8?q?.=20This=20flags=20normal=20tool=20output=20that=20merely=20men?= =?UTF-8?q?tions=20=20=20=20the=20word=20"error"=20(e.g.=20in=20file=20con?= =?UTF-8?q?tent,=20log=20lines,=20or=20a=20status=20=20=20=20message)=20as?= =?UTF-8?q?=20if=20the=20tool=20call=20itself=20failed.=202.=20The=20redun?= =?UTF-8?q?dant-read=20detector=20has=20a=20filename-normalization=20bug:?= =?UTF-8?q?=20it=20=20=20=20appears=20to=20produce=20`None`=20or=20`(None,?= =?UTF-8?q?=20None)`=20for=20some=20inputs=20when=20=20=20=20extracting/no?= =?UTF-8?q?rmalizing=20the=20file=20path=20from=20a=20Read=20tool=20call,?= =?UTF-8?q?=20which=20=20=20=20breaks=20path-equality=20comparisons=20acro?= =?UTF-8?q?ss=20reads=20and=20produces=20spurious=20=20=20=20"redundant=20?= =?UTF-8?q?read"=20matches=20(or=20misses=20real=20ones=20--=20confirm=20w?= =?UTF-8?q?hich,=20=20=20=20empirically,=20by=20writing=20the=20repro=20fi?= =?UTF-8?q?rst).=20Review=20claim:=20both=20detectors=20now=20key=20off=20?= =?UTF-8?q?the=20transcript's=20actual=20structured=20data=20(the=20real?= =?UTF-8?q?=20`is=5Ferror`=20flag;=20a=20correctly=20normalized=20file=20p?= =?UTF-8?q?ath)=20instead=20of=20a=20substring/lossy=20heuristic,=20and=20?= =?UTF-8?q?a=20real=20session=20transcript=20that=20previously=20mis-flagg?= =?UTF-8?q?ed=20both=20cases=20no=20longer=20does.=20Review=20lane:=20beha?= =?UTF-8?q?vior=20Safety=20invariant:=20This=20only=20changes=20detection?= =?UTF-8?q?=20logic=20inside=20`token=5Faudit.py`;=20it=20must=20not=20cha?= =?UTF-8?q?nge=20the=20script's=20output=20format/schema=20in=20a=20way=20?= =?UTF-8?q?that=20breaks=20other=20tooling=20that=20consumes=20its=20outpu?= =?UTF-8?q?t=20(check=20for=20other=20callers/consumers=20of=20this=20scri?= =?UTF-8?q?pt's=20output=20before=20changing=20shape).=20Slice=20rationale?= =?UTF-8?q?:=20Both=20bugs=20live=20in=20the=20same=20script=20and=20were?= =?UTF-8?q?=20found=20in=20the=20same=20review=20pass;=20if=20the=20repo's?= =?UTF-8?q?=20own=20conventions=20call=20for=20splitting=20them=20into=20t?= =?UTF-8?q?wo=20separate=20reviewable=20changes=20once=20you=20see=20the?= =?UTF-8?q?=20actual=20diff=20size,=20do=20that=20instead=20of=20forcing?= =?UTF-8?q?=20one=20combined=20change.=20Goal:=20`token=5Faudit.py`=20no?= =?UTF-8?q?=20longer=20flags=20a=20normal=20tool=20result=20that=20merely?= =?UTF-8?q?=20contains=20the=20word=20"error"=20as=20a=20tool=20error,=20a?= =?UTF-8?q?nd=20no=20longer=20produces=20spurious=20redundant-read=20match?= =?UTF-8?q?es=20(or=20misses)=20caused=20by=20a=20broken=20filename=20norm?= =?UTF-8?q?alization.=20Motivation:=20A=20recent=20real=20audit=20run=20on?= =?UTF-8?q?=20a=20genuine=20session=20transcript=20reported=20these=20two?= =?UTF-8?q?=20findings,=20and=20manual=20inspection=20showed=20both=20were?= =?UTF-8?q?=20false=20positives=20--=20undermining=20trust=20in=20every=20?= =?UTF-8?q?future=20run=20of=20this=20report=20until=20fixed.=20Implementa?= =?UTF-8?q?tion=20details:=20|=20=20=20Read=20the=20actual=20current=20sou?= =?UTF-8?q?rce=20of=20`token=5Faudit.py`=20first=20--=20treat=20the=20=20?= =?UTF-8?q?=20bug=20descriptions=20above=20as=20a=20starting=20hypothesis?= =?UTF-8?q?=20from=20an=20independent=20=20=20review=20pass,=20not=20verif?= =?UTF-8?q?ied=20ground=20truth;=20confirm=20the=20exact=20mechanism=20=20?= =?UTF-8?q?=20by=20reading=20the=20real=20code=20and,=20ideally,=20reprodu?= =?UTF-8?q?cing=20against=20a=20real=20or=20=20=20realistic=20session=20tr?= =?UTF-8?q?anscript=20fixture=20before=20changing=20anything.=20=20=20For?= =?UTF-8?q?=20the=20tool-error=20detector:=20find=20wherever=20it=20curren?= =?UTF-8?q?tly=20classifies=20a=20=20=20tool=20call=20as=20failed,=20and?= =?UTF-8?q?=20change=20it=20to=20read=20the=20transcript's=20actual=20=20?= =?UTF-8?q?=20`is=5Ferror`=20field=20on=20the=20relevant=20tool=5Fresult?= =?UTF-8?q?=20content=20block=20instead=20of=20=20=20any=20substring/keywo?= =?UTF-8?q?rd=20match.=20=20=20For=20the=20redundant-read=20detector:=20fi?= =?UTF-8?q?nd=20the=20file-path=20normalization=20=20=20function=20used=20?= =?UTF-8?q?to=20compare=20Read=20calls,=20and=20fix=20whatever=20produces?= =?UTF-8?q?=20`None`=20=20=20or=20`(None,=20None)`=20for=20valid=20inputs?= =?UTF-8?q?=20--=20likely=20a=20missing=20case=20in=20=20=20extracting=20t?= =?UTF-8?q?he=20path=20argument,=20or=20an=20unsafe=20attribute/key=20acce?= =?UTF-8?q?ss=20that=20=20=20silently=20returns=20None=20instead=20of=20ra?= =?UTF-8?q?ising=20or=20handling=20the=20actual=20input=20=20=20shape.=20N?= =?UTF-8?q?on-goals:=20Does=20not=20change=20any=20other=20detector=20or?= =?UTF-8?q?=20report=20section=20in=20`token=5Faudit.py`.=20Does=20not=20a?= =?UTF-8?q?dd=20new=20detectors.=20Does=20not=20touch=20other=20scripts=20?= =?UTF-8?q?in=20this=20repo=20unless=20the=20fix=20genuinely=20requires=20?= =?UTF-8?q?it=20(e.g.=20a=20shared=20helper=20both=20detectors=20call=20in?= =?UTF-8?q?to).=20Acceptance=20criteria:=20-=20A=20new=20test=20(or=20fixt?= =?UTF-8?q?ure-based=20repro,=20matching=20however=20this=20repo=20=20=20a?= =?UTF-8?q?lready=20tests=20scripts=20in=20this=20directory=20--=20check?= =?UTF-8?q?=20for=20an=20existing=20=20=20pattern=20first)=20that=20constr?= =?UTF-8?q?ucts=20a=20minimal=20transcript=20where=20a=20tool=20=20=20resu?= =?UTF-8?q?lt's=20content=20happens=20to=20contain=20the=20word=20"error"?= =?UTF-8?q?=20but=20its=20real=20=20=20`is=5Ferror`=20field=20is=20false/a?= =?UTF-8?q?bsent,=20asserting=20the=20tool-error=20detector=20=20=20does?= =?UTF-8?q?=20NOT=20flag=20it=20--=20proven=20failing=20before=20the=20fix?= =?UTF-8?q?,=20passing=20after.=20-=20A=20second=20test=20proving=20the=20?= =?UTF-8?q?tool-error=20detector=20still=20correctly=20flags=20a=20=20=20t?= =?UTF-8?q?ranscript=20entry=20whose=20`is=5Ferror`=20field=20is=20genuine?= =?UTF-8?q?ly=20true=20(no=20=20=20regression=20in=20real=20detection).=20?= =?UTF-8?q?-=20A=20third=20test=20proving=20the=20redundant-read=20detecto?= =?UTF-8?q?r's=20path=20normalization=20=20=20no=20longer=20returns=20None?= =?UTF-8?q?/(None,=20None)=20for=20the=20specific=20input=20shape=20that?= =?UTF-8?q?=20=20=20broke=20it,=20with=20a=20case=20showing=20two=20reads?= =?UTF-8?q?=20of=20the=20literal=20same=20file=20are=20=20=20still=20corre?= =?UTF-8?q?ctly=20detected=20as=20redundant,=20and=20two=20reads=20of=20ge?= =?UTF-8?q?nuinely=20=20=20different=20files=20are=20not.=20-=20Test=20and?= =?UTF-8?q?=20fix=20committed=20together,=20following=20this=20repo's=20ow?= =?UTF-8?q?n=20existing=20=20=20commit/PR=20conventions=20(read=20any=20CO?= =?UTF-8?q?NTRIBUTING/CLAUDE.md-equivalent=20=20=20docs,=20or=20infer=20fr?= =?UTF-8?q?om=20recent=20commit=20history,=20before=20assuming=20Invoker's?= =?UTF-8?q?=20=20=20specific=20PR-body=20schema=20applies=20here=20--=20th?= =?UTF-8?q?is=20is=20a=20different=20repo).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: a59b419c-ab0a-4ca5-83c8-863a5502efc9