From cd5ee7cba7b48bb5964bd8af1bbcaf984bba0639 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Jul 2026 12:30:47 +0200 Subject: [PATCH 1/6] Print summary of test failures --- .github/workflows/push.yml | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index dfc66648f3..0e5b156b44 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -152,6 +152,77 @@ jobs: if-no-files-found: warn retention-days: 7 + # `task test` prints a lot of output that GitHub is slow to render, so + # surface just the failed test names on the job summary page (and in this + # step's log) for a quick read of what broke. jq is already used by the + # testmask job above; guard for the rare matrix runner that lacks it + # rather than erroring on an already-failed job. + - name: Summarize failed tests + if: ${{ failure() }} + run: | + if ! command -v jq >/dev/null 2>&1; then + echo "jq not available; skipping failed-test summary" + exit 0 + fi + if [ ! -f test-output.json ]; then + echo "test-output.json not found; skipping failed-test summary" + exit 0 + fi + # test-output.json is the concatenated gotestsum --jsonfile output + # (go test -json events). A test can appear more than once because of + # --rerun-fails, so group by package+test and keep only those whose + # last result was a failure (recovered flakes end on "pass"). + rows=$(jq -rn ' + [ inputs | select(.Test != null and (.Action == "pass" or .Action == "fail" or .Action == "skip")) ] + | group_by([.Package, .Test]) + | map(select(.[-1].Action == "fail") | "| \(.[0].Package) | `\(.[0].Test)` |") + | .[] + ' test-output.json) + # A build error or panic surfaces as a package-level "fail" event (no + # Test) with no failing test in the package. go test -json also emits a + # package-level "fail" for every package that merely has a failing test, + # so exclude packages already listed above; otherwise those packages get + # double-reported here as spurious build errors. Group by package and + # keep only those whose last result was a failure (handles --rerun-fails + # attempts that later passed). + pkgs=$(jq -rn ' + [ inputs | select(.Action == "pass" or .Action == "fail" or .Action == "skip") ] + | ( map(select(.Test != null)) + | group_by([.Package, .Test]) + | map(select(.[-1].Action == "fail") | .[0].Package) + ) as $failed_test_pkgs + | map(select(.Test == null)) + | group_by(.Package) + | map(select(.[-1].Action == "fail") | .[0].Package) + | unique + | map(select(. as $p | $failed_test_pkgs | index($p) | not)) + | .[] + ' test-output.json) + { + echo "## Failed tests" + if [ -n "$rows" ]; then + echo + echo "| Package | Test |" + echo "| --- | --- |" + echo "$rows" + fi + if [ -n "$pkgs" ]; then + echo + echo "### Packages that failed without a test (build error / panic)" + echo + echo '```' + echo "$pkgs" + echo '```' + fi + if [ -z "$rows" ] && [ -z "$pkgs" ]; then + echo + echo "No failed tests found in test-output.json (the failure may be outside the test run)." + fi + } >> "$GITHUB_STEP_SUMMARY" + echo "Failed tests:" + echo "$rows" + echo "$pkgs" + - name: Check no files changed or appeared after running tests run: | # Register untracked files with intent-to-add so `git diff` reports them From 6d8cd67071faf0c371968cf500037639503a6bbb Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Jul 2026 15:36:58 +0200 Subject: [PATCH 2/6] Force some test failures --- cmd/apps/init_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 6d496dbcfe..3c50e11b9d 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -70,6 +70,7 @@ func TestIsTextFile(t *testing.T) { for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { + if tt.expected && filepath.Ext(tt.path) != "" && tt.path != ".env" { t.Fatal("intentional CI summary test failure") } result := isTextFile(tt.path) assert.Equal(t, tt.expected, result) }) From e6b01bb4211639a9f2878d56b09b15601d0635ef Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Jul 2026 15:37:34 +0200 Subject: [PATCH 3/6] Revert "Force some test failures" This reverts commit 6d8cd67071faf0c371968cf500037639503a6bbb. --- cmd/apps/init_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 3c50e11b9d..6d496dbcfe 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -70,7 +70,6 @@ func TestIsTextFile(t *testing.T) { for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { - if tt.expected && filepath.Ext(tt.path) != "" && tt.path != ".env" { t.Fatal("intentional CI summary test failure") } result := isTextFile(tt.path) assert.Equal(t, tt.expected, result) }) From 2b460b925808de3879c852b1f5fa6bca641d4140 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Jul 2026 16:52:53 +0200 Subject: [PATCH 4/6] Use tools script instead of inline bash --- .github/workflows/push.yml | 68 +-------------------- tools/summarize_failed_tests.py | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 66 deletions(-) create mode 100755 tools/summarize_failed_tests.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0e5b156b44..bd9a1972f7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -154,74 +154,10 @@ jobs: # `task test` prints a lot of output that GitHub is slow to render, so # surface just the failed test names on the job summary page (and in this - # step's log) for a quick read of what broke. jq is already used by the - # testmask job above; guard for the rare matrix runner that lacks it - # rather than erroring on an already-failed job. + # step's log) for a quick read of what broke. - name: Summarize failed tests if: ${{ failure() }} - run: | - if ! command -v jq >/dev/null 2>&1; then - echo "jq not available; skipping failed-test summary" - exit 0 - fi - if [ ! -f test-output.json ]; then - echo "test-output.json not found; skipping failed-test summary" - exit 0 - fi - # test-output.json is the concatenated gotestsum --jsonfile output - # (go test -json events). A test can appear more than once because of - # --rerun-fails, so group by package+test and keep only those whose - # last result was a failure (recovered flakes end on "pass"). - rows=$(jq -rn ' - [ inputs | select(.Test != null and (.Action == "pass" or .Action == "fail" or .Action == "skip")) ] - | group_by([.Package, .Test]) - | map(select(.[-1].Action == "fail") | "| \(.[0].Package) | `\(.[0].Test)` |") - | .[] - ' test-output.json) - # A build error or panic surfaces as a package-level "fail" event (no - # Test) with no failing test in the package. go test -json also emits a - # package-level "fail" for every package that merely has a failing test, - # so exclude packages already listed above; otherwise those packages get - # double-reported here as spurious build errors. Group by package and - # keep only those whose last result was a failure (handles --rerun-fails - # attempts that later passed). - pkgs=$(jq -rn ' - [ inputs | select(.Action == "pass" or .Action == "fail" or .Action == "skip") ] - | ( map(select(.Test != null)) - | group_by([.Package, .Test]) - | map(select(.[-1].Action == "fail") | .[0].Package) - ) as $failed_test_pkgs - | map(select(.Test == null)) - | group_by(.Package) - | map(select(.[-1].Action == "fail") | .[0].Package) - | unique - | map(select(. as $p | $failed_test_pkgs | index($p) | not)) - | .[] - ' test-output.json) - { - echo "## Failed tests" - if [ -n "$rows" ]; then - echo - echo "| Package | Test |" - echo "| --- | --- |" - echo "$rows" - fi - if [ -n "$pkgs" ]; then - echo - echo "### Packages that failed without a test (build error / panic)" - echo - echo '```' - echo "$pkgs" - echo '```' - fi - if [ -z "$rows" ] && [ -z "$pkgs" ]; then - echo - echo "No failed tests found in test-output.json (the failure may be outside the test run)." - fi - } >> "$GITHUB_STEP_SUMMARY" - echo "Failed tests:" - echo "$rows" - echo "$pkgs" + run: uv run tools/summarize_failed_tests.py test-output.json | tee -a "$GITHUB_STEP_SUMMARY" - name: Check no files changed or appeared after running tests run: | diff --git a/tools/summarize_failed_tests.py b/tools/summarize_failed_tests.py new file mode 100755 index 0000000000..dfcbb564ac --- /dev/null +++ b/tools/summarize_failed_tests.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# /// +""" +Summarize failed tests from a gotestsum --jsonfile output for a GitHub job summary. + +`task test` prints a lot of output that GitHub is slow to render, so this surfaces +just the failed test names as a Markdown table. It reads the concatenated +gotestsum --jsonfile output (go test -json events, one JSON object per line) and +prints Markdown to stdout; the workflow tees that into $GITHUB_STEP_SUMMARY. +""" + +import argparse +import json + +# go test -json result actions. Only these mark the terminal state of a test or +# package; intermediate actions like "run" and "output" are ignored. +RESULT_ACTIONS = ("pass", "fail", "skip") + + +def load_events(path): + """Yield the result events (parsed JSON objects) from a gotestsum jsonfile.""" + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + event = json.loads(line) + if event.get("Action") in RESULT_ACTIONS: + yield event + + +def last_action_by_key(events, key): + """Map each key to the Action of its last event.""" + last = {} + for event in events: + last[key(event)] = event["Action"] + return last + + +def failed_tests(events): + """Return sorted (package, test) pairs whose last result was a failure. + + A test can appear more than once because of --rerun-fails, so we group by + package+test and keep only those whose last result was a failure (recovered + flakes end on "pass"). + """ + test_events = [e for e in events if e.get("Test") is not None] + last = last_action_by_key(test_events, lambda e: (e["Package"], e["Test"])) + return sorted(key for key, action in last.items() if action == "fail") + + +def failed_packages_without_test(events, failed_test_packages): + """Return sorted packages that failed with no failing test (build error / panic). + + A build error or panic surfaces as a package-level "fail" event (Test == null) + with no failing test in the package. go test -json also emits a package-level + "fail" for every package that merely has a failing test, so exclude packages + already reported via failed_tests; otherwise those get double-reported here as + spurious build errors. + """ + package_events = [e for e in events if e.get("Test") is None] + last = last_action_by_key(package_events, lambda e: e["Package"]) + return sorted( + package for package, action in last.items() if action == "fail" and package not in failed_test_packages + ) + + +def render(tests, packages): + lines = ["## Failed tests"] + if tests: + lines += ["", "| Package | Test |", "| --- | --- |"] + lines += [f"| {package} | `{test}` |" for package, test in tests] + if packages: + lines += ["", "### Packages that failed without a test (build error / panic)", "", "```"] + lines += packages + lines += ["```"] + if not tests and not packages: + lines += ["", "No failed tests found in test-output.json (the failure may be outside the test run)."] + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("jsonfile", help="Path to the gotestsum --jsonfile output") + args = parser.parse_args() + + try: + events = list(load_events(args.jsonfile)) + except FileNotFoundError: + print(f"{args.jsonfile} not found; skipping failed-test summary") + return + + tests = failed_tests(events) + packages = failed_packages_without_test(events, {package for package, _ in tests}) + print(render(tests, packages)) + + +if __name__ == "__main__": + main() From 7497a34bb7e55949b0b63aac94772a8aa2e44d57 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Jul 2026 15:36:58 +0200 Subject: [PATCH 5/6] Force some test failures --- cmd/apps/init_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 6d496dbcfe..3c50e11b9d 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -70,6 +70,7 @@ func TestIsTextFile(t *testing.T) { for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { + if tt.expected && filepath.Ext(tt.path) != "" && tt.path != ".env" { t.Fatal("intentional CI summary test failure") } result := isTextFile(tt.path) assert.Equal(t, tt.expected, result) }) From ebe3bd7b04d8203415c5cadfc81228c243b24387 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Fri, 17 Jul 2026 17:08:24 +0200 Subject: [PATCH 6/6] Revert "Force some test failures" This reverts commit 7497a34bb7e55949b0b63aac94772a8aa2e44d57. --- cmd/apps/init_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/apps/init_test.go b/cmd/apps/init_test.go index 3c50e11b9d..6d496dbcfe 100644 --- a/cmd/apps/init_test.go +++ b/cmd/apps/init_test.go @@ -70,7 +70,6 @@ func TestIsTextFile(t *testing.T) { for _, tt := range tests { t.Run(tt.path, func(t *testing.T) { - if tt.expected && filepath.Ext(tt.path) != "" && tt.path != ".env" { t.Fatal("intentional CI summary test failure") } result := isTextFile(tt.path) assert.Equal(t, tt.expected, result) })