diff --git a/.gitignore b/.gitignore index 8e3be1127..e05bdbfe7 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,7 @@ node_modules/ # macOS Finder metadata .DS_Store + +# Python bytecode (generated when the site-data scripts are imported/run) +__pycache__/ +*.py[cod] diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index fadc448f6..72616569d 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -448,6 +448,22 @@ save_result() { EOF info "saved results/$profile/$CONNS/${FRAMEWORK}.json" + # rebuild_site_data.py refuses to publish a run whose responses were mostly + # errors — rps counts a 500 exactly like a 200. The row and the container + # log are still written above, since they're the post-mortem evidence, but + # say so here: otherwise a clean-looking --save implies a result that will + # never reach the board. Threshold shared via HTTPARENA_MAX_ERROR_PCT. + local ok_count err_count + ok_count=$(( ${BEST_M[status_2xx]:-0} + ${BEST_M[status_3xx]:-0} )) + err_count=$(( ${BEST_M[status_4xx]:-0} + ${BEST_M[status_5xx]:-0} )) + if [ "$ok_count" -eq 0 ]; then + warn "$profile/$CONNS: no successful responses ($err_count errors) — will not be published" + elif awk -v e="$err_count" -v n="$((ok_count + err_count))" \ + -v thr="${HTTPARENA_MAX_ERROR_PCT:-5}" 'BEGIN{exit !(e/n*100 > thr)}'; then + warn "$profile/$CONNS: $(awk -v e="$err_count" -v n="$((ok_count + err_count))" \ + 'BEGIN{printf "%.1f", e/n*100}')% non-2xx/3xx ($err_count of $((ok_count + err_count))) — will not be published" + fi + # Persist container logs alongside results for post-mortem. local log_dir="$ROOT_DIR/site/static/logs/$profile/$CONNS" mkdir -p "$log_dir" diff --git a/scripts/rebuild_site_data.py b/scripts/rebuild_site_data.py index 57598fda6..9842916dd 100755 --- a/scripts/rebuild_site_data.py +++ b/scripts/rebuild_site_data.py @@ -22,6 +22,32 @@ from pathlib import Path +# A run whose responses were overwhelmingly errors is not a measurement. rps +# counts a 500 exactly like a 200, so a framework that collapses under load +# can outrank one that serves every request — php-fpm published 294k rps on +# baseline-h2 with 99.2% 5xx. Rows above this error rate are dropped rather +# than ranked. Override for a one-off rebuild with HTTPARENA_MAX_ERROR_PCT. +MAX_ERROR_PCT = float(os.environ.get("HTTPARENA_MAX_ERROR_PCT", "5")) + + +def failure_reason(entry: dict) -> str | None: + """Why this result row isn't publishable, or None if it is. + + Every load generator in scripts/lib/tools/ reports status counts, so a row + whose 2xx+3xx is zero got nothing back at all — a crash, a hang, or a + refused connection. Publishing its rps=0 states "we measured zero + throughput", which ranks the framework last on a test that never ran. + """ + ok = entry.get("status_2xx", 0) + entry.get("status_3xx", 0) + err = entry.get("status_4xx", 0) + entry.get("status_5xx", 0) + if ok == 0: + return "no successful responses" + pct = err / (ok + err) * 100 + if pct > MAX_ERROR_PCT: + return f"{pct:.1f}% non-2xx/3xx" + return None + + def rebuild_frameworks_json(root: Path, site_data: Path) -> None: """Aggregate every frameworks/*/meta.json into site/data/frameworks.json. @@ -135,6 +161,17 @@ def merge_results(results_dir: Path, site_data: Path) -> None: if stale: print(f"[purged stale] {data_file.name}: {stale}", file=sys.stderr) final = [e for e in final if e.get("framework", "") in valid_names] + + # Applied to the merged list, not just this run's batch, so a bad + # row already sitting in site/data is dropped on the next rebuild + # rather than living on because its framework wasn't re-run. + dropped = [(e.get("framework", ""), failure_reason(e)) for e in final] + dropped = [(name, why) for name, why in dropped if why] + if dropped: + detail = ", ".join(f"{name} ({why})" for name, why in dropped) + print(f"[dropped failed] {data_file.name}: {detail}", file=sys.stderr) + final = [e for e in final if not failure_reason(e)] + data_file.write_text(json.dumps(final, indent=2)) print(f"[updated] {data_file}") diff --git a/site/content/docs/scoring/composite-score.md b/site/content/docs/scoring/composite-score.md index 07a5b0565..771e27015 100644 --- a/site/content/docs/scoring/composite-score.md +++ b/site/content/docs/scoring/composite-score.md @@ -34,6 +34,12 @@ Summing instead of averaging means the composite scales with the number of score Frameworks that don't participate in a scored profile receive 0 for that profile, which lowers their composite by the full 100-point ceiling of that profile. +### Failed runs are not scored + +A run whose responses were mostly errors is not a measurement: rps counts a 500 exactly like a 200, so a framework that collapses under load would otherwise outrank one that serves every request. When the site data is built, a result is dropped rather than ranked if it has **no successful responses at all** (a crash, a hang, or a refused connection — its rps of 0 would read as a measured zero) or if **more than 5% of its responses were non-2xx/3xx**. + +A dropped result is treated exactly like not participating in that profile: the framework scores 0 for it. The raw result and the container log are still kept under `results/` and `site/static/logs/` for post-mortem. + ## Scored vs reference-only profiles Not all profiles count toward the composite score. Profiles marked as **scored** contribute to the composite. Reference-only profiles (marked with **\***) are displayed for comparison but do not affect the ranking. diff --git a/site/data/api-16-1024.json b/site/data/api-16-1024.json index 1e4486c4c..94df3f2d3 100644 --- a/site/data/api-16-1024.json +++ b/site/data/api-16-1024.json @@ -1011,32 +1011,6 @@ "tpl_static": 0, "tpl_async_db": 388837 }, - { - "framework": "rage", - "language": "Ruby", - "rps": 9825, - "avg_latency": "84.56ms", - "p99_latency": "1.78s", - "cpu": "749.7%", - "memory": "6.0GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "47.17MB/s", - "input_bw": "566.09KB/s", - "reconnects": 34413, - "status_2xx": 147382, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 26433, - "tpl_baseline": 62443, - "tpl_json": 68252, - "tpl_db": 0, - "tpl_upload": 0, - "tpl_static": 0, - "tpl_async_db": 16687 - }, { "framework": "rails", "language": "Ruby", diff --git a/site/data/api-4-256.json b/site/data/api-4-256.json index 240b22219..f555bac6b 100644 --- a/site/data/api-4-256.json +++ b/site/data/api-4-256.json @@ -1011,32 +1011,6 @@ "tpl_static": 0, "tpl_async_db": 164332 }, - { - "framework": "rage", - "language": "Ruby", - "rps": 13999, - "avg_latency": "16.30ms", - "p99_latency": "266.00ms", - "cpu": "407.6%", - "memory": "5.9GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "68.87MB/s", - "input_bw": "806.58KB/s", - "reconnects": 45990, - "status_2xx": 209997, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 20194, - "tpl_baseline": 85974, - "tpl_json": 86523, - "tpl_db": 0, - "tpl_upload": 0, - "tpl_static": 0, - "tpl_async_db": 37500 - }, { "framework": "rails", "language": "Ruby", diff --git a/site/data/baseline-h2-1024.json b/site/data/baseline-h2-1024.json index 91dd37df0..66b12825b 100644 --- a/site/data/baseline-h2-1024.json +++ b/site/data/baseline-h2-1024.json @@ -404,26 +404,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "php-fpm", - "language": "PHP", - "rps": 294864, - "avg_latency": "143.45ms", - "p99_latency": "143.45ms", - "cpu": "2169.8%", - "memory": "4.8GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "60.80MB/s", - "input_bw": "", - "reconnects": 0, - "status_2xx": 11817, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 1462506 - }, { "framework": "pyronova", "language": "Python", @@ -558,25 +538,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "symfony-spawn-franken", - "language": "PHP", - "rps": 97817, - "avg_latency": "465.99ms", - "p99_latency": "465.99ms", - "cpu": "4817.0%", - "memory": "7.1GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "57.95MB/s", - "reconnects": 0, - "status_2xx": 494959, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 284226 - }, { "framework": "symfony-spawn-tas", "language": "PHP", diff --git a/site/data/baseline-h2-256.json b/site/data/baseline-h2-256.json index 72a8ecba8..31c83802d 100644 --- a/site/data/baseline-h2-256.json +++ b/site/data/baseline-h2-256.json @@ -558,25 +558,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "symfony-spawn-franken", - "language": "PHP", - "rps": 28243, - "avg_latency": "229.68ms", - "p99_latency": "229.68ms", - "cpu": "5180.8%", - "memory": "4.3GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "76.21MB/s", - "reconnects": 0, - "status_2xx": 142064, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 381252 - }, { "framework": "symfony-spawn-tas", "language": "PHP", diff --git a/site/data/crud-4096.json b/site/data/crud-4096.json index 1855e56ca..be770ba56 100644 --- a/site/data/crud-4096.json +++ b/site/data/crud-4096.json @@ -119,25 +119,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "fasthttp", - "language": "V", - "rps": 0, - "avg_latency": "0us", - "p99_latency": "0us", - "cpu": "0.0%", - "memory": "0MiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "4B/s", - "reconnects": 14506257, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, { "framework": "fishcake", "language": "Kotlin", diff --git a/site/data/pipelined-4096.json b/site/data/pipelined-4096.json index 52725ecf0..a217a0637 100644 --- a/site/data/pipelined-4096.json +++ b/site/data/pipelined-4096.json @@ -18,25 +18,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "apache", - "language": "C", - "rps": 0, - "avg_latency": "26.50ms", - "p99_latency": "250.40ms", - "cpu": "2043.6%", - "memory": "136MiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "136.05MB/s", - "reconnects": 14, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 1858306, - "status_5xx": 0 - }, { "framework": "aspnet-minimal", "language": "C#", @@ -171,25 +152,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "caddy", - "language": "Go", - "rps": 0, - "avg_latency": "47.40ms", - "p99_latency": "1.56s", - "cpu": "4491.3%", - "memory": "362MiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "105.93MB/s", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 3779515, - "status_5xx": 0 - }, { "framework": "dart-io", "language": "Dart", @@ -1059,25 +1021,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "pingora", - "language": "Rust", - "rps": 0, - "avg_latency": "54.31ms", - "p99_latency": "73.60ms", - "cpu": "561.1%", - "memory": "75MiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "10.01MB/s", - "reconnects": 372335, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 372355, - "status_5xx": 0 - }, { "framework": "pyronova", "language": "Python", @@ -1556,25 +1499,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "traefik", - "language": "Go", - "rps": 0, - "avg_latency": "573.13ms", - "p99_latency": "1.29s", - "cpu": "5316.4%", - "memory": "517MiB", - "connections": 4096, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "9.78MB/s", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 534152 - }, { "framework": "trillium", "language": "Rust", diff --git a/site/data/pipelined-512.json b/site/data/pipelined-512.json index 03ac6b812..fab187253 100644 --- a/site/data/pipelined-512.json +++ b/site/data/pipelined-512.json @@ -18,25 +18,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "apache", - "language": "C", - "rps": 0, - "avg_latency": "21.35ms", - "p99_latency": "197.50ms", - "cpu": "2018.6%", - "memory": "82MiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "137.31MB/s", - "reconnects": 5, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 1875336, - "status_5xx": 0 - }, { "framework": "aspnet-minimal", "language": "C#", @@ -171,25 +152,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "caddy", - "language": "Go", - "rps": 0, - "avg_latency": "13.48ms", - "p99_latency": "196.80ms", - "cpu": "3539.8%", - "memory": "122MiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "84.14MB/s", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 3002243, - "status_5xx": 0 - }, { "framework": "dart-io", "language": "Dart", @@ -1059,25 +1021,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "pingora", - "language": "Rust", - "rps": 0, - "avg_latency": "6.77ms", - "p99_latency": "7.65ms", - "cpu": "576.5%", - "memory": "69MiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "10.05MB/s", - "reconnects": 373945, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 373972, - "status_5xx": 0 - }, { "framework": "pyronova", "language": "Python", @@ -1556,25 +1499,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "traefik", - "language": "Go", - "rps": 0, - "avg_latency": "71.62ms", - "p99_latency": "138.20ms", - "cpu": "5337.0%", - "memory": "181MiB", - "connections": 512, - "threads": 64, - "duration": "5s", - "pipeline": 16, - "bandwidth": "10.45MB/s", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 571145 - }, { "framework": "trillium", "language": "Rust", diff --git a/site/data/static-h2-1024.json b/site/data/static-h2-1024.json index b18360084..819e12783 100644 --- a/site/data/static-h2-1024.json +++ b/site/data/static-h2-1024.json @@ -95,25 +95,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "effinitive", - "language": "C#", - "rps": 0, - "avg_latency": "", - "p99_latency": "", - "cpu": "6187.1%", - "memory": "4.3GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "0", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, { "framework": "fastendpoints", "language": "C#", @@ -558,25 +539,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "symfony-spawn-franken", - "language": "PHP", - "rps": 52582, - "avg_latency": "246.82ms", - "p99_latency": "246.82ms", - "cpu": "4915.1%", - "memory": "5.1GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "123.87MB/s", - "reconnects": 0, - "status_2xx": 266595, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 346287 - }, { "framework": "symfony-spawn-tas", "language": "PHP", @@ -615,25 +577,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "trillium-tuned", - "language": "Rust", - "rps": 0, - "avg_latency": "", - "p99_latency": "", - "cpu": "2590.0%", - "memory": "5.0GiB", - "connections": 1024, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "0", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, { "framework": "true-async-server", "language": "PHP", diff --git a/site/data/static-h2-256.json b/site/data/static-h2-256.json index 264571722..7fe6959ff 100644 --- a/site/data/static-h2-256.json +++ b/site/data/static-h2-256.json @@ -558,25 +558,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "symfony-spawn-franken", - "language": "PHP", - "rps": 63518, - "avg_latency": "64.13ms", - "p99_latency": "64.13ms", - "cpu": "5491.9%", - "memory": "2.2GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "637.75MB/s", - "reconnects": 0, - "status_2xx": 320134, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 286846 - }, { "framework": "symfony-spawn-tas", "language": "PHP", @@ -615,25 +596,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "trillium-tuned", - "language": "Rust", - "rps": 0, - "avg_latency": "", - "p99_latency": "", - "cpu": "1486.4%", - "memory": "5.1GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "0", - "reconnects": 0, - "status_2xx": 0, - "status_3xx": 0, - "status_4xx": 0, - "status_5xx": 0 - }, { "framework": "true-async-server", "language": "PHP", diff --git a/site/data/upload-256.json b/site/data/upload-256.json index 3c66d6358..22f664dd6 100644 --- a/site/data/upload-256.json +++ b/site/data/upload-256.json @@ -798,26 +798,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "phoenix-bandit", - "language": "Elixir", - "rps": 685, - "avg_latency": "194.10ms", - "p99_latency": "894.30ms", - "cpu": "1835.1%", - "memory": "3.1GiB", - "connections": 256, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "183.74KB/s", - "input_bw": "5.43GB/s", - "reconnects": 3351, - "status_2xx": 3425, - "status_3xx": 0, - "status_4xx": 1714, - "status_5xx": 0 - }, { "framework": "php-fpm", "language": "PHP", diff --git a/site/data/upload-32.json b/site/data/upload-32.json index 843c16c90..cbfafdc20 100644 --- a/site/data/upload-32.json +++ b/site/data/upload-32.json @@ -798,26 +798,6 @@ "status_4xx": 0, "status_5xx": 0 }, - { - "framework": "phoenix-bandit", - "language": "Elixir", - "rps": 704, - "avg_latency": "24.18ms", - "p99_latency": "100.00ms", - "cpu": "1208.7%", - "memory": "878MiB", - "connections": 32, - "threads": 64, - "duration": "5s", - "pipeline": 1, - "bandwidth": "188.98KB/s", - "input_bw": "5.58GB/s", - "reconnects": 3518, - "status_2xx": 3523, - "status_3xx": 0, - "status_4xx": 1761, - "status_5xx": 0 - }, { "framework": "php-fpm", "language": "PHP",