From 6acb9ef2094f71c5a3a43a126a5b5fbab20a24d5 Mon Sep 17 00:00:00 2001 From: nol4lej Date: Thu, 16 Jul 2026 11:17:28 -0400 Subject: [PATCH 1/3] feat(caddy): log resolved rate-limit key for debugging --- docker/testnet/Caddyfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/testnet/Caddyfile b/docker/testnet/Caddyfile index 2de211cd..072aaac1 100644 --- a/docker/testnet/Caddyfile +++ b/docker/testnet/Caddyfile @@ -67,6 +67,10 @@ # IP. The previous 100/10s (10 req/s) was low enough that one indexer load test # 429'd real users. handle { + # Debug: expose the resolved rate-limit key per request in the access + # log (extra.rl_key) — remove once per-IP bucketing is verified. + log_append rl_key {rl_key} + log_append cf_ip {http.request.header.CF-Connecting-IP} rate_limit { zone rpc { key {rl_key} From b9f470921eb428c7f5380cde64b47444112619c1 Mon Sep 17 00:00:00 2001 From: nol4lej Date: Thu, 16 Jul 2026 11:25:01 -0400 Subject: [PATCH 2/3] fix(caddy): split rate limiting into per-source zones, drop broken map key --- docker/testnet/Caddyfile | 49 ++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/docker/testnet/Caddyfile b/docker/testnet/Caddyfile index 072aaac1..1b18a2e3 100644 --- a/docker/testnet/Caddyfile +++ b/docker/testnet/Caddyfile @@ -15,14 +15,6 @@ format json } - # Rate-limit key: real client IP from Cloudflare; if the request did not come - # through CF the header is empty and every such client would share ONE bucket - # (mass 429s), so fall back to the direct peer IP. - map {header.CF-Connecting-IP} {rl_key} { - "" {remote_host} - default {header.CF-Connecting-IP} - } - # ── CORS — Caddy is the single source ─────────────────────────────────────── # The node also emits CORS (--rpc-cors all), so on proxied responses BOTH Caddy # and the node would add Access-Control-Allow-Origin → the browser rejects the @@ -60,20 +52,37 @@ } } - # HTTP JSON-RPC (POST): per-IP rate limit, keyed on the real client IP that - # Cloudflare forwards (the direct peer is always Cloudflare, so {remote_host} - # would limit everyone as one). 500 events / 10s = 50 req/s per IP — headroom - # for explorers/wallets/light bots that burst, while still capping an abusive - # IP. The previous 100/10s (10 req/s) was low enough that one indexer load test - # 429'd real users. + # HTTP JSON-RPC (POST): per-IP rate limit. 500 events / 10s = 50 req/s per + # IP — headroom for explorers/wallets/light bots that burst, while still + # capping an abusive IP. Two zones instead of one keyed via `map`: requests + # that came through Cloudflare are keyed on the real client IP CF forwards + # (the direct peer is always a CF edge, {remote_host} would limit everyone + # as one), and direct-origin requests fall back to their peer IP in a + # separate zone so an empty header can never collapse everyone into a + # single shared bucket. + @viaCf header CF-Connecting-IP * + handle @viaCf { + rate_limit { + zone rpc_cf { + key {http.request.header.CF-Connecting-IP} + events 500 + window 10s + } + } + reverse_proxy localhost:9944 { + # Strip the node's CORS so only Caddy's single copy survives. + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Expose-Headers + header_down -Vary + } + } + handle { - # Debug: expose the resolved rate-limit key per request in the access - # log (extra.rl_key) — remove once per-IP bucketing is verified. - log_append rl_key {rl_key} - log_append cf_ip {http.request.header.CF-Connecting-IP} rate_limit { - zone rpc { - key {rl_key} + zone rpc_direct { + key {remote_host} events 500 window 10s } From fe2fd3ce2fcc3be95cb97f4b29ac9db199e60ee8 Mon Sep 17 00:00:00 2001 From: nol4lej Date: Thu, 16 Jul 2026 11:42:16 -0400 Subject: [PATCH 3/3] refactor(caddy): dedupe proxy blocks into a snippet and fix stale comments --- docker/testnet/Caddyfile | 68 +++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/docker/testnet/Caddyfile b/docker/testnet/Caddyfile index 1b18a2e3..0209e12c 100644 --- a/docker/testnet/Caddyfile +++ b/docker/testnet/Caddyfile @@ -3,10 +3,24 @@ order rate_limit before basic_auth } +# Proxy to the node, stripping its CORS headers so only Caddy's single copy +# survives (the node also emits CORS via --rpc-cors all; duplicates make the +# browser reject the response with "multiple values '*, *'"). +(rpc_upstream) { + reverse_proxy localhost:9944 { + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Access-Control-Expose-Headers + header_down -Vary + } +} + {$RPC_DOMAIN:rpc-1.testnet.orbinum.io} { tls /etc/caddy/origin.pem /etc/caddy/origin.key - # Access log — needed to see who gets 429'd and with which client IP. + # Access log — who gets 429'd, with which client IP, and whether a 429 came + # from Caddy (Retry-After present) or the node ("Too many connections" body). log { output file /data/access.log { roll_size 50mb @@ -16,13 +30,9 @@ } # ── CORS — Caddy is the single source ─────────────────────────────────────── - # The node also emits CORS (--rpc-cors all), so on proxied responses BOTH Caddy - # and the node would add Access-Control-Allow-Origin → the browser rejects the - # duplicate ("multiple values '*, *'"). Each reverse_proxy below strips the - # node's CORS headers (header_down -...) so only Caddy's remain. Setting them - # at the Caddy layer (not just the proxy) also covers Caddy-generated responses - # — the rate-limit 429 — which never reach the node; a 429 without CORS is what - # the browser turned into an opaque "CORS error" and broke the explorer. + # Set at the Caddy layer (not just the proxy) so Caddy-generated responses — + # the rate-limit 429 — carry CORS too; a 429 without CORS is what the browser + # turned into an opaque "CORS error" and broke the explorer. header { Access-Control-Allow-Origin "*" Access-Control-Allow-Methods "GET, POST, OPTIONS" @@ -39,27 +49,19 @@ # request. The limiter counts requests, so without this split each reconnect # would burn a rate event and, under load, clients could not re-establish their # subscription (the WSS-failed errors). Route upgrades straight to the node, - # unthrottled; total connections stay capped by --rpc-max-connections (1000). + # unthrottled; total connections stay capped by --rpc-max-connections (5000). @websocket header Connection *Upgrade* handle @websocket { - reverse_proxy localhost:9944 { - # Strip the node's CORS so only Caddy's single copy survives. - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Access-Control-Expose-Headers - header_down -Vary - } + import rpc_upstream } - # HTTP JSON-RPC (POST): per-IP rate limit. 500 events / 10s = 50 req/s per + # HTTP JSON-RPC (POST): per-IP rate limit, 500 events / 10s = 50 req/s per # IP — headroom for explorers/wallets/light bots that burst, while still - # capping an abusive IP. Two zones instead of one keyed via `map`: requests - # that came through Cloudflare are keyed on the real client IP CF forwards - # (the direct peer is always a CF edge, {remote_host} would limit everyone - # as one), and direct-origin requests fall back to their peer IP in a - # separate zone so an empty header can never collapse everyone into a - # single shared bucket. + # capping an abusive IP. Two zones by traffic source: Cloudflare-proxied + # requests are keyed on the real client IP CF forwards (the direct peer is + # always a CF edge, {remote_host} would limit everyone as one bucket), and + # direct-origin requests are keyed on their peer IP in a separate zone — + # a missing header can never collapse everyone into one shared bucket. @viaCf header CF-Connecting-IP * handle @viaCf { rate_limit { @@ -69,14 +71,7 @@ window 10s } } - reverse_proxy localhost:9944 { - # Strip the node's CORS so only Caddy's single copy survives. - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Access-Control-Expose-Headers - header_down -Vary - } + import rpc_upstream } handle { @@ -87,13 +82,6 @@ window 10s } } - reverse_proxy localhost:9944 { - # Strip the node's CORS so only Caddy's single copy survives. - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Access-Control-Expose-Headers - header_down -Vary - } + import rpc_upstream } }