diff --git a/architecture/evm/eth_getBlockByNumber.go b/architecture/evm/eth_getBlockByNumber.go index 451a979fc..e823e99d2 100644 --- a/architecture/evm/eth_getBlockByNumber.go +++ b/architecture/evm/eth_getBlockByNumber.go @@ -231,19 +231,37 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common // local pollers lag inside their debounce window, force-poll // the leader once before deciding. Fall back to excluding the // stale responder when no local poller has caught up yet. - useUpstream := "" - if leader := network.EvmLeaderUpstream(ctx); leader != nil { - if eu, ok := leader.(common.EvmUpstream); ok { - if sp := eu.EvmStatePoller(); sp != nil && !sp.IsObjectNull() { - if sp.LatestBlock() < highestBlockNumber { - _, _ = sp.PollLatestBlockNumberNow(ctx) - } - if sp.LatestBlock() >= highestBlockNumber { - useUpstream = leader.Id() - } - } + var leaderId string + var leaderLatest int64 + resolveLeaderPin := func() string { + leader := network.EvmLeaderUpstream(ctx) + if leader == nil { + leaderId = "" + leaderLatest = 0 + return "" + } + leaderId = leader.Id() + eu, ok := leader.(common.EvmUpstream) + if !ok { + leaderLatest = 0 + return "" + } + sp := eu.EvmStatePoller() + if sp == nil || sp.IsObjectNull() { + leaderLatest = 0 + return "" + } + if sp.LatestBlock() < highestBlockNumber { + _, _ = sp.PollLatestBlockNumberNow(ctx) } + leaderLatest = sp.LatestBlock() + if leaderLatest >= highestBlockNumber { + return leader.Id() + } + return "" } + useUpstream := resolveLeaderPin() + firstPinnedToLeader := useUpstream != "" if useUpstream == "" && respBlockNumber > 0 { useUpstream = fmt.Sprintf("!%s", nr.UpstreamId()) } @@ -251,29 +269,75 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common // Do not use pickHighestBlock against the stale "latest" response — // that helper fail-opens to stale when the tip re-fetch misses, which // is exactly the MultiNode FOOS / EnforceRepeatableRead trigger. - nnr, ferr := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, useUpstream) + // SkipFallbackEscape: empty tip races must refuse-stale rather than + // escape to pay-per-call tier:fallback upstreams (Infura etc.). + nnr, ferr := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, useUpstream, true) if meetsTipFloor(ctx, nnr, highestBlockNumber) { if nr != nil { nr.Release() } return nnr, nil } + pin1Upstream := "" + pin1Empty := true if nnr != nil { + pin1Upstream = nnr.UpstreamId() + pin1Empty = nnr.IsResultEmptyish() nnr.Release() } - // Pinned / excluded re-fetch missed the tip (sibling fullnode - // lag, WS JSON-RPC miss, etc.). Retry with no UseUpstream pin - // so every upstream (including fallbacks via escape) can serve - // the concrete TipHW block. - nnr2, ferr2 := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, "") + // First re-fetch missed the tip. If it was NOT leader-pinned (the + // leader poller had not caught up to TipHW at resolve time — e.g. + // TipHW arrived via Redis before the local WS delivery), resolve + // the leader again: it has had the first Forward's retry budget + // plus a forced poll to catch up, and the node that delivered the + // head serves it immediately (same-node WS→HTTP gap is ~0ms). + // If the first re-fetch WAS leader-pinned and still missed, the + // leader genuinely cannot serve — sweep the remaining primaries + // unpinned instead (fallback escape stays suppressed either way). + pin2 := "" + if !firstPinnedToLeader { + pin2 = resolveLeaderPin() + } + staleUpstream := "" + if nr != nil { + staleUpstream = nr.UpstreamId() + } + logger.Warn(). + Int64("tipHW", highestBlockNumber). + Int64("staleBlockNumber", respBlockNumber). + Str("staleUpstream", staleUpstream). + Str("leaderId", leaderId). + Int64("leaderLatest", leaderLatest). + Bool("firstPinnedToLeader", firstPinnedToLeader). + Str("pin1", useUpstream). + Str("pin1Upstream", pin1Upstream). + Bool("pin1Empty", pin1Empty). + Err(ferr). + Str("pin2", pin2). + Msg("tip re-fetch miss after first attempt") + + nnr2, ferr2 := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, pin2, true) if meetsTipFloor(ctx, nnr2, highestBlockNumber) { + servedBy := "" + if nnr2 != nil { + servedBy = nnr2.UpstreamId() + } + logger.Warn(). + Int64("tipHW", highestBlockNumber). + Str("leaderId", leaderId). + Bool("firstPinnedToLeader", firstPinnedToLeader). + Str("pin2", pin2). + Str("servedBy", servedBy). + Msg("tip re-fetch recovered on second attempt") if nr != nil { nr.Release() } return nnr2, nil } + pin2Upstream := "" if nnr2 != nil { + pin2Upstream = nnr2.UpstreamId() nnr2.Release() } @@ -281,6 +345,13 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common logger.Warn(). Int64("highestBlockNumber", highestBlockNumber). Int64("staleBlockNumber", respBlockNumber). + Str("staleUpstream", staleUpstream). + Str("leaderId", leaderId). + Int64("leaderLatest", leaderLatest). + Bool("firstPinnedToLeader", firstPinnedToLeader). + Str("pin1", useUpstream). + Str("pin2", pin2). + Str("pin2Upstream", pin2Upstream). Err(ferr2). Msg("tip re-fetch could not reach TipHW; refusing stale latest") if nr != nil { @@ -330,7 +401,9 @@ func enforceHighestBlock(ctx context.Context, network common.Network, nq *common if respBlockNumber > 0 { useUpstream = fmt.Sprintf("!%s", nr.UpstreamId()) } - nnr, err := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, useUpstream) + // Finalized re-fetch keeps fallback escape available for HA — + // SkipFallbackEscape is tip/latest-only (paid tip-race burn). + nnr, err := forwardGetBlockByNumber(ctx, network, nq, highestBlockNumber, itx, useUpstream, false) return pickHighestBlock(ctx, nnr, nr, err) default: return nr, re @@ -392,6 +465,7 @@ func forwardGetBlockByNumber( blockNumber int64, includeTx bool, useUpstream string, + skipFallbackEscape bool, ) (*common.NormalizedResponse, error) { request, err := BuildGetBlockByNumberRequest(blockNumber, includeTx) if err != nil { @@ -404,6 +478,7 @@ func forwardGetBlockByNumber( dr := original.Directives().Clone() dr.SkipCacheRead = "true" dr.UseUpstream = useUpstream + dr.SkipFallbackEscape = skipFallbackEscape newReq.SetDirectives(dr) newReq.SetNetwork(network) newReq.CopyHttpContextFrom(original) diff --git a/common/request.go b/common/request.go index f7dd6161c..4400d80c5 100644 --- a/common/request.go +++ b/common/request.go @@ -147,6 +147,12 @@ type RequestDirectives struct { // timeout still applies. Never set from HTTP headers. IsInternal bool `json:"-"` + // SkipFallbackEscape suppresses the per-request tier:fallback escape + // hatch for this request. Used by TipHW tip re-fetch so empty tip races + // on healthy primaries do not fan out to pay-per-call fallbacks + // (Infura etc.). Never set from HTTP headers. + SkipFallbackEscape bool `json:"-"` + // Instruct the normalization layer to avoid mutating JSON-RPC params for block tag interpolation. // When true, the system will still compute and cache block references (for finality/metrics), // but will NOT replace tags like "latest"/"finalized" with hex numbers in outbound requests. @@ -244,6 +250,8 @@ func (d *RequestDirectives) Clone() *RequestDirectives { SkipCacheRead: d.SkipCacheRead, UseUpstream: d.UseUpstream, ByPassMethodExclusion: d.ByPassMethodExclusion, + IsInternal: d.IsInternal, + SkipFallbackEscape: d.SkipFallbackEscape, SkipInterpolation: d.SkipInterpolation, SkipConsensus: d.SkipConsensus, EnforceHighestBlock: d.EnforceHighestBlock, diff --git a/erpc/http_server_ws_tip_leader_test.go b/erpc/http_server_ws_tip_leader_test.go index 5dda808b3..ee9308c17 100644 --- a/erpc/http_server_ws_tip_leader_test.go +++ b/erpc/http_server_ws_tip_leader_test.go @@ -11,9 +11,11 @@ import ( "github.com/bytedance/sonic" "github.com/erpc/erpc/common" "github.com/erpc/erpc/internal/policy" + "github.com/erpc/erpc/telemetry" "github.com/erpc/erpc/upstream" "github.com/erpc/erpc/util" "github.com/h2non/gock" + promUtil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -269,3 +271,374 @@ func TestHttpServer_GetBlockByNumberLatest_RefusesStaleFailOpen(t *testing.T) { require.True(t, hasErr || statusCode >= 400, "expected error when tip re-fetch cannot reach TipHW, got status=%d body=%s", statusCode, body) } + +// TipHW tip re-fetch must refuse-stale when primaries miss the concrete tip, +// without escaping to tier:fallback pay-per-call upstreams. Goes through the +// HTTP → project doForward → HandleNetworkPostForward path (not bare +// Network.Forward, which skips TipHW enforcement). +func TestHttpServer_GetBlockByNumberLatest_TipRefetchSkipsFallbackEscape(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + util.SetupMocksForEvmStatePoller() + // Persist tip mocks on primary + fallback remain pending by design. + defer util.AssertNoPendingMocks(t, 2) + + const tip = int64(0x11118889) + tipHex := "0x11118889" + staleHex := "0x11118888" + + var fallbackHits atomic.Int64 + // Primary tip re-fetch misses. + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, tipHex) + }). + Reply(200). + JSON([]byte(`{"result":null}`)) + // Fallback would serve TipHW — must not be reached via escape on tip re-fetch. + gock.New("http://rpc2.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + if strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, tipHex) { + fallbackHits.Add(1) + return true + } + return false + }). + Reply(200). + JSON([]byte(`{"result":{"number":"0x11118889","hash":"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","parentHash":"0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","timestamp":"0x6702a8f1"}}`)) + + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: common.Duration(100 * time.Second).Ptr(), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{ + { + Architecture: "evm", + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + Integrity: &common.EvmIntegrityConfig{ + EnforceHighestBlock: util.BoolPtr(true), + }, + }, + // Freeze policy ticks so a lazy method-slot eval cannot + // re-introduce the cordoned fallback mid-request. + SelectionPolicy: &common.SelectionPolicyConfig{ + EvalInterval: 0, + }, + Failover: &common.FailoverConfig{ + OnDefaultsExhausted: util.BoolPtr(true), + }, + Failsafe: []*common.FailsafeConfig{ + { + Retry: &common.RetryPolicyConfig{MaxAttempts: 2}, + }, + }, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Id: "rpc1", + Endpoint: "http://rpc1.localhost", + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(10 * time.Second), + }, + }, + { + Id: "rpc2", + Endpoint: "http://rpc2.localhost", + Type: common.UpstreamTypeEvm, + Tags: []string{common.TagTierFallback}, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(10 * time.Second), + }, + }, + }, + }, + }, + } + + sendRequest, _, _, shutdown, erpcInstance := createServerTestFixtures(cfg, t) + defer shutdown() + + prj, err := erpcInstance.GetProject("test_project") + require.NoError(t, err) + // Pin ordered list to primary only — mirrors preferTag cordoning + // fallbacks while a primary is healthy. Escape would be the only way + // to reach rpc2; SkipFallbackEscape must block that. + policy.OverrideOrderForTest(prj.policyEngine, "evm:123", "rpc1") + + time.Sleep(500 * time.Millisecond) + + nw, err := prj.GetNetwork(context.Background(), "evm:123") + require.NoError(t, err) + require.Equal(t, []string{"rpc1"}, nw.PolicyOrderedUpstreams("eth_getBlockByNumber"), + "fallback must stay cordoned so only escape could reach it") + nw.NoteObservedLatestBlock(context.Background(), tip) + require.Equal(t, tip, nw.EvmHighestLatestBlockNumber(context.Background())) + + escapeCounter := telemetry.MetricNetworkFallbackEscapeTotal.WithLabelValues( + "test_project", "evm:123", "eth_getBlockByNumber", + ) + escapeBefore := promUtil.ToFloat64(escapeCounter) + + statusCode, _, body := sendRequest(`{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getBlockByNumber", + "params": ["latest", false] + }`, nil, nil) + + var respObject map[string]interface{} + require.NoError(t, sonic.UnmarshalString(body, &respObject)) + if result, ok := respObject["result"].(map[string]interface{}); ok { + require.NotEqual(t, staleHex, result["number"], + "must not fail-open to stale tip below TipHW; status=%d body=%s", statusCode, body) + require.NotEqual(t, tipHex, result["number"], + "must not serve TipHW from fallback escape; body=%s", body) + } + _, hasErr := respObject["error"] + require.True(t, hasErr || statusCode >= 400, + "expected refuse-stale error when tip re-fetch misses without fallback escape; status=%d body=%s", + statusCode, body) + + assert.Equal(t, escapeBefore, promUtil.ToFloat64(escapeCounter), + "TipHW tip re-fetch must not fire fallback escape") + // fallbackHits may still move from background pollers probing tip hex; + // escape counter + refuse-stale response are the request-path proofs. + _ = fallbackHits +} + +// When the leader poller has not caught up to TipHW at first resolve (e.g. +// TipHW adopted from Redis before the local WS delivery), the SECOND tip +// re-fetch must re-resolve the leader — whose forced poll now returns the +// tip — and pin to it, instead of sweeping lagging siblings unpinned. +// +// Discriminator: rpc1 (the stale responder) must never receive the concrete +// tip fetch. Without the second-resolve pin, re-fetch #2 goes out unpinned +// and hits rpc1 first. +func TestHttpServer_GetBlockByNumberLatest_SecondRefetchResolvesLeaderPin(t *testing.T) { + util.ResetGock() + defer util.ResetGock() + + const tipHex = "0x3eb" // 1003 — TipHW + const rpc2Latest = "0x3ea" // 1002 — leader poller before catch-up + const rpc1Latest = "0x3e8" // 1000 — lagging stale responder + + var rpc1TipHits atomic.Int64 + var rpc2TipCalls atomic.Int64 + var leaderCaughtUp atomic.Bool + + for _, host := range []string{"rpc1.localhost", "rpc2.localhost"} { + h := host + gock.New("http://" + h). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_chainId") + }). + Reply(200). + JSON([]byte(`{"result":"0x7b"}`)) + gock.New("http://" + h). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + return strings.Contains(util.SafeReadBody(r), "eth_syncing") + }). + Reply(200). + JSON([]byte(`{"result":false}`)) + gock.New("http://" + h). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, "finalized") + }). + Reply(200). + JSON([]byte(`{"result":{"number":"0x300","timestamp":"0x6702a8e0"}}`)) + } + + // rpc1: "latest" polls and the client "latest" both see a lagging head. + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, `"latest"`) + }). + Reply(200). + JSON([]byte(`{"result":{"number":"` + rpc1Latest + `","hash":"0x1111111111111111111111111111111111111111111111111111111111111111","parentHash":"0x2222222222222222222222222222222222222222222222222222222222222222","timestamp":"0x6702a8f0"}}`)) + // rpc1: concrete tip fetch — must never happen when re-fetch #2 is pinned. + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + if r.URL.Host != "rpc1.localhost" { + return false + } + body := util.SafeReadBody(r) + if strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, `"`+tipHex+`"`) { + rpc1TipHits.Add(1) + return true + } + return false + }). + Reply(200). + JSON([]byte(`{"result":null}`)) + + // rpc2 "latest": stale until the leader "catches up" (flips after its + // first concrete-tip miss), then the forced poll returns the tip. + gock.New("http://rpc2.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, `"latest"`) && !leaderCaughtUp.Load() + }). + Reply(200). + JSON([]byte(`{"result":{"number":"` + rpc2Latest + `","hash":"0x3333333333333333333333333333333333333333333333333333333333333333","parentHash":"0x4444444444444444444444444444444444444444444444444444444444444444","timestamp":"0x6702a8f0"}}`)) + gock.New("http://rpc2.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + body := util.SafeReadBody(r) + return strings.Contains(body, "eth_getBlockByNumber") && strings.Contains(body, `"latest"`) && leaderCaughtUp.Load() + }). + Reply(200). + JSON([]byte(`{"result":{"number":"` + tipHex + `","hash":"0x5555555555555555555555555555555555555555555555555555555555555555","parentHash":"0x6666666666666666666666666666666666666666666666666666666666666666","timestamp":"0x6702a8f1"}}`)) + // rpc2 concrete tip: first call misses (and marks the node caught-up so + // the next forced poll sees the tip); subsequent calls serve the block. + gock.New("http://rpc2.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + if r.URL.Host != "rpc2.localhost" { + return false + } + body := util.SafeReadBody(r) + if !strings.Contains(body, "eth_getBlockByNumber") || !strings.Contains(body, `"`+tipHex+`"`) { + return false + } + if rpc2TipCalls.Add(1) == 1 { + // The miss itself marks the node caught-up: the next forced + // poll (second leader resolve) sees the tip. + leaderCaughtUp.Store(true) + return true + } + return false + }). + Reply(200). + JSON([]byte(`{"result":null}`)) + gock.New("http://rpc2.localhost"). + Post(""). + Persist(). + Filter(func(r *http.Request) bool { + if r.URL.Host != "rpc2.localhost" { + return false + } + body := util.SafeReadBody(r) + if !strings.Contains(body, "eth_getBlockByNumber") || !strings.Contains(body, `"`+tipHex+`"`) { + return false + } + return rpc2TipCalls.Load() >= 1 + }). + Reply(200). + JSON([]byte(`{"result":{"number":"` + tipHex + `","hash":"0x7777777777777777777777777777777777777777777777777777777777777777","parentHash":"0x8888888888888888888888888888888888888888888888888888888888888888","timestamp":"0x6702a8f1"}}`)) + + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: common.Duration(100 * time.Second).Ptr(), + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{ + { + Architecture: "evm", + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + Integrity: &common.EvmIntegrityConfig{ + EnforceHighestBlock: util.BoolPtr(true), + }, + }, + SelectionPolicy: &common.SelectionPolicyConfig{ + EvalInterval: 0, + }, + Failsafe: []*common.FailsafeConfig{ + { + Retry: &common.RetryPolicyConfig{MaxAttempts: 1}, + }, + }, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Id: "rpc1", + Endpoint: "http://rpc1.localhost", + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(10 * time.Second), + }, + }, + { + Id: "rpc2", + Endpoint: "http://rpc2.localhost", + Type: common.UpstreamTypeEvm, + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + StatePollerInterval: common.Duration(10 * time.Second), + }, + }, + }, + }, + }, + } + + sendRequest, _, _, shutdown, erpcInstance := createServerTestFixtures(cfg, t) + defer shutdown() + + prj, err := erpcInstance.GetProject("test_project") + require.NoError(t, err) + nw, err := prj.GetNetwork(context.Background(), "evm:123") + require.NoError(t, err) + + time.Sleep(500 * time.Millisecond) + + policy.OverrideAllForTest(prj.policyEngine) + policy.OverrideOrderForTest(prj.policyEngine, "evm:123", "rpc1", "rpc2") + require.Equal(t, []string{"rpc1", "rpc2"}, nw.PolicyOrderedUpstreams("eth_getBlockByNumber"), + "rpc1 must serve the stale latest so it becomes the excluded responder") + + nw.NoteObservedLatestBlock(context.Background(), 1003) + require.Equal(t, int64(1003), nw.EvmHighestLatestBlockNumber(context.Background())) + + statusCode, _, body := sendRequest(`{ + "jsonrpc": "2.0", + "id": 1, + "method": "eth_getBlockByNumber", + "params": ["latest", false] + }`, nil, nil) + + require.Equal(t, http.StatusOK, statusCode, "expected tip served via leader-pinned second re-fetch; body=%s", body) + var respObject map[string]interface{} + require.NoError(t, sonic.UnmarshalString(body, &respObject)) + result, ok := respObject["result"].(map[string]interface{}) + require.True(t, ok, "response should have a result object, got: %s", body) + assert.Equal(t, tipHex, result["number"], "must serve the TipHW block") + assert.Equal(t, int64(0), rpc1TipHits.Load(), + "second re-fetch must pin to the caught-up leader, not sweep the lagging stale responder") + assert.GreaterOrEqual(t, rpc2TipCalls.Load(), int64(2), + "leader must be re-tried via pin after its first tip miss") +} diff --git a/erpc/networks.go b/erpc/networks.go index 6ee3e0821..372bca50a 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -536,8 +536,10 @@ func (n *Network) EvmLowestFinalizedBlockNumber(ctx context.Context) int64 { // latest tip. Fallback-tier upstreams are ignored while any primary is up — // otherwise tip re-fetch pins UseUpstream to a cordoned fallback that is not // in the ordered primary list. TipHW may still advance from fallback WS -// (fan-out invariant); unconstrained tip re-fetch + emptyish escape reaches -// those fallbacks when primaries miss. +// (fan-out invariant). Tip re-fetch sets SkipFallbackEscape so empty tip +// races on healthy primaries refuse-stale instead of burning pay-per-call +// fallbacks; real HA still reaches fallbacks via selectionPolicy when +// primaries are down. func (n *Network) EvmLeaderUpstream(ctx context.Context) common.Upstream { var leader, fallbackLeader common.Upstream var leaderLastBlock, fallbackLastBlock int64 @@ -1085,8 +1087,32 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // couldn't serve; try a different one" — exactly the escape's job. // - Consensus requires strict per-upstream semantics; don't modify // the candidate set mid-execution. + // - SkipFallbackEscape (TipHW tip re-fetch) blocks escape so tip + // races on healthy primaries refuse-stale instead of fanning + // out to pay-per-call tier:fallback upstreams. + dirs := effectiveReq.Directives() + skipFallbackEscape := dirs != nil && dirs.SkipFallbackEscape + if !skipFallbackEscape { + // Tip-race miss: the requested block is at or one ahead of the + // primary leader's poller — primaries import it within ~a block + // time (sibling lag, measured 100-200ms), so escaping to + // pay-per-call fallbacks buys nothing. Let the failsafe retry + // (emptyResultDelay / blockUnavailableDelay) re-visit primaries. + // Blocks further ahead (primaries stuck), older-block data gaps, + // and block-less methods (receipts) escape as before. + if bn, ok := effectiveReq.EvmBlockNumber().(int64); ok && bn > 0 { + if leader, ok2 := n.EvmLeaderUpstream(execSpanCtx).(common.EvmUpstream); ok2 && leader != nil { + if sp := leader.EvmStatePoller(); sp != nil && !sp.IsObjectNull() { + if l := sp.LatestBlock(); l > 0 && bn >= l && bn <= l+1 { + skipFallbackEscape = true + } + } + } + } + } bestRespEmptyish := bestResp != nil && bestResp.IsResultEmptyish() if (bestResp == nil || bestRespEmptyish) && + !skipFallbackEscape && !effectiveReq.HasEscalatedToFallbacks() && lastErr != nil && n.cfg.Failover != nil && n.cfg.Failover.Enabled() && diff --git a/erpc/networks_failover_escape_test.go b/erpc/networks_failover_escape_test.go index 6aae47a6c..de6a7beab 100644 --- a/erpc/networks_failover_escape_test.go +++ b/erpc/networks_failover_escape_test.go @@ -627,15 +627,18 @@ func TestFailover_EscapeHatch(t *testing.T) { "escape hatch must fire exactly once for the non-retryable gate-skip case") }) - t.Run("EscapesOnEmptyishGetBlockByNumber", func(t *testing.T) { + t.Run("NearTipEmptyishDoesNotEscape", func(t *testing.T) { defer util.ResetGock() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // Primaries and fallbacks both report tip 1002 via poller, so the - // availability gate fails open / passes. Primaries return null for - // the concrete tip block (missing data); fallbacks return the header. - // Before the emptyish-escape fix, bestResp=null blocked escalation. + // Primaries and fallbacks both report tip 1002 via poller, and the + // request targets that tip. Primaries returning null for a block at + // (or one ahead of) the primary leader's poller is the sibling + // import race — primaries serve it within ~a block time, so the + // escape hatch must NOT burn pay-per-call fallbacks. The failsafe + // retry re-visits primaries instead. (Blocks further ahead — stuck + // primaries — still escape: see the gate-skip subtests above.) network, _, _ := setupFailoverFixture(t, ctx, failoverFixtureOpts{ primaryLatest: "0x3ea", // 1002 fallbackLatest: "0x3ea", // 1002 @@ -675,22 +678,22 @@ func TestFailover_EscapeHatch(t *testing.T) { )) req.SetNetwork(network) resp, err := network.Forward(ctx, req) - require.NoError(t, err, "null from primaries must escalate to fallbacks on the same request") - require.NotNil(t, resp) - defer resp.Release() - - jrr, err := resp.JsonRpcResponse() - require.NoError(t, err) - require.False(t, jrr.IsResultEmptyish(), "fallback must return a non-null block header") - num, err := jrr.PeekStringByPath(ctx, "number") - require.NoError(t, err) - assert.Equal(t, "0x3ea", num) + if resp != nil { + defer resp.Release() + } - assert.Contains(t, []string{"fallback-1", "fallback-2"}, resp.UpstreamId(), - "emptyish primary miss must be served by a fallback") + if err == nil { + require.NotNil(t, resp) + jrr, jerr := resp.JsonRpcResponse() + require.NoError(t, jerr) + require.True(t, jrr.IsResultEmptyish(), + "near-tip miss must stay emptyish from primaries, not be served by a fallback") + assert.NotContains(t, []string{"fallback-1", "fallback-2"}, resp.UpstreamId(), + "near-tip miss must not be served via fallback escape") + } after := promUtil.ToFloat64(counter) - assert.Equal(t, before+1, after, - "escape hatch must fire for emptyish eth_getBlockByNumber primary misses") + assert.Equal(t, before, after, + "escape hatch must NOT fire for near-tip emptyish primary misses") }) } diff --git a/internal/policy/testing.go b/internal/policy/testing.go index 47bd1a333..457c77610 100644 --- a/internal/policy/testing.go +++ b/internal/policy/testing.go @@ -19,18 +19,12 @@ import ( // the duration of the test should set `EvalInterval: 0` on the network's // SelectionPolicy. func OverrideOrderForTest(e *Engine, networkID string, ids ...string) { - e.mu.RLock() - slot, ok := e.slots[slotKey{networkID, "*", "*"}] + e.mu.Lock() + defer e.mu.Unlock() reg := e.networks[networkID] - e.mu.RUnlock() - if !ok || reg == nil { + if reg == nil { return } - // Stop the ticker for the lifetime of this override — tests that pin - // don't want background re-eval clobbering their cache mid-test, and - // thousands of test fixtures running at 1s tick each pushes the - // race-detector CI suite past its 20-minute budget. - slot.stop() ups := reg.upstreamsFn() index := make(map[string]common.Upstream, len(ups)) for _, u := range ups { @@ -49,7 +43,21 @@ func OverrideOrderForTest(e *Engine, networkID string, ids ...string) { ordered = append(ordered, u) } } - slot.cache.Store(&ordered) + // Pin every slot for this network (wildcard + method/finality-narrow). + // GetOrdered prefers a populated narrow slot over the wildcard, so + // overriding only ("*", "*") left tip-re-fetch tests selecting + // cordoned fallbacks from a stale eth_getBlockByNumber cache. + for k, slot := range e.slots { + if k.network != networkID { + continue + } + // Stop the ticker for the lifetime of this override — tests that pin + // don't want background re-eval clobbering their cache mid-test, and + // thousands of test fixtures running at 1s tick each pushes the + // race-detector CI suite past its 20-minute budget. + slot.stop() + slot.cache.Store(&ordered) + } } // OverrideAllForTest applies OverrideOrderForTest to EVERY network this