diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ecad82c3..61b29de6a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ * [BUGFIX] Store Gateway: Fix misleading "no index cache backend addresses" validation error being reported for chunks-cache, metadata-cache, and parquet caches when their memcached or redis backend is configured without addresses. The message is now the cache-type-agnostic "no cache backend addresses". #7675 * [BUGFIX] Querier/Query Frontend: Fix DNS watcher dropping all query-frontend/scheduler worker connections on a transient DNS lookup failure. #7698 * [BUGFIX] Ring: Fix DynamoDB KV CAS not retrying on transactional conditional check failures. `TransactWriteItems` reports condition failures as `TransactionCanceledException` with a `ConditionalCheckFailed` cancellation reason, which was not recognized as retryable, so any concurrent ring update conflict (e.g. many ingesters joining during a rolling update) failed immediately instead of re-reading and retrying. `TransactionConflict` cancellation reasons are also treated as retryable. #7706 +* [BUGFIX] Ingester: Size the active queried series worker pool from `GOMAXPROCS` instead of `runtime.NumCPU()`, so it respects the container CPU quota (set via automaxprocs) rather than the host core count. #7671 ## 1.21.1 2026-06-04 diff --git a/pkg/ingester/active_queried_series.go b/pkg/ingester/active_queried_series.go index 96134a0515..32685d0f91 100644 --- a/pkg/ingester/active_queried_series.go +++ b/pkg/ingester/active_queried_series.go @@ -385,8 +385,10 @@ type ActiveQueriedSeriesService struct { // NewActiveQueriedSeriesService creates a new ActiveQueriedSeriesService service. func NewActiveQueriedSeriesService(logger log.Logger, registerer prometheus.Registerer) *ActiveQueriedSeriesService { - // Cap at 4 workers to avoid excessive goroutines - numWorkers := max(min(runtime.NumCPU()/2, 4), 1) + // Cap at 4 workers to avoid excessive goroutines. Use GOMAXPROCS (which + // automaxprocs sets from the CPU cgroup quota) rather than NumCPU, so the + // pool is sized to the container's CPU budget instead of the host cores. + numWorkers := max(min(runtime.GOMAXPROCS(0)/2, 4), 1) m := &ActiveQueriedSeriesService{ updateChan: make(chan activeQueriedSeriesUpdate, 10000), // Buffered channel to avoid blocking diff --git a/pkg/ingester/active_queried_series_test.go b/pkg/ingester/active_queried_series_test.go index 9fdbd27c4f..c036039186 100644 --- a/pkg/ingester/active_queried_series_test.go +++ b/pkg/ingester/active_queried_series_test.go @@ -3,6 +3,7 @@ package ingester import ( "context" "fmt" + "runtime" "sync" "testing" "time" @@ -546,3 +547,30 @@ func TestActiveQueriedSeriesService_NoSendOnClosedChannelOnShutdown(t *testing.T t.Fatalf("producer goroutine panicked during shutdown: %v", panicMsg.Load()) } } + +func TestActiveQueriedSeriesService_NumWorkersFollowsGOMAXPROCS(t *testing.T) { + // The worker pool must be sized from GOMAXPROCS (which automaxprocs derives + // from the CPU cgroup quota) and not from runtime.NumCPU (host cores, which + // ignores cgroup limits). Save and restore GOMAXPROCS around the test. + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + + for _, tc := range []struct { + gomaxprocs int + expectedWorkers int + }{ + // GOMAXPROCS(1)/2 == 0, floored to the minimum of 1 worker. Against the + // buggy runtime.NumCPU() this yields min(NumCPU/2, 4) on a multi-core + // host and fails. + {gomaxprocs: 1, expectedWorkers: 1}, + {gomaxprocs: 2, expectedWorkers: 1}, + {gomaxprocs: 4, expectedWorkers: 2}, + // Capped at 4 workers regardless of how many CPUs are available. + {gomaxprocs: 16, expectedWorkers: 4}, + } { + t.Run(fmt.Sprintf("gomaxprocs=%d", tc.gomaxprocs), func(t *testing.T) { + runtime.GOMAXPROCS(tc.gomaxprocs) + svc := NewActiveQueriedSeriesService(log.NewNopLogger(), nil) + assert.Equal(t, tc.expectedWorkers, svc.numWorkers) + }) + } +}