From 49baead3a66033beaf5cadef25a0ab6985466c75 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Thu, 30 Jul 2026 12:01:54 +0900 Subject: [PATCH] map gRPC codes.Cancled to 499, not 500 in querier Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + pkg/querier/error_translate_queryable.go | 9 ++++++++- pkg/querier/error_translate_queryable_test.go | 12 ++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b519356ee..f5fa89a46a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ * [BUGFIX] Querier/Query Frontend: Fix DNS watcher dropping all query-frontend/scheduler worker connections on a transient DNS lookup failure. #7698 * [BUGFIX] Ring: Fix DynamoDB KV CAS not retrying on transactional conditional check failures. `TransactWriteItems` reports condition failures as `TransactionCanceledException` with a `ConditionalCheckFailed` cancellation reason, which was not recognized as retryable, so any concurrent ring update conflict (e.g. many ingesters joining during a rolling update) failed immediately instead of re-reading and retrying. `TransactionConflict` cancellation reasons are also treated as retryable. #7706 * [BUGFIX] Distributor: Return HTTP 499 (Client Closed Request) instead of 500 when a remote-write or OTLP push is canceled by the client, so client-side cancellations are no longer counted as server-side errors. #7717 +* [BUGFIX] Querier: Fix gRPC `codes.Canceled` errors being mapped to HTTP 500 instead of 499 when a client cancels a query. #7738 ## 1.21.1 2026-06-04 diff --git a/pkg/querier/error_translate_queryable.go b/pkg/querier/error_translate_queryable.go index 7e0418d463c..9f4eb91eacc 100644 --- a/pkg/querier/error_translate_queryable.go +++ b/pkg/querier/error_translate_queryable.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" + "google.golang.org/grpc/codes" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -46,7 +47,7 @@ func TranslateToPromqlAPIError(err error) error { return err default: if errors.Is(err, context.Canceled) { - return err // 422 + return err // 499 } if search.IsResourceExhausted(err) { @@ -63,6 +64,12 @@ func TranslateToPromqlAPIError(err error) error { if ok { code := s.Code() + // gRPC context cancellation (codes.Canceled) should be treated as a canceled query, + // not as a storage error (500). + if code == codes.Canceled { + return promql.ErrQueryCanceled("grpc context canceled") + } + // Treat these as HTTP status codes, even though they are supposed to be grpc codes. if code >= 400 && code < 500 { // Return directly, will be mapped to 422 diff --git a/pkg/querier/error_translate_queryable_test.go b/pkg/querier/error_translate_queryable_test.go index ef9f04c70c2..2554e1cfe86 100644 --- a/pkg/querier/error_translate_queryable_test.go +++ b/pkg/querier/error_translate_queryable_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/user" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util/limiter" @@ -126,6 +128,16 @@ func TestApiStatusCodes(t *testing.T) { expectedString: limiter.ErrResourceLimitReachedStr, expectedCode: 500, }, + { + err: status.Error(codes.Canceled, "context canceled"), + expectedString: "query was canceled", + expectedCode: 499, + }, + { + err: errors.Wrap(status.Error(codes.Canceled, "context canceled"), "wrapped grpc error"), + expectedString: "query was canceled", + expectedCode: 499, + }, } { for k, q := range map[string]storage.SampleAndChunkQueryable{ "error from queryable": errorTestQueryable{err: tc.err},