diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b519356e..f5fa89a46a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ * [BUGFIX] Querier/Query Frontend: Fix DNS watcher dropping all query-frontend/scheduler worker connections on a transient DNS lookup failure. #7698 * [BUGFIX] Ring: Fix DynamoDB KV CAS not retrying on transactional conditional check failures. `TransactWriteItems` reports condition failures as `TransactionCanceledException` with a `ConditionalCheckFailed` cancellation reason, which was not recognized as retryable, so any concurrent ring update conflict (e.g. many ingesters joining during a rolling update) failed immediately instead of re-reading and retrying. `TransactionConflict` cancellation reasons are also treated as retryable. #7706 * [BUGFIX] Distributor: Return HTTP 499 (Client Closed Request) instead of 500 when a remote-write or OTLP push is canceled by the client, so client-side cancellations are no longer counted as server-side errors. #7717 +* [BUGFIX] Querier: Fix gRPC `codes.Canceled` errors being mapped to HTTP 500 instead of 499 when a client cancels a query. #7738 ## 1.21.1 2026-06-04 diff --git a/pkg/querier/error_translate_queryable.go b/pkg/querier/error_translate_queryable.go index 7e0418d463..9f4eb91eac 100644 --- a/pkg/querier/error_translate_queryable.go +++ b/pkg/querier/error_translate_queryable.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/annotations" + "google.golang.org/grpc/codes" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -46,7 +47,7 @@ func TranslateToPromqlAPIError(err error) error { return err default: if errors.Is(err, context.Canceled) { - return err // 422 + return err // 499 } if search.IsResourceExhausted(err) { @@ -63,6 +64,12 @@ func TranslateToPromqlAPIError(err error) error { if ok { code := s.Code() + // gRPC context cancellation (codes.Canceled) should be treated as a canceled query, + // not as a storage error (500). + if code == codes.Canceled { + return promql.ErrQueryCanceled("grpc context canceled") + } + // Treat these as HTTP status codes, even though they are supposed to be grpc codes. if code >= 400 && code < 500 { // Return directly, will be mapped to 422 diff --git a/pkg/querier/error_translate_queryable_test.go b/pkg/querier/error_translate_queryable_test.go index ef9f04c70c..2554e1cfe8 100644 --- a/pkg/querier/error_translate_queryable_test.go +++ b/pkg/querier/error_translate_queryable_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/user" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/util/limiter" @@ -126,6 +128,16 @@ func TestApiStatusCodes(t *testing.T) { expectedString: limiter.ErrResourceLimitReachedStr, expectedCode: 500, }, + { + err: status.Error(codes.Canceled, "context canceled"), + expectedString: "query was canceled", + expectedCode: 499, + }, + { + err: errors.Wrap(status.Error(codes.Canceled, "context canceled"), "wrapped grpc error"), + expectedString: "query was canceled", + expectedCode: 499, + }, } { for k, q := range map[string]storage.SampleAndChunkQueryable{ "error from queryable": errorTestQueryable{err: tc.err},