Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@
* [BUGFIX] Querier/Query Frontend: Fix DNS watcher dropping all query-frontend/scheduler worker connections on a transient DNS lookup failure. #7698
* [BUGFIX] Ring: Fix DynamoDB KV CAS not retrying on transactional conditional check failures. `TransactWriteItems` reports condition failures as `TransactionCanceledException` with a `ConditionalCheckFailed` cancellation reason, which was not recognized as retryable, so any concurrent ring update conflict (e.g. many ingesters joining during a rolling update) failed immediately instead of re-reading and retrying. `TransactionConflict` cancellation reasons are also treated as retryable. #7706
* [BUGFIX] Distributor: Return HTTP 499 (Client Closed Request) instead of 500 when a remote-write or OTLP push is canceled by the client, so client-side cancellations are no longer counted as server-side errors. #7717
* [BUGFIX] Querier: Fix gRPC `codes.Canceled` errors being mapped to HTTP 500 instead of 499 when a client cancels a query. #7738

## 1.21.1 2026-06-04

Expand Down
9 changes: 8 additions & 1 deletion pkg/querier/error_translate_queryable.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/annotations"
"google.golang.org/grpc/codes"

"github.com/cortexproject/cortex/pkg/util/validation"
)
Expand Down Expand Up @@ -46,7 +47,7 @@ func TranslateToPromqlAPIError(err error) error {
return err
default:
if errors.Is(err, context.Canceled) {
return err // 422
return err // 499
}

if search.IsResourceExhausted(err) {
Expand All @@ -63,6 +64,12 @@ func TranslateToPromqlAPIError(err error) error {
if ok {
code := s.Code()

// gRPC context cancellation (codes.Canceled) should be treated as a canceled query,
// not as a storage error (500).
if code == codes.Canceled {
return promql.ErrQueryCanceled("grpc context canceled")
}

// Treat these as HTTP status codes, even though they are supposed to be grpc codes.
if code >= 400 && code < 500 {
// Return directly, will be mapped to 422
Expand Down
12 changes: 12 additions & 0 deletions pkg/querier/error_translate_queryable_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"github.com/stretchr/testify/require"
"github.com/weaveworks/common/httpgrpc"
"github.com/weaveworks/common/user"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/cortexproject/cortex/pkg/storegateway"
"github.com/cortexproject/cortex/pkg/util/limiter"
Expand Down Expand Up @@ -126,6 +128,16 @@ func TestApiStatusCodes(t *testing.T) {
expectedString: limiter.ErrResourceLimitReachedStr,
expectedCode: 500,
},
{
err: status.Error(codes.Canceled, "context canceled"),
expectedString: "query was canceled",
expectedCode: 499,
},
{
err: errors.Wrap(status.Error(codes.Canceled, "context canceled"), "wrapped grpc error"),
expectedString: "query was canceled",
expectedCode: 499,
},
} {
for k, q := range map[string]storage.SampleAndChunkQueryable{
"error from queryable": errorTestQueryable{err: tc.err},
Expand Down
Loading