From 2ca9ff8c72998e0f9345075d0b8de443387c58ac Mon Sep 17 00:00:00 2001 From: piket Date: Thu, 25 Jun 2026 13:15:41 -0700 Subject: [PATCH 01/11] fix: preserve millisecond precision for UNIX_TIMESTAMP sort keys (EAPC-22316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SortedFeatureView rows whose UNIX_TIMESTAMP sort keys differed by less than one second were being assigned the same sort key score in Redis/Valkey (wrong ordering) and the same Cassandra clustering key (row overwrites). Root cause: _python_datetime_to_int_timestamp() truncated all UNIX_TIMESTAMP values to integer seconds before storing in unix_timestamp_val. The * 1000 multiplications in zset_score() attempted ms precision but had no effect. Fix: - Add _python_datetime_to_int_ms_timestamp() for millisecond conversion. - In _convert_arrow_fv_to_proto(), detect UNIX_TIMESTAMP sort key columns and use the ms function for those columns only; all other UNIX_TIMESTAMP columns remain at second precision (backward compatible). - Remove the * 1000 from zset_score() in Redis, Valkey, and Cassandra — unix_timestamp_val is now already ms for sort key columns. - Add a threshold discriminator (val > 1e11) in feast_value_type_to_python_type() so sort key ms values are correctly read back as datetimes without affecting the seconds interpretation for regular feature columns. Co-Authored-By: Claude Sonnet 4.6 --- .../cassandra_online_store.py | 12 ++---- .../feast/infra/online_stores/eg_valkey.py | 4 +- sdk/python/feast/infra/online_stores/redis.py | 4 +- sdk/python/feast/type_map.py | 38 +++++++++++++++++-- sdk/python/feast/utils.py | 29 +++++++++++--- .../unit/infra/online_store/test_redis.py | 2 +- .../unit/infra/online_store/test_valkey.py | 2 +- 7 files changed, 66 insertions(+), 25 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py index 06b0f3ac4d7..e3aee93675a 100644 --- a/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py @@ -538,18 +538,14 @@ def on_failure(exc, concurrent_queue): if feature_name in sort_key_names: feast_value_type = valProto.WhichOneof("val") if feast_value_type == "unix_timestamp_val": - feature_value = ( - valProto.unix_timestamp_val * 1000 - ) # Convert to milliseconds + feature_value = valProto.unix_timestamp_val # already milliseconds elif feast_value_type is None: feature_value = None elif feast_value_type in feast_array_types: if feast_value_type == "unix_timestamp_list_val": - # Convert list of timestamps to milliseconds - feature_value = [ - ts * 1000 - for ts in valProto.unix_timestamp_list_val.val # type:ignore - ] + feature_value = list( + valProto.unix_timestamp_list_val.val # type:ignore + ) # already milliseconds else: feature_value = getattr( valProto, str(feast_value_type) diff --git a/sdk/python/feast/infra/online_stores/eg_valkey.py b/sdk/python/feast/infra/online_stores/eg_valkey.py index 93f147f51b8..01f5bdf51f4 100644 --- a/sdk/python/feast/infra/online_stores/eg_valkey.py +++ b/sdk/python/feast/infra/online_stores/eg_valkey.py @@ -774,9 +774,7 @@ def zset_score(sort_key_value: ValueProto): """ feast_value_type = sort_key_value.WhichOneof("val") if feast_value_type == "unix_timestamp_val": - feature_value = ( - sort_key_value.unix_timestamp_val * 1000 - ) # Convert to milliseconds + feature_value = sort_key_value.unix_timestamp_val # already milliseconds else: feature_value = getattr(sort_key_value, str(feast_value_type)) return feature_value diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 8d3d3185ca5..06e90c25586 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -500,9 +500,7 @@ def zset_score(sort_key_value: ValueProto): """ feast_value_type = sort_key_value.WhichOneof("val") if feast_value_type == "unix_timestamp_val": - feature_value = ( - sort_key_value.unix_timestamp_val * 1000 - ) # Convert to milliseconds + feature_value = sort_key_value.unix_timestamp_val # already milliseconds else: feature_value = getattr(sort_key_value, str(feast_value_type)) return feature_value diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index ebf6f0eae19..209bb200d54 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -78,11 +78,16 @@ def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: if hasattr(val, "val"): val = list(val.val) - # Convert UNIX_TIMESTAMP values to `datetime` + # Convert UNIX_TIMESTAMP values to `datetime`. + # unix_timestamp_val stores seconds for regular features and milliseconds for sort key + # columns. Values > 1e11 are unambiguously ms (current-era seconds ~1.7e9, ms ~1.7e12; + # threshold is safe until year ~5138 in seconds). if val_attr == "unix_timestamp_list_val": val = [ ( - datetime.fromtimestamp(v, tz=timezone.utc) + datetime.fromtimestamp( + v / 1000.0 if v > 1e11 else float(v), tz=timezone.utc + ) if v != NULL_TIMESTAMP_INT_VALUE else None ) @@ -90,7 +95,9 @@ def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: ] elif val_attr == "unix_timestamp_val": val = ( - datetime.fromtimestamp(val, tz=timezone.utc) + datetime.fromtimestamp( + val / 1000.0 if val > 1e11 else float(val), tz=timezone.utc + ) if val != NULL_TIMESTAMP_INT_VALUE else None ) @@ -358,6 +365,31 @@ def _python_datetime_to_int_timestamp( return int_timestamps +def _python_datetime_to_int_ms_timestamp( + values: Sequence[Any], +) -> Sequence[Union[int, np.int_]]: + """Convert datetime values to milliseconds since epoch (used for sort key columns).""" + # Fast path for Numpy array. + if isinstance(values, np.ndarray) and isinstance(values.dtype, np.datetime64): + if values.ndim != 1: + raise ValueError("Only 1 dimensional arrays are supported.") + return cast(Sequence[np.int_], values.astype("datetime64[ms]").astype(np.int_)) + + int_timestamps = [] + for value in values: + if isinstance(value, datetime): + int_timestamps.append(int(round(value.timestamp() * 1000))) + elif isinstance(value, Timestamp): + int_timestamps.append(int(value.ToMilliseconds())) + elif isinstance(value, np.datetime64): + int_timestamps.append(value.astype("datetime64[ms]").astype(np.int_)) # type: ignore[attr-defined] + elif isinstance(value, type(np.nan)): + int_timestamps.append(NULL_TIMESTAMP_INT_VALUE) + else: + int_timestamps.append(int(value)) + return int_timestamps + + def _python_value_to_proto_value( feast_value_type: ValueType, values: List[Any] ) -> List[ProtoValue]: diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index af55266c682..2a8ee42625b 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -42,7 +42,10 @@ from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import RepeatedValue as RepeatedValueProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto -from feast.type_map import python_values_to_proto_values +from feast.type_map import ( + _python_datetime_to_int_ms_timestamp, + python_values_to_proto_values, +) from feast.types import ComplexFeastType, PrimitiveFeastType, from_feast_to_pyarrow_type from feast.value_type import ValueType from feast.version import get_version @@ -281,13 +284,27 @@ def _convert_arrow_fv_to_proto( (field.name, field.dtype.to_value_type()) for field in feature_view.features ] + list(join_keys.items()) - proto_values_by_column = { - column: python_values_to_proto_values( - table.column(column).to_numpy(zero_copy_only=False), value_type - ) - for column, value_type in columns + # Sort key UNIX_TIMESTAMP columns use ms precision so that timestamps differing + # by < 1 second produce distinct sort key scores in the online store. + sort_key_ts_names = { + sk.name + for sk in getattr(feature_view, "sort_keys", []) + if sk.value_type == ValueType.UNIX_TIMESTAMP } + proto_values_by_column = {} + for column, value_type in columns: + raw = table.column(column).to_numpy(zero_copy_only=False) + if column in sort_key_ts_names: + ms_vals = _python_datetime_to_int_ms_timestamp(raw) + proto_values_by_column[column] = [ + ValueProto(unix_timestamp_val=int(ts)) for ts in ms_vals + ] + else: + proto_values_by_column[column] = python_values_to_proto_values( + raw, value_type + ) + entity_keys = [ EntityKeyProto( join_keys=join_keys, diff --git a/sdk/python/tests/unit/infra/online_store/test_redis.py b/sdk/python/tests/unit/infra/online_store/test_redis.py index e2aad12bd7e..4bacd876f44 100644 --- a/sdk/python/tests/unit/infra/online_store/test_redis.py +++ b/sdk/python/tests/unit/infra/online_store/test_redis.py @@ -508,7 +508,7 @@ def _make_rows(n=10): { "trip_id": ValueProto(int32_val=i), "rating": ValueProto(float_val=i + 0.5), - "event_timestamp": ValueProto(unix_timestamp_val=base_ts + i * 60), + "event_timestamp": ValueProto(unix_timestamp_val=(base_ts + i * 60) * 1000), }, datetime.fromtimestamp(base_ts + i * 60, tz=timezone.utc), None, diff --git a/sdk/python/tests/unit/infra/online_store/test_valkey.py b/sdk/python/tests/unit/infra/online_store/test_valkey.py index ed911bcb92a..5a78cd28dc4 100644 --- a/sdk/python/tests/unit/infra/online_store/test_valkey.py +++ b/sdk/python/tests/unit/infra/online_store/test_valkey.py @@ -402,7 +402,7 @@ def _make_rows(n=10): { "trip_id": ValueProto(int32_val=i), "rating": ValueProto(float_val=i + 0.5), - "event_timestamp": ValueProto(unix_timestamp_val=base_ts + i * 60), + "event_timestamp": ValueProto(unix_timestamp_val=(base_ts + i * 60) * 1000), }, datetime.fromtimestamp(base_ts + i * 60, tz=timezone.utc), None, From 2e760edc824894666a8fa37da3c67c742880fbe8 Mon Sep 17 00:00:00 2001 From: piket Date: Thu, 25 Jun 2026 13:33:35 -0700 Subject: [PATCH 02/11] style: apply ruff formatting Co-Authored-By: Claude Sonnet 4.6 --- .../cassandra_online_store/cassandra_online_store.py | 4 +++- sdk/python/tests/unit/infra/online_store/test_redis.py | 4 +++- sdk/python/tests/unit/infra/online_store/test_valkey.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py b/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py index e3aee93675a..631adc11abc 100644 --- a/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py +++ b/sdk/python/feast/infra/online_stores/cassandra_online_store/cassandra_online_store.py @@ -538,7 +538,9 @@ def on_failure(exc, concurrent_queue): if feature_name in sort_key_names: feast_value_type = valProto.WhichOneof("val") if feast_value_type == "unix_timestamp_val": - feature_value = valProto.unix_timestamp_val # already milliseconds + feature_value = ( + valProto.unix_timestamp_val + ) # already milliseconds elif feast_value_type is None: feature_value = None elif feast_value_type in feast_array_types: diff --git a/sdk/python/tests/unit/infra/online_store/test_redis.py b/sdk/python/tests/unit/infra/online_store/test_redis.py index 4bacd876f44..372cf97923c 100644 --- a/sdk/python/tests/unit/infra/online_store/test_redis.py +++ b/sdk/python/tests/unit/infra/online_store/test_redis.py @@ -508,7 +508,9 @@ def _make_rows(n=10): { "trip_id": ValueProto(int32_val=i), "rating": ValueProto(float_val=i + 0.5), - "event_timestamp": ValueProto(unix_timestamp_val=(base_ts + i * 60) * 1000), + "event_timestamp": ValueProto( + unix_timestamp_val=(base_ts + i * 60) * 1000 + ), }, datetime.fromtimestamp(base_ts + i * 60, tz=timezone.utc), None, diff --git a/sdk/python/tests/unit/infra/online_store/test_valkey.py b/sdk/python/tests/unit/infra/online_store/test_valkey.py index 5a78cd28dc4..2af1e8060a1 100644 --- a/sdk/python/tests/unit/infra/online_store/test_valkey.py +++ b/sdk/python/tests/unit/infra/online_store/test_valkey.py @@ -402,7 +402,9 @@ def _make_rows(n=10): { "trip_id": ValueProto(int32_val=i), "rating": ValueProto(float_val=i + 0.5), - "event_timestamp": ValueProto(unix_timestamp_val=(base_ts + i * 60) * 1000), + "event_timestamp": ValueProto( + unix_timestamp_val=(base_ts + i * 60) * 1000 + ), }, datetime.fromtimestamp(base_ts + i * 60, tz=timezone.utc), None, From bd9cf1ac14f82516cd7445a20a28420ca8fe15ea Mon Sep 17 00:00:00 2001 From: piket Date: Tue, 30 Jun 2026 15:36:20 -0700 Subject: [PATCH 03/11] fix: update Go serving layer for ms-precision UNIX_TIMESTAMP sort keys (EAPC-22316) After the Python SDK was updated to store sort key unix_timestamp_val as milliseconds, the Go serving layer needed matching changes to avoid treating ms values as epoch-seconds (which would produce year ~57000 timestamps). Production changes: - typeconversion.go: add unixTsToTime() helper with a >1e11 threshold to distinguish ms from s; use it for all UnixTimestampVal conversions - redis_read_range_utils.go: use UnixMilli() for ZRANGEBYSCORE bounds to match the ms ZADD scores written by the Python materializer Test/infra changes: - go_integration_test_utils.go: support FEAST_BIN env var for local venv feast; return UnixMilli() from ReadParquetDynamically for timestamp columns - valkey_integration_test.go: fix ms/s conversions in getValueType and EventTimestamps assertions - scylladb_integration_test.go: update equals filter to ms precision (1744769171919) - http_integration_test.go: update equals filter to ms precision (1744769171919) - redisonlinestore_test.go, redis_read_range_utils_test.go: update expected ZADD scores and range bounds to ms scale Co-Authored-By: Claude Sonnet 4.6 --- .../scylladb/http/http_integration_test.go | 2 +- .../scylladb/scylladb_integration_test.go | 2 +- .../valkey/valkey_integration_test.go | 28 ++++++++++++++----- .../onlinestore/redisonlinestore_test.go | 24 ++++++++-------- .../feast/utils/redis_read_range_utils.go | 4 +-- .../utils/redis_read_range_utils_test.go | 2 +- go/internal/test/go_integration_test_utils.go | 11 ++++++-- go/types/typeconversion.go | 18 +++++++++--- 8 files changed, 60 insertions(+), 31 deletions(-) diff --git a/go/internal/feast/integration_tests/scylladb/http/http_integration_test.go b/go/internal/feast/integration_tests/scylladb/http/http_integration_test.go index 4bb1d3e3037..4dd0accfad9 100644 --- a/go/internal/feast/integration_tests/scylladb/http/http_integration_test.go +++ b/go/internal/feast/integration_tests/scylladb/http/http_integration_test.go @@ -215,7 +215,7 @@ func TestGetOnlineFeaturesRange_Http_withOnlyEqualsFilter(t *testing.T) { "sort_key_filters": [ { "sort_key_name": "event_timestamp", - "equals": 1744769171 + "equals": 1744769171919 } ], "limit": 10 diff --git a/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go b/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go index 11ea2e5bd07..3a3b3536241 100644 --- a/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go +++ b/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go @@ -129,7 +129,7 @@ func TestGetOnlineFeaturesRange_withOnlyEqualsFilter(t *testing.T) { { SortKeyName: "event_timestamp", Query: &serving.SortKeyFilter_Equals{ - Equals: &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: 1744769171}}, + Equals: &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: 1744769171919}}, }, }, }, diff --git a/go/internal/feast/integration_tests/valkey/valkey_integration_test.go b/go/internal/feast/integration_tests/valkey/valkey_integration_test.go index fed9d16b8ed..834c0838b59 100644 --- a/go/internal/feast/integration_tests/valkey/valkey_integration_test.go +++ b/go/internal/feast/integration_tests/valkey/valkey_integration_test.go @@ -153,7 +153,13 @@ func getValueType(value interface{}, featureName string) *types.Value { case int64: // Check if featureName contains "timestamp" if strings.Contains(featureName, "timestamp") { - return &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: value.(int64)}} + val := value.(int64) + if featureName != "event_timestamp" { + // Regular timestamp features are stored as seconds in Redis; + // ReadParquetDynamically returns ms, so divide by 1000. + val = val / 1000 + } + return &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: val}} } else { if value == nil { return &types.Value{} @@ -186,10 +192,13 @@ func getValueType(value interface{}, featureName string) *types.Value { if strings.Contains(featureName, "timestamp") { for _, v := range arrayInterface { - if v.(int64) == 0 { + elem := v.(int64) + if elem == 0 { arrayValue = append(arrayValue, -9223372036854775808) } else { - arrayValue = append(arrayValue, v.(int64)) + // Array timestamps stored as seconds in Redis; + // ReadParquetDynamically returns ms, divide by 1000. + arrayValue = append(arrayValue, elem/1000) } } return &types.Value{Val: &types.Value_UnixTimestampListVal{&types.Int64List{Val: arrayValue}}} @@ -323,7 +332,9 @@ func assertRangeFeatureMatchesParquet( expectedTS := expectedRows[i]["event_timestamp"].(int64) actualTS := result.EventTimestamps[eIdx].Val[i].GetUnixTimestampVal() - assert.Equal(t, expectedTS, actualTS) + // EventTimestamps comes from _ts: hash field stored in seconds; + // expectedTS is in ms (from ReadParquetDynamically), so divide by 1000. + assert.Equal(t, expectedTS/1000, actualTS) } } } @@ -487,7 +498,8 @@ func TestGetOnlineFeaturesRangeValkey_ReverseSortOrder(t *testing.T) { expectedTS := expectedRows[i]["event_timestamp"].(int64) actualTS := result.EventTimestamps[eIdx].Val[i].GetUnixTimestampVal() - assert.Equal(t, expectedTS, actualTS, + // EventTimestamps is seconds; expectedTS is ms from parquet. + assert.Equal(t, expectedTS/1000, actualTS, "timestamp mismatch feature=%s entity=%d idx=%d", feature, entityID, i) } @@ -569,7 +581,8 @@ func TestGetOnlineFeaturesRangeValkey_RangeStartInclusive(t *testing.T) { expTS := expected[i]["event_timestamp"].(int64) actTS := result.EventTimestamps[0].Val[i].GetUnixTimestampVal() - assert.Equal(t, expTS, actTS) + // EventTimestamps is seconds; expTS is ms from parquet. + assert.Equal(t, expTS/1000, actTS) } } } @@ -658,7 +671,8 @@ func TestGetOnlineFeaturesRangeValkey_LimitAppliedAfterHMGET(t *testing.T) { expTS := expectedRows[i]["event_timestamp"].(int64) actTS := actualTS[i].GetUnixTimestampVal() - assert.Equal(t, expTS, actTS, + // EventTimestamps is seconds; expTS is ms from parquet. + assert.Equal(t, expTS/1000, actTS, "timestamp mismatch feature=%s idx=%d", feature, i) if expectedRows[i][feature] == nil { diff --git a/go/internal/feast/onlinestore/redisonlinestore_test.go b/go/internal/feast/onlinestore/redisonlinestore_test.go index 7d27fa4d0b5..8c3f031a2f2 100644 --- a/go/internal/feast/onlinestore/redisonlinestore_test.go +++ b/go/internal/feast/onlinestore/redisonlinestore_test.go @@ -423,7 +423,7 @@ func writeTestRecord( fieldHash, valB, ).Err()) require.NoError(t, store.client.ZAdd(ctx, zkey, redis.Z{ - Score: float64(ts.AsTime().Unix()), + Score: float64(ts.AsTime().UnixMilli()), Member: string(sortKeyBytes), }).Err()) } @@ -461,8 +461,8 @@ func TestOnlineReadRange(t *testing.T) { SortKeyFilters: []*model.SortKeyFilter{ { SortKeyName: "ts", - RangeStart: int64(1000), - RangeEnd: int64(2500), + RangeStart: int64(1000000), + RangeEnd: int64(2500000), StartInclusive: true, EndInclusive: true, }, @@ -510,7 +510,7 @@ func TestOnlineReadRange_Reverse(t *testing.T) { { SortKeyName: "ts", RangeStart: int64(0), - RangeEnd: int64(30)}, + RangeEnd: int64(30000)}, }, } @@ -539,7 +539,7 @@ func TestOnlineReadRange_EmptyResult(t *testing.T) { Limit: 10, IsReverseSortOrder: false, SortKeyFilters: []*model.SortKeyFilter{ - {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(10)}, + {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(10000)}, }, } @@ -577,7 +577,7 @@ func TestOnlineReadRange_WithLimit(t *testing.T) { Limit: 3, // Only get 3 IsReverseSortOrder: false, SortKeyFilters: []*model.SortKeyFilter{ - {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(10000)}, + {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(10000000)}, }, } @@ -623,7 +623,7 @@ func TestOnlineReadRange_MultipleEntityKeys(t *testing.T) { Limit: 10, IsReverseSortOrder: false, SortKeyFilters: []*model.SortKeyFilter{ - {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(5000)}, + {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(5000000)}, }, } @@ -669,7 +669,7 @@ func TestOnlineReadRange_MultipleFeatureViews(t *testing.T) { Limit: 10, IsReverseSortOrder: false, SortKeyFilters: []*model.SortKeyFilter{ - {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(5000)}, + {SortKeyName: "ts", RangeStart: int64(0), RangeEnd: int64(5000000)}, }, } @@ -767,10 +767,10 @@ func TestOnlineReadRange_ExclusiveRangeBounds(t *testing.T) { SortKeyFilters: []*model.SortKeyFilter{ { SortKeyName: "ts", - RangeStart: int64(1000), - RangeEnd: int64(3000), - StartInclusive: false, // Exclude 1000 - EndInclusive: false, // Exclude 3000 + RangeStart: int64(1000000), + RangeEnd: int64(3000000), + StartInclusive: false, // Exclude 1000s + EndInclusive: false, // Exclude 3000s }, }, } diff --git a/go/internal/feast/utils/redis_read_range_utils.go b/go/internal/feast/utils/redis_read_range_utils.go index e9af3700b47..9468b68f0a6 100644 --- a/go/internal/feast/utils/redis_read_range_utils.go +++ b/go/internal/feast/utils/redis_read_range_utils.go @@ -205,8 +205,8 @@ func fmtInterface(v interface{}) (string, error) { return fmt.Sprintf("%g", x), nil case time.Time: - // Feast timestamps become time.Time for sort keys - return fmt.Sprintf("%d", x.Unix()), nil + // Sort key scores are stored as milliseconds; use UnixMilli to match. + return fmt.Sprintf("%d", x.UnixMilli()), nil case string: return x, nil diff --git a/go/internal/feast/utils/redis_read_range_utils_test.go b/go/internal/feast/utils/redis_read_range_utils_test.go index 21e192a6c6f..71f531aa981 100644 --- a/go/internal/feast/utils/redis_read_range_utils_test.go +++ b/go/internal/feast/utils/redis_read_range_utils_test.go @@ -89,7 +89,7 @@ func TestGetScoreRange(t *testing.T) { StartInclusive: true, EndInclusive: true, }}, - "1000", "2000", + "1000000", "2000000", }, } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index b23afdfec97..707621b6277 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -29,7 +29,12 @@ type Row struct { Created int64 } -var feastExec = "feast" +var feastExec = func() string { + if bin := os.Getenv("FEAST_BIN"); bin != "" { + return bin + } + return "feast" +}() func ReadParquet(filePath string) ([]*Row, error) { allocator := memory.NewGoAllocator() @@ -122,7 +127,7 @@ func ReadParquetDynamically(filePath string) ([]map[string]interface{}, error) { case *array.Timestamp: nanoseconds := int64(col.Value(rowIdx).ToTime(arrow.Second).Unix()) t := time.Unix(0, nanoseconds) - row[field.Name] = t.Unix() + row[field.Name] = t.UnixMilli() case *array.List: // Handle array (list) types listValues := []interface{}{} @@ -146,7 +151,7 @@ func ReadParquetDynamically(filePath string) ([]map[string]interface{}, error) { case *array.Timestamp: nanoseconds := int64(childCol.Value(int(i)).ToTime(arrow.Second).Unix()) t := time.Unix(0, nanoseconds) - listValues = append(listValues, t.Unix()) + listValues = append(listValues, t.UnixMilli()) default: listValues = append(listValues, nil) // Handle unsupported types } diff --git a/go/types/typeconversion.go b/go/types/typeconversion.go index 15cbda67640..0f07bf71385 100644 --- a/go/types/typeconversion.go +++ b/go/types/typeconversion.go @@ -830,20 +830,20 @@ func valueTypeToGoTypeTimestampAsString(value *types.Value, timestampAsString bo return v case *types.Value_UnixTimestampVal: if timestampAsString { - return time.Unix(x.UnixTimestampVal, 0).UTC().Format(TimestampFormat) + return unixTsToTime(x.UnixTimestampVal).Format(TimestampFormat) } - return time.Unix(x.UnixTimestampVal, 0).UTC() + return unixTsToTime(x.UnixTimestampVal) case *types.Value_UnixTimestampListVal: if timestampAsString { timestamps := make([]string, len(x.UnixTimestampListVal.Val)) for i, ts := range x.UnixTimestampListVal.Val { - timestamps[i] = time.Unix(ts, 0).UTC().Format(TimestampFormat) + timestamps[i] = unixTsToTime(ts).Format(TimestampFormat) } return timestamps } timestamps := make([]time.Time, len(x.UnixTimestampListVal.Val)) for i, ts := range x.UnixTimestampListVal.Val { - timestamps[i] = time.Unix(ts, 0).UTC() + timestamps[i] = unixTsToTime(ts) } return timestamps default: @@ -851,6 +851,16 @@ func valueTypeToGoTypeTimestampAsString(value *types.Value, timestampAsString bo } } +// unixTsToTime converts a unix_timestamp_val int64 to time.Time using a threshold to +// distinguish seconds (regular features) from milliseconds (sort key columns). +// Values > 1e11 are unambiguously milliseconds; current-era seconds ~1.7e9 < 1e11. +func unixTsToTime(val int64) time.Time { + if val > 1e11 { + return time.UnixMilli(val).UTC() + } + return time.Unix(val, 0).UTC() +} + func transformStringToBytes(str string) []byte { bytes, decodeErr := base64.StdEncoding.DecodeString(str) if decodeErr != nil { From 9b97584772f5f22148e79b9054718538bca49b5d Mon Sep 17 00:00:00 2001 From: piket Date: Thu, 9 Jul 2026 15:03:27 -0700 Subject: [PATCH 04/11] fix: preserve millisecond precision when reading UNIX_TIMESTAMP sort keys (EAPC-22316) InterfaceToProtoValue's `time.Time` case truncated to whole seconds via .Unix(), even though sort key UNIX_TIMESTAMP columns are always written at millisecond precision (per the earlier fix in this branch). This is the gocql-specific path: only SortedFeatureView sort key columns are stored as native Cassandra/Scylla `timestamp` columns (regular features are opaque blobs read via UnmarshalStoredProto), so a raw time.Time only ever reaches this function for ms-precision sort keys. Reproduced against a real Cassandra instance: after `feast materialize`, cqlsh showed three rows with fully distinct millisecond clustering values, but /get-online-features-range collapsed two of them (18ms apart) to an identical second-precision string. Fix: - InterfaceToProtoValue: time.Time/[]time.Time cases now use UnixMilli() instead of Unix(), matching the >1e11 ms/seconds threshold already used by the decode side (unixTsToTime). - Added GetTimestampMillis helper; used it for the EventTimestamps metadata field in grpc_server.go, which had the same truncation bug. - Added regression tests: typeconversion_test.go (round-trip + distinctness of sub-second values), serving_test.go (range value-conversion loop with two sort-key timestamps 18ms apart), and a new ScyllaDB integration test + fixture (regression_repo.py / sub_second_data.parquet) covering the same scenario against a real backend. Verified end-to-end: rebuilt the Go server from this commit and re-ran the exact repro (real Cassandra + feast materialize) that caught the bug - /get-online-features-range now returns three distinct millisecond-precision values instead of two collapsed ones. Co-Authored-By: Claude Sonnet 5 --- .../scylladb/feature_repo/regression_repo.py | 49 +++++++++++ .../feature_repo/sub_second_data.parquet | Bin 0 -> 3428 bytes .../scylladb/scylladb_integration_test.go | 77 ++++++++++++++++++ .../feast/onlineserving/serving_test.go | 65 +++++++++++++++ go/internal/feast/server/grpc_server.go | 2 +- go/types/typeconversion.go | 19 ++++- go/types/typeconversion_test.go | 53 +++++++++++- 7 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py create mode 100644 go/internal/feast/integration_tests/scylladb/feature_repo/sub_second_data.parquet diff --git a/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py b/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py new file mode 100644 index 00000000000..4fbf324be8c --- /dev/null +++ b/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py @@ -0,0 +1,49 @@ +# Regression fixture for EAPC-22316 follow-up: a SortedFeatureView whose +# UNIX_TIMESTAMP sort key is requested as a feature must retain millisecond +# precision for rows that differ by less than one second (Go read-side fix in +# InterfaceToProtoValue, go/types/typeconversion.go). + +from datetime import timedelta + +from feast import Entity, Field, FileSource, SortedFeatureView +from feast.protos.feast.core.SortedFeatureView_pb2 import SortOrder +from feast.sort_key import SortKey +from feast.types import String, UnixTimestamp +from feast.value_type import ValueType + +tags = {"team": "Feast"} +owner = "test@test.com" + +sub_second_entity: Entity = Entity( + name="sub_second_entity", + join_keys=["sub_second_entity_id"], + value_type=ValueType.STRING, + tags=tags, + owner=owner, +) + +sub_second_source: FileSource = FileSource( + path="sub_second_data.parquet", timestamp_field="event_timestamp" +) + +sub_second_sort_key_view: SortedFeatureView = SortedFeatureView( + name="sub_second_sort_key_view", + entities=[sub_second_entity], + ttl=timedelta(days=0), + source=sub_second_source, + tags=tags, + description="Regression fixture: sort key rows <1s apart for the same entity", + owner=owner, + online=True, + sort_keys=[ + SortKey( + name="event_timestamp", + value_type=ValueType.UNIX_TIMESTAMP, + default_sort_order=SortOrder.DESC, + ) + ], + schema=[ + Field(name="value", dtype=String), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], +) diff --git a/go/internal/feast/integration_tests/scylladb/feature_repo/sub_second_data.parquet b/go/internal/feast/integration_tests/scylladb/feature_repo/sub_second_data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a10514da6c5c18962991ee6220fe2dbd747f427d GIT binary patch literal 3428 zcmb_fU2ofD6n3&U1gOL`)xnW+QAJw>Qd;s^)~QH1X_F=-q0Kfs@is;NO5)UZyx2~f zjY|6gz%^HhOD zUZEK}Qlb4cb&aCd;z5d{RMXL%@n({y=xcN+P6z0#@Cih=sds`?88cgf!Rs4-xCeZ` z00nqA>D>zGBVI&b1wmUI*iHZzwkr2c1&^GrHc$Y6on{vBSBZdGB_^Qv#&r+bf8{zt zC+N%+nz{evr{~WET1kryh^W6a}$j0 ze*5Kj84hARmlq~Dm7J~6!SX3MSjbiQewJbaw%k<>kq%Py+6_OmZo5*;%T&wDc1u%k zP?ZQ66NS~GsJrU)WCcv#Qiq_~ax_D=9nrA#Pp?IGK^+WG{F*#aMMqU`tXDt~CxTEQ z90^xqRD@m&XQ{x|@Wp;2K?dghD$xe|M}0-lzueMt{o?kV)kYA6s3hW=iq!f_^GiKZgj%#VS`QO#7; zla}1qUBk4aw>IuQjvm7qtfB+a?5NTBM${Ay6%I^S*I@^ZFmPa-AnU#b%QRW4xJ;ft zfQ^UOBH-!-b`hnmsk+iyJo9L^oUw)OqOPgQeZ^zo8rFCw=u16Sc95B&I-(*vBB(!( zswSvwW(RHW-YIO}jGoSsS!5Bt=v_d>1zwkG{U@fF$J00vok4$x!r?HtLj& zY**s%qK>QZCw3*zYI3^XmCYkKV@H$yP|laOOM6wjlrKM0irffss*R+A@@8tsR`{gb zNO``6#>}VN~g)r*xo9j$$i zyF3Q{f&=-R^1?iDS|eT4!M`gMc676bmTa#M^b9!vhz8RQssvNUrTuPO6p3n z02uY|EUyO%8)Kd0k!tcM$w|%(k>_mSKdVK_6uQtGoI01*ieW z3AJ-%hw zQG?z~5*{#LVJoHm}M2C0Q2iEtnwMuhah0nL7W4ew{RVu z8SNKHkY4Cyi3&K1;r2+m?ENQ7IN#@>SoQ;o*|5aHLsxY+Wy{(mUqTy`FPRPR>nDsq Q&}V Date: Fri, 10 Jul 2026 10:48:33 -0700 Subject: [PATCH 05/11] test: add secondary custom-named sort key regression fixture (EAPC-22316) The existing sub-second regression fixture used "event_timestamp" as the sort key name - the same name used everywhere else in this test suite. Add a second fixture where the sort key is a distinctly-named feature ("viewed_at"), separate from the source's own "event_timestamp" watermark field, mirroring the real customer schema (personalization_pdp_features) that originally surfaced this bug. Guards against the fix being accidentally tied to the "event_timestamp" column name rather than working for any UNIX_TIMESTAMP column declared as a sort key. Verified against a real Cassandra instance: materialized both fixtures, confirmed via cqlsh that the new table's clustering column is literally named "viewed_at" with full millisecond precision, while the independent "event_ts" watermark column remains at second precision as expected. Both new Go integration tests pass with the exact expected millisecond values. Co-Authored-By: Claude Sonnet 5 --- .../scylladb/feature_repo/regression_repo.py | 40 +++++++++ .../sub_second_custom_sortkey_data.parquet | Bin 0 -> 3478 bytes .../scylladb/scylladb_integration_test.go | 76 ++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 go/internal/feast/integration_tests/scylladb/feature_repo/sub_second_custom_sortkey_data.parquet diff --git a/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py b/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py index 4fbf324be8c..d2163d2c344 100644 --- a/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py +++ b/go/internal/feast/integration_tests/scylladb/feature_repo/regression_repo.py @@ -47,3 +47,43 @@ Field(name="event_timestamp", dtype=UnixTimestamp), ], ) + +# Second regression fixture: the sort key is a secondary feature with a name +# other than "event_timestamp" (mirroring the real customer schema, where the +# sort key was named "viewed_at" and was distinct from the source's own +# ingestion timestamp field). This guards against the fix accidentally being +# tied to the "event_timestamp" column name rather than working for any +# UNIX_TIMESTAMP column declared as a sort key. +custom_sortkey_entity: Entity = Entity( + name="custom_sortkey_entity", + join_keys=["custom_sortkey_entity_id"], + value_type=ValueType.STRING, + tags=tags, + owner=owner, +) + +sub_second_custom_sortkey_source: FileSource = FileSource( + path="sub_second_custom_sortkey_data.parquet", timestamp_field="event_timestamp" +) + +sub_second_custom_sortkey_view: SortedFeatureView = SortedFeatureView( + name="sub_second_custom_sortkey_view", + entities=[custom_sortkey_entity], + ttl=timedelta(days=0), + source=sub_second_custom_sortkey_source, + tags=tags, + description="Regression fixture: sort key named 'viewed_at' (not 'event_timestamp'), rows <1s apart", + owner=owner, + online=True, + sort_keys=[ + SortKey( + name="viewed_at", + value_type=ValueType.UNIX_TIMESTAMP, + default_sort_order=SortOrder.DESC, + ) + ], + schema=[ + Field(name="value", dtype=String), + Field(name="viewed_at", dtype=UnixTimestamp), + ], +) diff --git a/go/internal/feast/integration_tests/scylladb/feature_repo/sub_second_custom_sortkey_data.parquet b/go/internal/feast/integration_tests/scylladb/feature_repo/sub_second_custom_sortkey_data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a9193905dd7e9c3f3d9bf4334045231d24809f1b GIT binary patch literal 3478 zcmb_f&u`jx6gM=n=oD#DH8Ue|Kqy6 z0!=@D{_Cq(U%&qCL*k3SU*r4b=HrcT;C&UA4oxIoFQn^mOH8;QXzs`yB(xy*|nXX z(X@Mg=S&?n{meHt$o0r_+f*=2hQAGjzY8QjON5qnL}U|lMg<92*5S1m zW+G%HL0}Ken(@8&Fkb}1KL!%dgNg40h&tZ~i+Hg)yp^%rm$7@8zX!s91`;oBB>o6S z|3ecjSo?U^_ld2m>GUlA6bSzUQLH9ju7JhGx-aR5Nc>6#kMVBAYa6iKZgj z^v~q8xMnKqc~kD`u3_5oyRk>l;%C4ED&7~(wi@4z#ZA#rfnmD34m+rN-+?XT(LD>u zv`4B)CNG}A#>Z=V;OZE59;KzJy3(BI`EE6xk%i{suBpjA#rME9tkG1^le((xpl62a zh>GZlp#ChbnxL+kZ9Mz%w6L`sznF1m-i!FMTY;!^zWuA_|MD2~d9ewOkFyZ>F5#jB zQ}#QyHo4%}SezAIu?p!{@$%y-bFM5C$ zy}s8xx+e9-{s?==8j1aWZ-^cZ)V{6tOys7LcbGd&910d6yD=-glfILVUl8;f%W?OG zyZ#8KU#@+E&rbV*W!Yo+#pfQ&ZnL>ITVUBXn{Ru(Ty9Q2L?)aW%S*@m{vK8j#gms< zcF4jxcq2$ohAr;l#3}*2uZh=@@+S2Q=4vGNb0C7nY=CK(O$!92H{5`a_hLGt>DbA7%*~8;` zu2?>8YppB3%OlV)IFP@IUl3QChgh>tEnOYRCf9G&^cLVTVvWl{tY?1yuEz70=gb_` z0P>uiiN)kyQK$|Y5Ig1-a^}c~dP_OfhYW4)CI1i6c48LYn=?+ zQ0r;lOJ<`3eK4-|qOXckfZF-?6+_pR6c0U6V3y@@LFc-B zLH?i@84-KT@8zUa%VlQ!2YUf}!h#-8RSS+QCz-d;ox!KD-hc<^u@?#s+ym@GLlk(s zQOI8AdAaVe4)?BJtIpOD_MEO1b!T~BU&dDLa*||L6+gx#oGak@&h%a#j_+amNC*Bv z*8JX=in=Bng^}v_&QKK!BiYENJsh*4%VyP*Qrq6WsB{n9N`8A-G8v5T(r1Go$tisx zA!fcbDt9YxWptEf**eajv0epe98!62NGU+E`;&Rbvho9%hqyiNd6Nu=7jC>27f-c# zO|IGf6C9^WA7EIOg?LWi)`WP>l z<2)7y;m=QB1LiZ!j_(9ec?R{t$U~gBfOClR7ajn|7d{6C&k3B?v;KHSb6urYWu#QH zC8x3@rReA|yL8FqEV$DxQSfsBPylh64}y+aV*hhjb+%;7+VVb3vGIo|=6{kR_-Fj| NhalGo;sN|e_76916j%TN literal 0 HcmV?d00001 diff --git a/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go b/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go index 96de7452ff8..c2e7d9faf36 100644 --- a/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go +++ b/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go @@ -447,6 +447,82 @@ func TestGetOnlineFeaturesRange_SubSecondSortKeyValuesRemainDistinct(t *testing. } } +// TestGetOnlineFeaturesRange_CustomNamedSortKeyValuesRemainDistinct guards +// against the EAPC-22316 fix being tied to the "event_timestamp" column name +// rather than working for any UNIX_TIMESTAMP column declared as a sort key. +// The fixture (sub_second_custom_sortkey_view, seeded from +// sub_second_custom_sortkey_data.parquet) declares "viewed_at" - a name +// distinct from the source's own "event_timestamp" watermark field - as the +// sort key, mirroring the real customer schema. Same three-row shape: two +// rows 18ms apart within the same second, one two seconds later. +func TestGetOnlineFeaturesRange_CustomNamedSortKeyValuesRemainDistinct(t *testing.T) { + entities := make(map[string]*types.RepeatedValue) + entities["custom_sortkey_entity_id"] = &types.RepeatedValue{ + Val: []*types.Value{ + {Val: &types.Value_StringVal{StringVal: "entity-1"}}, + }, + } + + request := &serving.GetOnlineFeaturesRangeRequest{ + Kind: &serving.GetOnlineFeaturesRangeRequest_Features{ + Features: &serving.FeatureList{ + Val: []string{ + "sub_second_custom_sortkey_view:viewed_at", + "sub_second_custom_sortkey_view:value", + }, + }, + }, + Entities: entities, + SortKeyFilters: []*serving.SortKeyFilter{ + { + SortKeyName: "viewed_at", + Query: &serving.SortKeyFilter_Range{ + Range: &serving.SortKeyFilter_RangeQuery{ + RangeStart: &types.Value{Val: &types.Value_UnixTimestampVal{UnixTimestampVal: 0}}, + }, + }, + }, + }, + Limit: 10, + } + + response, err := client.GetOnlineFeaturesRange(ctx, request) + require.NoError(t, err) + require.NotNil(t, response) + require.Len(t, response.Results, 2, "expected viewed_at and value results") + + viewedAtIdx := -1 + for i, name := range response.Metadata.FeatureNames.Val { + if name == "viewed_at" { + viewedAtIdx = i + } + } + require.NotEqual(t, -1, viewedAtIdx, "expected to find viewed_at in the response") + + viewedAtVector := response.Results[viewedAtIdx] + require.Len(t, viewedAtVector.Values, 1, "expected results for the single requested entity") + + timestamps := viewedAtVector.Values[0].Val + require.Len(t, timestamps, 3, "expected all 3 sub-second-apart rows to be returned") + + seen := make(map[int64]bool) + for _, v := range timestamps { + ts := v.GetUnixTimestampVal() + assert.False(t, seen[ts], "sort key value %d was returned more than once; sub-second rows collapsed", ts) + seen[ts] = true + } + assert.Len(t, seen, 3, "expected 3 distinct millisecond-precision sort key values") + + expected := map[int64]bool{ + 1717244257886: true, // 2024-06-01 12:17:37.886 + 1717244257904: true, // 2024-06-01 12:17:37.904 + 1717244259035: true, // 2024-06-01 12:17:39.035 + } + for ts := range seen { + assert.True(t, expected[ts], "unexpected sort key value %d", ts) + } +} + func assertResponseData(t *testing.T, response *serving.GetOnlineFeaturesRangeResponse, featureNames []string, entitiesRequested int, includeMetadata bool) { assert.NotNil(t, response) assert.Equal(t, 1, len(response.Entities), "Should have 1 list of entity") From 43f2874ece4ba26f36bc711c865ef198ea58bd28 Mon Sep 17 00:00:00 2001 From: piket Date: Fri, 10 Jul 2026 11:14:15 -0700 Subject: [PATCH 06/11] fix: disambiguate raw integer sort-key timestamps by magnitude (EAPC-22316) _python_datetime_to_int_ms_timestamp's fallback for non-datetime values (`else: int_timestamps.append(int(value))`) was copied verbatim from the sibling seconds-precision function and silently assumed any raw integer was already milliseconds. A raw (non-datetime) value reaching this branch means the sort key column arrived as a plain integer rather than a typed timestamp - e.g. a Spark LongType column instead of TimestampType, which happens if the source Avro schema's field lacks a timestamp-millis/ timestamp-micros logicalType. There's no way to know the unit of a bare integer without a heuristic, and this code used none. Fix: apply the same magnitude threshold already used everywhere else in this codebase (MS_TIMESTAMP_THRESHOLD = 1e11, matching unixTsToTime in Go and feast_value_type_to_python_type here) to decide whether a raw integer is already milliseconds or is seconds needing conversion, instead of assuming. Also logs a warning, since hitting this branch at all for a sort key column is itself a signal of an upstream typing issue worth surfacing rather than silently masking. Extracted the threshold into a shared MS_TIMESTAMP_THRESHOLD constant used by both the read-side (feast_value_type_to_python_type) and write-side (_python_datetime_to_int_ms_timestamp) disambiguation, replacing the previously duplicated inline 1e11 literals. Co-Authored-By: Claude Sonnet 5 --- sdk/python/feast/type_map.py | 38 ++++++++++++++++++++++---- sdk/python/tests/unit/test_type_map.py | 28 +++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 209bb200d54..df08e860208 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -55,6 +55,11 @@ # null timestamps get converted to -9223372036854775808 NULL_TIMESTAMP_INT_VALUE: int = np.datetime64("NaT").astype(int) +# Threshold for disambiguating a raw UNIX_TIMESTAMP integer as milliseconds vs. +# seconds: current-era seconds are ~1.7e9, milliseconds ~1.7e12, so anything +# above this is unambiguously milliseconds (safe until year ~5138 in seconds). +MS_TIMESTAMP_THRESHOLD = 1e11 + logger = logging.getLogger(__name__) @@ -80,13 +85,13 @@ def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: # Convert UNIX_TIMESTAMP values to `datetime`. # unix_timestamp_val stores seconds for regular features and milliseconds for sort key - # columns. Values > 1e11 are unambiguously ms (current-era seconds ~1.7e9, ms ~1.7e12; - # threshold is safe until year ~5138 in seconds). + # columns; see MS_TIMESTAMP_THRESHOLD for the disambiguation rule. if val_attr == "unix_timestamp_list_val": val = [ ( datetime.fromtimestamp( - v / 1000.0 if v > 1e11 else float(v), tz=timezone.utc + v / 1000.0 if v > MS_TIMESTAMP_THRESHOLD else float(v), + tz=timezone.utc, ) if v != NULL_TIMESTAMP_INT_VALUE else None @@ -96,7 +101,8 @@ def feast_value_type_to_python_type(field_value_proto: ProtoValue) -> Any: elif val_attr == "unix_timestamp_val": val = ( datetime.fromtimestamp( - val / 1000.0 if val > 1e11 else float(val), tz=timezone.utc + val / 1000.0 if val > MS_TIMESTAMP_THRESHOLD else float(val), + tz=timezone.utc, ) if val != NULL_TIMESTAMP_INT_VALUE else None @@ -386,7 +392,29 @@ def _python_datetime_to_int_ms_timestamp( elif isinstance(value, type(np.nan)): int_timestamps.append(NULL_TIMESTAMP_INT_VALUE) else: - int_timestamps.append(int(value)) + # A raw (non-datetime) value here means the sort key column + # arrived as a plain integer rather than a typed timestamp - + # e.g. a Spark LongType column instead of TimestampType, + # which can happen if the source Avro schema's field lacks a + # timestamp-millis/timestamp-micros logicalType. There is no + # reliable way to know whether that raw integer is already + # milliseconds or is seconds, so apply the same magnitude + # threshold used everywhere else in this codebase to decide + # (unixTsToTime in Go, feast_value_type_to_python_type here) + # rather than silently assuming it's already milliseconds. + raw = int(value) + if raw > int(MS_TIMESTAMP_THRESHOLD): + int_timestamps.append(raw) + else: + logger.warning( + "Sort key column received a raw integer timestamp (%d) instead of " + "a typed datetime value; assuming seconds and converting to " + "milliseconds. This usually means the upstream source (e.g. an " + "Avro schema without a timestamp-millis/timestamp-micros " + "logicalType) is not producing typed timestamps.", + raw, + ) + int_timestamps.append(raw * 1000) return int_timestamps diff --git a/sdk/python/tests/unit/test_type_map.py b/sdk/python/tests/unit/test_type_map.py index be8a25c1639..55406cdafe3 100644 --- a/sdk/python/tests/unit/test_type_map.py +++ b/sdk/python/tests/unit/test_type_map.py @@ -3,6 +3,8 @@ import pytest from feast.type_map import ( + MS_TIMESTAMP_THRESHOLD, + _python_datetime_to_int_ms_timestamp, feast_value_type_to_python_type, python_values_to_proto_values, ) @@ -30,6 +32,32 @@ def test_null_unix_timestamp_list(): assert converted[0] is None +def test_python_datetime_to_int_ms_timestamp_raw_int_below_threshold_is_treated_as_seconds(): + """A raw (non-datetime) integer below MS_TIMESTAMP_THRESHOLD is assumed to be + seconds and converted to milliseconds - e.g. a sort key column that arrived + as a Spark LongType rather than TimestampType (source Avro schema missing a + timestamp-millis/timestamp-micros logicalType).""" + + seconds_value = 1717244257 # 2024-06-01 12:17:37 UTC, well below the threshold + assert seconds_value < MS_TIMESTAMP_THRESHOLD + + result = _python_datetime_to_int_ms_timestamp([seconds_value]) + + assert result == [seconds_value * 1000] + + +def test_python_datetime_to_int_ms_timestamp_raw_int_above_threshold_passes_through(): + """A raw integer already above MS_TIMESTAMP_THRESHOLD is assumed to already be + milliseconds and is passed through unchanged.""" + + ms_value = 1717244257886 # 2024-06-01 12:17:37.886 UTC + assert ms_value > MS_TIMESTAMP_THRESHOLD + + result = _python_datetime_to_int_ms_timestamp([ms_value]) + + assert result == [ms_value] + + @pytest.mark.parametrize( "values", ( From 7bdaa69bdce54cb50a80b6b39a856225ebdf60e3 Mon Sep 17 00:00:00 2001 From: piket Date: Fri, 10 Jul 2026 12:42:00 -0700 Subject: [PATCH 07/11] feat: always-on version banner + always-on materialized-row-count logging Always-on version banner: - Fixed a pre-existing bug in version.get_version(): this repo publishes its SDK distribution as "eg-feast" (see setup.py), not "feast", so the existing importlib.metadata.version("feast") lookup has been silently returning "unknown" in every real deployment of this fork. Now checks both names. This also fixes three other existing callers that were affected (utils.py's USER_AGENT, transformation_server.py, and lambda_engine.py's Lambda tags) - no changes needed there, they just get a working version now. - Added get_installed_version(package_name), an opportunistic lookup for a wrapper package (e.g. the "feature-store-materialization" job image) that returns "not installed" gracefully if absent. - Both batch materialization entry points (_print_materialization_log, covering materialize() and materialize_incremental()) and the streaming entry point (SparkKafkaProcessor.ingest_stream_feature_view) now log "feast= feature-store-materialization=" on startup, unconditionally - no config needed to see it. Always-on row-count logging: - _convert_arrow_to_proto (the single choke point all three materialization write paths converge on - local batch engine, Spark batch engine, and Spark Structured Streaming) now logs a "[materialize] feature_view= rows=" line after every batch, unconditionally. No raw entity/feature data is logged. Tests: 6 new tests for get_version()/get_installed_version() fallback behavior (test_version.py). Verified no regressions across test_type_map.py, test_feature_views.py, repo_config, online_store, and compute_engines test suites (198 passed, 5 pre-existing skips). Co-Authored-By: Claude Sonnet 5 --- docs/reference/feature-store-yaml.md | 1 - sdk/python/feast/feature_store.py | 6 ++ .../infra/contrib/spark_kafka_processor.py | 6 ++ sdk/python/feast/utils.py | 15 +++- sdk/python/feast/version.py | 23 +++++- .../infra/compute_engines/local/test_nodes.py | 3 +- sdk/python/tests/unit/test_version.py | 80 +++++++++++++++++++ 7 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 sdk/python/tests/unit/test_version.py diff --git a/docs/reference/feature-store-yaml.md b/docs/reference/feature-store-yaml.md index 01f586c047b..bac483a2ebc 100644 --- a/docs/reference/feature-store-yaml.md +++ b/docs/reference/feature-store-yaml.md @@ -24,7 +24,6 @@ online_store: * **path** \(a local filepath\) — Parameter for the sqlite online store. Defines the path to the SQLite database file. * **project\_id** — Optional parameter for the datastore online store. Sets the GCP project id used by Feast, if not set Feast will use the default GCP project id in the local environment. * **project** — Defines a namespace for the entire feature store. Can be used to isolate multiple deployments in a single installation of Feast. - ## Providers The `provider` field defines the environment in which Feast will execute data flows. As a result, it also determines the default values for other fields. diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index b1b5660f35e..10bf7cc7673 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -98,6 +98,7 @@ from feast.transformation.pandas_transformation import PandasTransformation from feast.transformation.python_transformation import PythonTransformation from feast.utils import _get_feature_view_vector_field_metadata, _utc_now +from feast.version import get_installed_version, get_version warnings.simplefilter("once", DeprecationWarning) @@ -3244,6 +3245,11 @@ async def close(self) -> None: def _print_materialization_log( start_date, end_date, num_feature_views: int, online_store: str ): + logger.info( + "Materialization starting: feast=%s feature-store-materialization=%s", + get_version(), + get_installed_version("feature-store-materialization"), + ) if start_date: print( f"Materializing {Style.BRIGHT + Fore.GREEN}{num_feature_views}{Style.RESET_ALL} feature views" diff --git a/sdk/python/feast/infra/contrib/spark_kafka_processor.py b/sdk/python/feast/infra/contrib/spark_kafka_processor.py index 36bc3a6242d..1a4c7bec234 100644 --- a/sdk/python/feast/infra/contrib/spark_kafka_processor.py +++ b/sdk/python/feast/infra/contrib/spark_kafka_processor.py @@ -25,6 +25,7 @@ from feast.infra.provider import get_provider from feast.sorted_feature_view import SortedFeatureView from feast.stream_feature_view import StreamFeatureView +from feast.version import get_installed_version, get_version logger = logging.getLogger(__name__) @@ -216,6 +217,11 @@ def _create_infra_if_necessary(self): def ingest_stream_feature_view( self, to: PushMode = PushMode.ONLINE ) -> StreamingQuery: + logger.info( + "Streaming materialization starting: feast=%s feature-store-materialization=%s", + get_version(), + get_installed_version("feature-store-materialization"), + ) self._create_infra_if_necessary() ingested_stream_df = self._ingest_stream_data() transformed_df = self._construct_transformation_plan(ingested_stream_df) diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 2a8ee42625b..48ff94a1e16 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1,5 +1,6 @@ import copy import itertools +import logging import os import typing import warnings @@ -57,6 +58,8 @@ from feast.infra.registry.base_registry import BaseRegistry from feast.on_demand_feature_view import OnDemandFeatureView +logger = logging.getLogger(__name__) + APPLICATION_NAME = "feast-dev/feast" USER_AGENT = "{}/{}".format(APPLICATION_NAME, get_version()) @@ -265,9 +268,17 @@ def _convert_arrow_to_proto( getattr(feature_view, "source_request_sources", None) is not None or getattr(feature_view, "source_feature_view_projections", None) is not None ): - return _convert_arrow_odfv_to_proto(table, feature_view, join_keys) # type: ignore[arg-type] + result = _convert_arrow_odfv_to_proto(table, feature_view, join_keys) # type: ignore[arg-type] else: - return _convert_arrow_fv_to_proto(table, feature_view, join_keys) # type: ignore[arg-type] + result = _convert_arrow_fv_to_proto(table, feature_view, join_keys) # type: ignore[arg-type] + + logger.info( + "[materialize] feature_view=%s rows=%d", + feature_view.name, + len(result), + ) + + return result def _convert_arrow_fv_to_proto( diff --git a/sdk/python/feast/version.py b/sdk/python/feast/version.py index 85d8476a66d..444efbd8503 100644 --- a/sdk/python/feast/version.py +++ b/sdk/python/feast/version.py @@ -1,10 +1,27 @@ from importlib.metadata import PackageNotFoundError, version +# This repo publishes its own SDK distribution as "eg-feast" (see setup.py), not +# "feast", so the installed package name differs from upstream OSS Feast. Check +# both, upstream name first for compatibility with non-fork installs. +_SDK_DISTRIBUTION_NAMES = ("feast", "eg-feast") + def get_version(): """Returns version information of the Feast Python Package.""" + for candidate in _SDK_DISTRIBUTION_NAMES: + try: + return version(candidate) + except PackageNotFoundError: + continue + return "unknown" + + +def get_installed_version(package_name: str) -> str: + """Returns the installed version of an arbitrary package, or "not installed" + if it isn't present. Useful for opportunistically reporting the version of a + wrapper/consumer package (e.g. a materialization job image) without requiring + it to be installed for Feast to function.""" try: - sdk_version = version("feast") + return version(package_name) except PackageNotFoundError: - sdk_version = "unknown" - return sdk_version + return "not installed" diff --git a/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py b/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py index 20e23c35e03..3e68cc5d6e8 100644 --- a/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py +++ b/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py @@ -37,9 +37,10 @@ def create_context(node_outputs): # Setup execution context + repo_config = MagicMock() return ExecutionContext( project="test_proj", - repo_config=MagicMock(), + repo_config=repo_config, offline_store=MagicMock(), online_store=MagicMock(), entity_defs=MagicMock(), diff --git a/sdk/python/tests/unit/test_version.py b/sdk/python/tests/unit/test_version.py new file mode 100644 index 00000000000..ac2428c5dde --- /dev/null +++ b/sdk/python/tests/unit/test_version.py @@ -0,0 +1,80 @@ +from importlib.metadata import PackageNotFoundError + +import pytest + +from feast.feature_store import _print_materialization_log +from feast.utils import _utc_now +from feast.version import get_installed_version, get_version + + +def test_get_version_returns_real_version(): + """get_version() should find whichever of the "feast" / "eg-feast" + distributions is actually installed, not silently fall back to "unknown".""" + assert get_version() != "unknown" + + +def test_get_version_falls_back_from_feast_to_eg_feast(monkeypatch): + """Regression test: this repo publishes its SDK distribution as "eg-feast" + (see setup.py), not "feast". get_version() must fall back to "eg-feast" when + "feast" isn't installed, instead of only checking "feast" and returning + "unknown".""" + + def fake_version(name): + if name == "feast": + raise PackageNotFoundError() + if name == "eg-feast": + return "1.2.3" + raise PackageNotFoundError() + + monkeypatch.setattr("feast.version.version", fake_version) + + assert get_version() == "1.2.3" + + +def test_get_version_falls_back_from_eg_feast_to_feast(monkeypatch): + """The reverse case: upstream OSS installs where the distribution is named + "feast" rather than "eg-feast" must still resolve correctly.""" + + def fake_version(name): + if name == "feast": + return "0.9.0" + raise PackageNotFoundError() + + monkeypatch.setattr("feast.version.version", fake_version) + + assert get_version() == "0.9.0" + + +def test_get_version_returns_unknown_when_neither_installed(monkeypatch): + def fake_version(name): + raise PackageNotFoundError() + + monkeypatch.setattr("feast.version.version", fake_version) + + assert get_version() == "unknown" + + +def test_get_installed_version_for_missing_package(): + assert get_installed_version("definitely-not-a-real-package-xyz") == "not installed" + + +def test_get_installed_version_for_installed_package(): + # "feast" or "eg-feast" (whichever is actually installed) should resolve. + result = get_installed_version("pytest") + assert result != "not installed" + assert pytest.__version__ == result + + +def test_materialization_log_always_includes_version_banner(caplog): + """The version banner is always-on - it must appear regardless of any + opt-in config, unlike the row-content logging.""" + now = _utc_now() + + with caplog.at_level("INFO"): + _print_materialization_log( + start_date=now, end_date=now, num_feature_views=1, online_store="sqlite" + ) + + assert "Materialization starting" in caplog.text + assert "feast=" in caplog.text + assert "feature-store-materialization=" in caplog.text From f2f6ee377a24a75d5155c1393a6ae8b259ea1e57 Mon Sep 17 00:00:00 2001 From: piket Date: Fri, 17 Jul 2026 11:46:34 -0700 Subject: [PATCH 08/11] fix: correct EventTimestamps precision and update HTTP fixtures for ms sort keys (EAPC-22316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grpc_server.go used GetTimestampMillis() when building EventTimestamps in the GetOnlineFeaturesRange response. EventTimestamps come from the _ts: hash field, which stores a timestamppb.Timestamp (seconds/nanos) written by the Python online-write path — they are NOT sort-key scores. Returning them as milliseconds produced values 1000x too large vs what the Valkey integration tests expect. Changed to GetTimestampSeconds(), restoring the original behaviour. HTTP fixture updates: event_timestamp is both the sort key and a schema field in all_dtypes_sorted. After the EAPC-22316 materialisation fix it is stored as milliseconds, so Go's ValueTypeToGoTypeTimestampAsString now formats it with sub-second precision (e.g. "2025-04-16 02:06:10.919Z" instead of "2025-04-16 02:06:10Z"). Updated valid_response.json (60 timestamps) and valid_equals_response.json (2 timestamps) to match. Co-Authored-By: Claude Sonnet 4.6 --- .../scylladb/http/valid_equals_response.json | 4 ++-- .../feast/integration_tests/scylladb/http/valid_response.json | 4 ++-- go/internal/feast/server/grpc_server.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json b/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json index 5f837738990..6b528dcd2e4 100644 --- a/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json +++ b/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json @@ -217,7 +217,7 @@ "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", - "2025-04-16 02:04:31Z", + "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] @@ -357,7 +357,7 @@ { "values": [ [ - "2025-04-16 02:06:11Z" + "2025-04-16 02:06:11.919Z" ] ] } diff --git a/go/internal/feast/integration_tests/scylladb/http/valid_response.json b/go/internal/feast/integration_tests/scylladb/http/valid_response.json index 55db0470247..5e02ff0b090 100644 --- a/go/internal/feast/integration_tests/scylladb/http/valid_response.json +++ b/go/internal/feast/integration_tests/scylladb/http/valid_response.json @@ -34,7 +34,7 @@ },{ "values" : [ [ [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ] ], [ [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ] ], [ [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ] ] ] }, { - "values" : [ [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ] ] + "values" : [ [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ] ] }, { "values" : [ [ [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ] ], [ [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ] ], [ [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ] ] ] }, { @@ -70,6 +70,6 @@ }, { "values" : [ [ null, null, null, null, null, null, null, null, null, null ], [ null, null, null, null, null, null, null, null, null, null ], [ null, null, null, null, null, null, null, null, null, null ] ] }, { - "values" : [ [ "2025-04-16 02:06:10Z", "2025-04-16 02:06:09Z", "2025-04-16 02:06:08Z", "2025-04-16 02:06:07Z", "2025-04-16 02:06:06Z", "2025-04-16 02:06:05Z", "2025-04-16 02:06:04Z", "2025-04-16 02:06:03Z", "2025-04-16 02:06:02Z", "2025-04-16 02:06:01Z" ], [ "2025-04-16 02:06:11Z", "2025-04-16 02:06:10Z", "2025-04-16 02:06:09Z", "2025-04-16 02:06:08Z", "2025-04-16 02:06:07Z", "2025-04-16 02:06:06Z", "2025-04-16 02:06:05Z", "2025-04-16 02:06:04Z", "2025-04-16 02:06:03Z", "2025-04-16 02:06:02Z" ], [ "2025-04-16 02:06:11Z", "2025-04-16 02:06:10Z", "2025-04-16 02:06:09Z", "2025-04-16 02:06:08Z", "2025-04-16 02:06:07Z", "2025-04-16 02:06:06Z", "2025-04-16 02:06:05Z", "2025-04-16 02:06:04Z", "2025-04-16 02:06:03Z", "2025-04-16 02:06:02Z" ] ] + "values" : [ [ "2025-04-16 02:06:10.919Z", "2025-04-16 02:06:09.919Z", "2025-04-16 02:06:08.919Z", "2025-04-16 02:06:07.919Z", "2025-04-16 02:06:06.919Z", "2025-04-16 02:06:05.919Z", "2025-04-16 02:06:04.919Z", "2025-04-16 02:06:03.919Z", "2025-04-16 02:06:02.919Z", "2025-04-16 02:06:01.919Z" ], [ "2025-04-16 02:06:11.919Z", "2025-04-16 02:06:10.919Z", "2025-04-16 02:06:09.919Z", "2025-04-16 02:06:08.919Z", "2025-04-16 02:06:07.919Z", "2025-04-16 02:06:06.919Z", "2025-04-16 02:06:05.919Z", "2025-04-16 02:06:04.919Z", "2025-04-16 02:06:03.919Z", "2025-04-16 02:06:02.919Z" ], [ "2025-04-16 02:06:11.919Z", "2025-04-16 02:06:10.919Z", "2025-04-16 02:06:09.919Z", "2025-04-16 02:06:08.919Z", "2025-04-16 02:06:07.919Z", "2025-04-16 02:06:06.919Z", "2025-04-16 02:06:05.919Z", "2025-04-16 02:06:04.919Z", "2025-04-16 02:06:03.919Z", "2025-04-16 02:06:02.919Z" ] ] } ] } \ No newline at end of file diff --git a/go/internal/feast/server/grpc_server.go b/go/internal/feast/server/grpc_server.go index 6ac2381c9fd..a95bd182909 100644 --- a/go/internal/feast/server/grpc_server.go +++ b/go/internal/feast/server/grpc_server.go @@ -232,7 +232,7 @@ func (s *grpcServingServiceServer) GetOnlineFeaturesRange(ctx context.Context, r for k, ts := range timestamps { timestampValues[k] = &prototypes.Value{ Val: &prototypes.Value_UnixTimestampVal{ - UnixTimestampVal: types.GetTimestampMillis(ts), + UnixTimestampVal: types.GetTimestampSeconds(ts), }, } } From 6bbe8d505892e07104df7e35ead8b5ec57278a27 Mon Sep 17 00:00:00 2001 From: piket Date: Fri, 17 Jul 2026 12:05:49 -0700 Subject: [PATCH 09/11] test: plug 8 coverage gaps for ms-precision sort key handling (EAPC-22316) Adds targeted tests across four layers to guard the millisecond-precision UNIX_TIMESTAMP sort key fix from silent regression: - go/types: threshold boundary (exact 1e11 semantics), zero/negative edge cases, ms-range list branch in ValueTypeToGoTypeTimestampAsString, and nil-safety of GetTimestampSeconds - cassandraonlinestore: resolveFeatureValue preserves time.Time unmodified; InterfaceToProtoValue encodes it as UnixMilli (not Unix seconds) - redisonlinestore: EventTimestamps in OnlineReadRange match the _ts field seconds value for each returned row - ScyllaDB integration: both sub-second regression tests now set IncludeMetadata=true and assert EventTimestamps are seconds-precision - Valkey integration: new sub-second fixture (regression_repo.py + sub_second_data.parquet) and TestGetOnlineFeaturesRangeValkey_SubSecondSortKey Co-Authored-By: Claude Sonnet 4.6 --- .../scylladb/scylladb_integration_test.go | 26 +++++- .../valkey/feature_repo/regression_repo.py | 47 ++++++++++ .../feature_repo/sub_second_data.parquet | Bin 0 -> 3428 bytes .../valkey/valkey_integration_test.go | 83 ++++++++++++++++++ .../onlinestore/cassandraonlinestore_test.go | 33 +++++++ .../onlinestore/redisonlinestore_test.go | 5 ++ go/types/typeconversion_test.go | 63 +++++++++++++ 7 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 go/internal/feast/integration_tests/valkey/feature_repo/regression_repo.py create mode 100644 go/internal/feast/integration_tests/valkey/feature_repo/sub_second_data.parquet diff --git a/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go b/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go index c2e7d9faf36..7d2d7170136 100644 --- a/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go +++ b/go/internal/feast/integration_tests/scylladb/scylladb_integration_test.go @@ -407,7 +407,8 @@ func TestGetOnlineFeaturesRange_SubSecondSortKeyValuesRemainDistinct(t *testing. }, }, }, - Limit: 10, + Limit: 10, + IncludeMetadata: true, } response, err := client.GetOnlineFeaturesRange(ctx, request) @@ -445,6 +446,16 @@ func TestGetOnlineFeaturesRange_SubSecondSortKeyValuesRemainDistinct(t *testing. for ts := range seen { assert.True(t, expected[ts], "unexpected sort key value %d", ts) } + + // EventTimestamps come from the _ts: hash field (a timestamppb.Timestamp). + // grpc_server calls GetTimestampSeconds, so values must be whole-second Unix timestamps. + require.Len(t, eventTimestampVector.EventTimestamps, 1, "expected EventTimestamps for 1 entity") + require.Len(t, eventTimestampVector.EventTimestamps[0].Val, 3, "expected EventTimestamp for each of the 3 rows") + for _, tsVal := range eventTimestampVector.EventTimestamps[0].Val { + secs := tsVal.GetUnixTimestampVal() + assert.True(t, secs > 1_000_000_000 && secs < 2_000_000_000, + "EventTimestamp must be seconds-precision (~2001–2033), got %d", secs) + } } // TestGetOnlineFeaturesRange_CustomNamedSortKeyValuesRemainDistinct guards @@ -483,7 +494,8 @@ func TestGetOnlineFeaturesRange_CustomNamedSortKeyValuesRemainDistinct(t *testin }, }, }, - Limit: 10, + Limit: 10, + IncludeMetadata: true, } response, err := client.GetOnlineFeaturesRange(ctx, request) @@ -521,6 +533,16 @@ func TestGetOnlineFeaturesRange_CustomNamedSortKeyValuesRemainDistinct(t *testin for ts := range seen { assert.True(t, expected[ts], "unexpected sort key value %d", ts) } + + // EventTimestamps come from the _ts: hash field (a timestamppb.Timestamp). + // grpc_server calls GetTimestampSeconds, so values must be whole-second Unix timestamps. + require.Len(t, viewedAtVector.EventTimestamps, 1, "expected EventTimestamps for 1 entity") + require.Len(t, viewedAtVector.EventTimestamps[0].Val, 3, "expected EventTimestamp for each of the 3 rows") + for _, tsVal := range viewedAtVector.EventTimestamps[0].Val { + secs := tsVal.GetUnixTimestampVal() + assert.True(t, secs > 1_000_000_000 && secs < 2_000_000_000, + "EventTimestamp must be seconds-precision (~2001–2033), got %d", secs) + } } func assertResponseData(t *testing.T, response *serving.GetOnlineFeaturesRangeResponse, featureNames []string, entitiesRequested int, includeMetadata bool) { diff --git a/go/internal/feast/integration_tests/valkey/feature_repo/regression_repo.py b/go/internal/feast/integration_tests/valkey/feature_repo/regression_repo.py new file mode 100644 index 00000000000..c5849ed3adc --- /dev/null +++ b/go/internal/feast/integration_tests/valkey/feature_repo/regression_repo.py @@ -0,0 +1,47 @@ +# Regression fixture for EAPC-22316: UNIX_TIMESTAMP sort keys must retain +# millisecond precision for rows that differ by less than one second. + +from datetime import timedelta + +from feast import Entity, Field, FileSource, SortedFeatureView +from feast.protos.feast.core.SortedFeatureView_pb2 import SortOrder +from feast.sort_key import SortKey +from feast.types import String, UnixTimestamp +from feast.value_type import ValueType + +tags = {"team": "Feast"} +owner = "test@test.com" + +sub_second_entity: Entity = Entity( + name="sub_second_entity", + join_keys=["sub_second_entity_id"], + value_type=ValueType.STRING, + tags=tags, + owner=owner, +) + +sub_second_source: FileSource = FileSource( + path="sub_second_data.parquet", timestamp_field="event_timestamp" +) + +sub_second_sort_key_view: SortedFeatureView = SortedFeatureView( + name="sub_second_sort_key_view", + entities=[sub_second_entity], + ttl=timedelta(days=0), + source=sub_second_source, + tags=tags, + description="Regression fixture: sort key rows <1s apart for the same entity", + owner=owner, + online=True, + sort_keys=[ + SortKey( + name="event_timestamp", + value_type=ValueType.UNIX_TIMESTAMP, + default_sort_order=SortOrder.DESC, + ) + ], + schema=[ + Field(name="value", dtype=String), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], +) diff --git a/go/internal/feast/integration_tests/valkey/feature_repo/sub_second_data.parquet b/go/internal/feast/integration_tests/valkey/feature_repo/sub_second_data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a10514da6c5c18962991ee6220fe2dbd747f427d GIT binary patch literal 3428 zcmb_fU2ofD6n3&U1gOL`)xnW+QAJw>Qd;s^)~QH1X_F=-q0Kfs@is;NO5)UZyx2~f zjY|6gz%^HhOD zUZEK}Qlb4cb&aCd;z5d{RMXL%@n({y=xcN+P6z0#@Cih=sds`?88cgf!Rs4-xCeZ` z00nqA>D>zGBVI&b1wmUI*iHZzwkr2c1&^GrHc$Y6on{vBSBZdGB_^Qv#&r+bf8{zt zC+N%+nz{evr{~WET1kryh^W6a}$j0 ze*5Kj84hARmlq~Dm7J~6!SX3MSjbiQewJbaw%k<>kq%Py+6_OmZo5*;%T&wDc1u%k zP?ZQ66NS~GsJrU)WCcv#Qiq_~ax_D=9nrA#Pp?IGK^+WG{F*#aMMqU`tXDt~CxTEQ z90^xqRD@m&XQ{x|@Wp;2K?dghD$xe|M}0-lzueMt{o?kV)kYA6s3hW=iq!f_^GiKZgj%#VS`QO#7; zla}1qUBk4aw>IuQjvm7qtfB+a?5NTBM${Ay6%I^S*I@^ZFmPa-AnU#b%QRW4xJ;ft zfQ^UOBH-!-b`hnmsk+iyJo9L^oUw)OqOPgQeZ^zo8rFCw=u16Sc95B&I-(*vBB(!( zswSvwW(RHW-YIO}jGoSsS!5Bt=v_d>1zwkG{U@fF$J00vok4$x!r?HtLj& zY**s%qK>QZCw3*zYI3^XmCYkKV@H$yP|laOOM6wjlrKM0irffss*R+A@@8tsR`{gb zNO``6#>}VN~g)r*xo9j$$i zyF3Q{f&=-R^1?iDS|eT4!M`gMc676bmTa#M^b9!vhz8RQssvNUrTuPO6p3n z02uY|EUyO%8)Kd0k!tcM$w|%(k>_mSKdVK_6uQtGoI01*ieW z3AJ-%hw zQG?z~5*{#LVJoHm}M2C0Q2iEtnwMuhah0nL7W4ew{RVu z8SNKHkY4Cyi3&K1;r2+m?ENQ7IN#@>SoQ;o*|5aHLsxY+Wy{(mUqTy`FPRPR>nDsq Q&}V hash field (a timestamppb.Timestamp). + // grpc_server calls GetTimestampSeconds, so values must be whole-second Unix timestamps. + require.Len(t, eventTimestampVector.EventTimestamps, 1, "expected EventTimestamps for 1 entity") + require.Len(t, eventTimestampVector.EventTimestamps[0].Val, 3, "expected EventTimestamp for each of the 3 rows") + for _, tsVal := range eventTimestampVector.EventTimestamps[0].Val { + secs := tsVal.GetUnixTimestampVal() + assert.True(t, secs > 1_000_000_000 && secs < 2_000_000_000, + "EventTimestamp must be seconds-precision (~2001–2033), got %d", secs) + } +} diff --git a/go/internal/feast/onlinestore/cassandraonlinestore_test.go b/go/internal/feast/onlinestore/cassandraonlinestore_test.go index 89c3722d6f1..c840d4b15a2 100644 --- a/go/internal/feast/onlinestore/cassandraonlinestore_test.go +++ b/go/internal/feast/onlinestore/cassandraonlinestore_test.go @@ -7,7 +7,9 @@ import ( "fmt" "reflect" "testing" + "time" + featuretypes "github.com/feast-dev/feast/go/types" "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/internal/feast/utils" @@ -530,6 +532,37 @@ func TestResolveFeatureValue_SortKeyMissingFromRow(t *testing.T) { assert.Nil(t, val) } +func TestResolveFeatureValue_SortKey_TimeTime(t *testing.T) { + // gocql surfaces Cassandra/Scylla native timestamp columns as time.Time. + // resolveFeatureValue returns the raw time.Time for sort keys; the serving + // layer then calls InterfaceToProtoValue which uses .UnixMilli() — the + // ms-precision encode path added in EAPC-22316. This test verifies both: + // (1) resolveFeatureValue preserves the time.Time without truncation, and + // (2) InterfaceToProtoValue encodes it as milliseconds, not whole seconds. + msVal := int64(1717244257886) // 2024-06-01 12:17:37.886Z + tVal := time.UnixMilli(msVal).UTC() + readValues := map[string]interface{}{ + "eventsortkey": tVal, + } + + val, status, err := resolveFeatureValue(readValues, canonicalColumnName("EventSortKey"), true) + assert.NoError(t, err) + assert.Equal(t, serving.FieldStatus_PRESENT, status) + require.NotNil(t, val) + + // resolveFeatureValue returns the raw interface{} for sort keys. + // Assert it's the exact time.Time we put in (no truncation). + gotTime, ok := val.(time.Time) + require.True(t, ok, "sort key value should be returned as time.Time") + assert.Equal(t, tVal, gotTime, "time.Time must be returned unmodified by resolveFeatureValue") + + // InterfaceToProtoValue (called by the serving layer) must encode it as ms. + protoVal, err := featuretypes.InterfaceToProtoValue(val) + require.NoError(t, err) + assert.Equal(t, msVal, protoVal.GetUnixTimestampVal(), + "time.Time sort key must be encoded as UnixMilli, not Unix seconds") +} + func TestResolveFeatureValue_AlreadyLowercaseFeature(t *testing.T) { testVal := &types.Value{Val: &types.Value_StringVal{StringVal: "hello"}} serialized := mustMarshalValueProto(t, testVal) diff --git a/go/internal/feast/onlinestore/redisonlinestore_test.go b/go/internal/feast/onlinestore/redisonlinestore_test.go index 8c3f031a2f2..263deb2a9a9 100644 --- a/go/internal/feast/onlinestore/redisonlinestore_test.go +++ b/go/internal/feast/onlinestore/redisonlinestore_test.go @@ -479,6 +479,11 @@ func TestOnlineReadRange(t *testing.T) { require.Len(t, out.Values, 2) assert.Equal(t, "one", out.Values[0].(*types.Value).GetStringVal()) assert.Equal(t, "two", out.Values[1].(*types.Value).GetStringVal()) + + // EventTimestamps come from the _ts: hash field decoded as timestamppb.Timestamp. + require.Len(t, out.EventTimestamps, 2, "each returned row should have an event timestamp") + assert.Equal(t, int64(1000), out.EventTimestamps[0].Seconds, "row 0 event ts must match ts1") + assert.Equal(t, int64(2000), out.EventTimestamps[1].Seconds, "row 1 event ts must match ts2") } func TestOnlineReadRange_Reverse(t *testing.T) { diff --git a/go/types/typeconversion_test.go b/go/types/typeconversion_test.go index 0e97e6f0b8d..264a336f674 100644 --- a/go/types/typeconversion_test.go +++ b/go/types/typeconversion_test.go @@ -11,6 +11,7 @@ import ( "github.com/apache/arrow/go/v17/arrow/memory" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" "github.com/feast-dev/feast/go/protos/feast/types" @@ -439,6 +440,43 @@ func TestGetTimestampMillis(t *testing.T) { "expected millisecond precision to be retained") } +func TestGetTimestampSeconds_Nil(t *testing.T) { + assert.Equal(t, int64(0), GetTimestampSeconds(nil), "nil timestamp should return 0") + + ts := ×tamppb.Timestamp{Seconds: 1783678657, Nanos: 886_000_000} + assert.Equal(t, int64(1783678657), GetTimestampSeconds(ts), + "GetTimestampSeconds must discard sub-second nanos and return only whole seconds") +} + +func TestUnixTsToTime_Boundary(t *testing.T) { + // The threshold is val > 1e11 (100_000_000_000). + // Values at or below the threshold are seconds; values above are milliseconds. + // This test pins the exact boundary so a change from > to >= (or a different + // constant) is immediately caught. + const threshold = int64(100_000_000_000) // 1e11 + + below := unixTsToTime(threshold - 1) + assert.Equal(t, time.Unix(threshold-1, 0).UTC(), below, + "val == 1e11-1 must be treated as seconds") + + exact := unixTsToTime(threshold) + assert.Equal(t, time.Unix(threshold, 0).UTC(), exact, + "val == 1e11 must be treated as seconds (condition is >, not >=)") + + above := unixTsToTime(threshold + 1) + assert.Equal(t, time.UnixMilli(threshold+1).UTC(), above, + "val == 1e11+1 must be treated as milliseconds") +} + +func TestUnixTsToTime_ZeroAndNegative(t *testing.T) { + assert.Equal(t, time.Unix(0, 0).UTC(), unixTsToTime(0), + "zero must map to Unix epoch via seconds branch") + assert.Equal(t, time.Unix(-1, 0).UTC(), unixTsToTime(-1), + "negative values must take seconds branch (all negatives < 1e11)") + assert.Equal(t, time.Unix(math.MinInt64, 0).UTC(), unixTsToTime(math.MinInt64), + "MinInt64 must take seconds branch without panic") +} + func TestValueTypeToGoType(t *testing.T) { timestamp := time.Unix(1744769099, 0).UTC() testCases := []*types.Value{ @@ -522,6 +560,31 @@ func TestValueTypeToGoTypeTimestampAsString(t *testing.T) { } } +func TestValueTypeToGoTypeTimestampAsString_MsRangeList(t *testing.T) { + // These are millisecond-precision sort key values (> 1e11); the list branch of + // unixTsToTime must take the ms path, not the seconds path, for each element. + ms1 := int64(1717244257886) // 2024-06-01 12:17:37.886Z + ms2 := int64(1717244257904) // 2024-06-01 12:17:37.904Z + + val := &types.Value{Val: &types.Value_UnixTimestampListVal{ + UnixTimestampListVal: &types.Int64List{Val: []int64{ms1, ms2}}, + }} + + result := ValueTypeToGoTypeTimestampAsString(val) + strs, ok := result.([]string) + require.True(t, ok, "expected []string from UnixTimestampListVal") + require.Len(t, strs, 2) + + assert.Equal(t, time.UnixMilli(ms1).UTC().Format(TimestampFormat), strs[0], + "ms-range list element 0 must use UnixMilli, not Unix") + assert.Equal(t, time.UnixMilli(ms2).UTC().Format(TimestampFormat), strs[1], + "ms-range list element 1 must use UnixMilli, not Unix") + + // Confirm both strings carry sub-second precision (the ".886" / ".904" suffix). + assert.Contains(t, strs[0], ".886", "expected sub-second suffix .886 in formatted timestamp") + assert.Contains(t, strs[1], ".904", "expected sub-second suffix .904 in formatted timestamp") +} + func TestConvertToValueType_String(t *testing.T) { testCases := []struct { input *types.Value From bebb1c1907d88c5135a24b5d2afce4462ab5b943 Mon Sep 17 00:00:00 2001 From: piket Date: Fri, 17 Jul 2026 13:19:42 -0700 Subject: [PATCH 10/11] fix: revert incorrect .919Z on array_timestamp_val in HTTP fixtures The previous fixture update applied .919Z sub-second suffix to ALL "2025-04-16 02:04:31Z" occurrences. That was only correct for the event_timestamp sort key feature values (02:06:xx.919Z, which are stored as ms). The array_timestamp_val element for 2025-04-16 is stored as whole seconds (1744769071), so it formats as "02:04:31Z" without sub-second suffix. Revert the 30 wrong entries in valid_response.json and 1 in valid_equals_response.json. Co-Authored-By: Claude Sonnet 4.6 --- .../integration_tests/scylladb/http/valid_equals_response.json | 2 +- .../feast/integration_tests/scylladb/http/valid_response.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json b/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json index 6b528dcd2e4..be7664a120c 100644 --- a/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json +++ b/go/internal/feast/integration_tests/scylladb/http/valid_equals_response.json @@ -217,7 +217,7 @@ "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", - "2025-04-16 02:04:31.919Z", + "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] diff --git a/go/internal/feast/integration_tests/scylladb/http/valid_response.json b/go/internal/feast/integration_tests/scylladb/http/valid_response.json index 5e02ff0b090..5170162c712 100644 --- a/go/internal/feast/integration_tests/scylladb/http/valid_response.json +++ b/go/internal/feast/integration_tests/scylladb/http/valid_response.json @@ -34,7 +34,7 @@ },{ "values" : [ [ [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ] ], [ [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ] ], [ [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ], [ "9QECZ", "GLPQJ", "RWCS4", "4J2ZQ", "S2ZJG", "TLS1U", "J3ZNM", "CPILQ", "9QPKL", "ZJELB" ] ] ] }, { - "values" : [ [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31.919Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ] ] + "values" : [ [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ], [ [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ], [ "2024-06-05 02:04:31Z", "2024-09-20 02:04:31Z", "2024-10-16 02:04:31Z", "2024-05-26 02:04:31Z", "2024-08-21 02:04:31Z", "2025-01-11 03:04:31Z", "2025-01-12 03:04:31Z", "2025-04-16 02:04:31Z", "2024-10-21 02:04:31Z", "2025-04-07 02:04:31Z" ] ] ] }, { "values" : [ [ [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ] ], [ [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ] ], [ [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ], [ false, true, false, true, false, true, false, false, true, false ] ] ] }, { From 2259602225bc9b1270f99339f4ecde4eccec0b57 Mon Sep 17 00:00:00 2001 From: piket Date: Mon, 20 Jul 2026 11:33:12 -0700 Subject: [PATCH 11/11] fix: use np.issubdtype for datetime64 dtype check (EAPC-22316) isinstance(values.dtype, np.datetime64) is always False since dtype is a numpy.dtype instance, never the np.datetime64 scalar type - so the vectorized fast path never ran for ndarray inputs in either the seconds or ms timestamp conversion helpers. Flagged in PR #369 review. Co-Authored-By: Claude Opus 4.8 --- sdk/python/feast/type_map.py | 4 +- sdk/python/tests/unit/test_type_map.py | 54 ++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index df08e860208..e308d7102fa 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -351,7 +351,7 @@ def _python_datetime_to_int_timestamp( values: Sequence[Any], ) -> Sequence[Union[int, np.int_]]: # Fast path for Numpy array. - if isinstance(values, np.ndarray) and isinstance(values.dtype, np.datetime64): + if isinstance(values, np.ndarray) and np.issubdtype(values.dtype, np.datetime64): if values.ndim != 1: raise ValueError("Only 1 dimensional arrays are supported.") return cast(Sequence[np.int_], values.astype("datetime64[s]").astype(np.int_)) @@ -376,7 +376,7 @@ def _python_datetime_to_int_ms_timestamp( ) -> Sequence[Union[int, np.int_]]: """Convert datetime values to milliseconds since epoch (used for sort key columns).""" # Fast path for Numpy array. - if isinstance(values, np.ndarray) and isinstance(values.dtype, np.datetime64): + if isinstance(values, np.ndarray) and np.issubdtype(values.dtype, np.datetime64): if values.ndim != 1: raise ValueError("Only 1 dimensional arrays are supported.") return cast(Sequence[np.int_], values.astype("datetime64[ms]").astype(np.int_)) diff --git a/sdk/python/tests/unit/test_type_map.py b/sdk/python/tests/unit/test_type_map.py index 55406cdafe3..967c8baee40 100644 --- a/sdk/python/tests/unit/test_type_map.py +++ b/sdk/python/tests/unit/test_type_map.py @@ -5,6 +5,7 @@ from feast.type_map import ( MS_TIMESTAMP_THRESHOLD, _python_datetime_to_int_ms_timestamp, + _python_datetime_to_int_timestamp, feast_value_type_to_python_type, python_values_to_proto_values, ) @@ -58,6 +59,59 @@ def test_python_datetime_to_int_ms_timestamp_raw_int_above_threshold_passes_thro assert result == [ms_value] +def test_ms_timestamp_ndarray_fast_path_preserves_millis(): + """A 1-D datetime64 ndarray should take the vectorized fast path (guarded by + np.issubdtype, not isinstance - values.dtype is a numpy.dtype instance and + is never an instance of the np.datetime64 scalar type) and retain + millisecond precision.""" + + arr = np.array(["2024-06-01T12:17:37.886"], dtype="datetime64[ns]") + + result = _python_datetime_to_int_ms_timestamp(arr) + + # The fast path returns an ndarray; the scalar fallback loop returns a + # plain list. Asserting the type pins down that the fast path actually ran. + assert isinstance(result, np.ndarray) + assert result.tolist() == [1717244257886] + + +def test_ms_timestamp_ndarray_matches_scalar_loop(): + """The ndarray fast path and the scalar fallback loop must agree.""" + + arr = np.array(["2024-06-01T12:17:37.886"], dtype="datetime64[ns]") + + fast_path_result = _python_datetime_to_int_ms_timestamp(arr) + scalar_loop_result = _python_datetime_to_int_ms_timestamp(list(arr)) + + assert list(fast_path_result) == list(scalar_loop_result) + + +def test_seconds_timestamp_ndarray_fast_path(): + """Same fast-path bug (isinstance vs np.issubdtype) also existed in the + pre-existing seconds-precision function; verify it takes the vectorized + path and truncates to whole seconds.""" + + arr = np.array(["2024-06-01T12:17:37.886"], dtype="datetime64[ns]") + + result = _python_datetime_to_int_timestamp(arr) + + assert isinstance(result, np.ndarray) + assert result.tolist() == [1717244257] + + +def test_ndarray_non_1d_raises(): + """The ndim check is only reachable once the dtype fast-path guard + actually matches datetime64 ndarrays.""" + + arr = np.array([["2024-06-01"]], dtype="datetime64[ns]") + + with pytest.raises(ValueError, match="Only 1 dimensional arrays are supported."): + _python_datetime_to_int_ms_timestamp(arr) + + with pytest.raises(ValueError, match="Only 1 dimensional arrays are supported."): + _python_datetime_to_int_timestamp(arr) + + @pytest.mark.parametrize( "values", (