From 0a077e04a1440173b6ff2c370d2e034c77bc847d Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 7 Jul 2026 12:14:33 -0700 Subject: [PATCH] Detect BIGINT overflow in SUM and fix wrong AVG on the Calcite engine SUM and AVG over a BIGINT (long) column near 2^63 silently produced wrong results on the Calcite engine: - SUM(long): the enumerable accumulator is a plain long, so a running sum past 2^63 wrapped to a negative value (e.g. SUM(UserID) returned -5594372458244005145 instead of erroring). - AVG(long): AVG reduces to SUM(field)/COUNT(field); the intermediate long SUM wrapped the same way, so AVG returned a wrong (often negative) result. Fix, applied in the SQL-plugin lowering so both the DSL-pushdown and the in-memory (no-pushdown) paths behave identically: - SUM(long): the aggregate argument (a bare BIGINT column) is summed in DECIMAL so it cannot wrap, and the result is narrowed back to BIGINT with CHECKED_LONG_NARROW. Narrowing errors (HTTP 4xx) when the sum clearly overflowed 2^63, and otherwise saturates. The output type stays bigint. - AVG(long): the bare BIGINT argument is averaged in DOUBLE, which holds the true average without an intermediate long wrap. The DOUBLE output type is unchanged. Only bare BIGINT column references are widened; expressions such as sum(field + 1) are left untouched so PPLAggregateConvertRule can still rewrite them to pushdown-friendly form. Narrower integer sums (byte/short/int) already widen to a BIGINT accumulator and cannot overflow, so they are unchanged. Boundary note: OpenSearch computes pushed-down sums in double, and (double) Long.MAX_VALUE rounds to exactly 2^63, indistinguishable from a small overflow. To avoid erroring on a legitimate near-Long.MAX_VALUE sum, only magnitudes strictly beyond 2^63 are treated as overflow; a genuine overflow lands far outside that boundary, and the in-memory path detects it exactly. AVG pushdown note: casting the AVG argument to DOUBLE keeps native avg pushdown in most cases, but an AVG with a preceding sort can fall back to an in-memory sum/count reduction instead of a pushed-down native avg. The result stays correct; only that narrow case loses aggregate pushdown. Adds CalcitePPLAggregationIT coverage (overflow -> 4xx, avg correct, in-range and Long.MAX sums), a REST yaml test (issues/5164_agg.yml), and regenerates the affected explain fixtures. Signed-off-by: Kai Huang --- .../sql/calcite/CalciteRelNodeVisitor.java | 61 +++++++++ .../function/PPLBuiltinOperators.java | 3 + .../expression/function/PPLFuncImpTable.java | 67 +++++++++- .../udf/CheckedLongNarrowFunction.java | 78 +++++++++++ docs/user/ppl/functions/expressions.md | 10 +- .../remote/CalcitePPLAggregationIT.java | 68 ++++++++++ .../agg_case_composite_cannot_push.yaml | 4 +- .../agg_composite2_range_count_push.yaml | 4 +- ...agg_composite2_range_range_count_push.yaml | 4 +- .../agg_composite_range_metric_push.yaml | 4 +- .../agg_range_metric_complex_push.yaml | 4 +- .../calcite/agg_range_range_metric_push.yaml | 4 +- .../range_auto_date_histo_with_metrics.yaml | 6 +- .../calcite/chart_multiple_group_keys.yaml | 8 +- .../calcite/chart_null_str.yaml | 20 +-- .../calcite/chart_single_group_key.yaml | 4 +- .../calcite/chart_with_limit.yaml | 4 +- .../expectedOutput/calcite/clickbench/q4.yaml | 4 +- .../explain_agg_paginating_having3.yaml | 4 +- .../calcite/explain_agg_sort_on_measure2.yaml | 8 +- .../calcite/explain_agg_sort_on_measure4.yaml | 11 +- .../explain_agg_sort_on_measure_complex1.yaml | 9 +- .../explain_agg_sort_on_measure_complex2.yaml | 9 +- ...t_on_measure_multi_buckets_not_pushed.yaml | 9 +- .../explain_agg_with_sum_enhancement.yaml | 8 +- .../calcite/explain_bin_minspan.json | 7 +- .../calcite/explain_filter_agg_push.yaml | 4 +- ...gg_with_sort_on_one_measure_not_push1.yaml | 7 +- ...gg_with_sort_on_one_measure_not_push2.yaml | 7 +- .../calcite/explain_nested_agg_push.yaml | 4 +- .../calcite/explain_output.yaml | 4 +- .../calcite/explain_output_cost.json | 4 +- .../calcite/explain_output_cost.yaml | 38 +++--- .../calcite/explain_output_extended.json | 4 +- .../calcite/explain_output_extended.yaml | 8 +- .../calcite/explain_output_standard.json | 4 +- .../calcite/explain_output_standard.yaml | 4 +- .../calcite/explain_sort_agg_push.json | 4 +- .../calcite/explain_sort_then_agg_push.json | 4 +- .../calcite/explain_streamstats_global.yaml | 28 ++-- ...xplain_streamstats_global_null_bucket.yaml | 32 ++--- .../calcite/explain_streamstats_reset.yaml | 27 ++-- ...explain_streamstats_reset_null_bucket.yaml | 27 ++-- .../agg_case_composite_cannot_push.yaml | 6 +- .../agg_composite2_range_count_push.yaml | 6 +- ...agg_composite2_range_range_count_push.yaml | 6 +- .../agg_composite_range_metric_push.yaml | 6 +- .../agg_range_metric_complex_push.yaml | 6 +- .../agg_range_range_metric_push.yaml | 8 +- .../chart_multiple_group_keys.yaml | 16 +-- .../calcite_no_pushdown/chart_null_str.yaml | 20 +-- .../chart_single_group_key.yaml | 8 +- .../calcite_no_pushdown/chart_with_limit.yaml | 8 +- .../explain_agg_with_sum_enhancement.yaml | 10 +- .../explain_bin_minspan.json | 7 +- .../explain_filter_agg_push.yaml | 8 +- .../calcite_no_pushdown/explain_output.yaml | 8 +- .../explain_sort_agg_push.json | 4 +- .../explain_sort_then_agg_push.json | 4 +- .../explain_streamstats_global.yaml | 26 ++-- ...xplain_streamstats_global_null_bucket.yaml | 30 +++-- .../explain_streamstats_reset.yaml | 27 ++-- ...explain_streamstats_reset_null_bucket.yaml | 27 ++-- .../rest-api-spec/test/issues/5164_agg.yml | 123 ++++++++++++++++++ 64 files changed, 715 insertions(+), 281 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/CheckedLongNarrowFunction.java create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 0df3c571c6b..9e643a42192 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -189,6 +189,7 @@ import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.executor.OpenSearchTypeSystem; import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLBuiltinOperators; @@ -1772,6 +1773,12 @@ private void visitAggregation( // Add aggregation results first List aggRexList = outputFields.subList(numOfOutputFields - numOfAggList, numOfOutputFields); + // SUM(bigint) accumulates in DECIMAL (OpenSearchTypeSystem#deriveSumType) so a running sum near + // 2^63 cannot silently wrap to a negative long. Cast the user-visible result back to BIGINT: + // the cast errors (ArithmeticException -> 4xx) on genuine overflow while keeping the output + // type bigint. AVG is unaffected (its output stays DOUBLE and its reduced internal SUM is not + // surfaced here). + aggRexList = castBigintSumResults(context, aggExprList, aggRexList); List aliasedGroupByList = aggregationAttributes.getLeft().stream() .map(this::extractAliasLiteral) @@ -1821,6 +1828,60 @@ private static AggregateFunction extractAggregateFunction(UnresolvedExpression e return null; } + /** + * Narrow the results of {@code SUM(bigint)} aggregations back to BIGINT. {@code SUM} over a + * BIGINT column is accumulated in a wider type (its argument is cast to DECIMAL in {@code + * PPLFuncImpTable}) so a running sum near 2^63 cannot silently wrap to a negative long. This + * restores the declared BIGINT output type via {@link PPLBuiltinOperators#CHECKED_LONG_NARROW}, + * which errors on a genuine overflow (surfaced as a 4xx) while saturating a value that merely + * rounds to 2^63 on the double-based pushdown path, so both execution paths behave identically. + * Non-SUM aggregations and sums whose accumulator is not the widened DECIMAL (e.g. {@code + * SUM(double)}, user {@code SUM(decimal)}) are returned unchanged. + */ + private static List castBigintSumResults( + CalcitePlanContext context, + List aggExprList, + List aggRexList) { + if (aggExprList.size() != aggRexList.size()) { + return aggRexList; + } + List result = new ArrayList<>(aggRexList.size()); + for (int i = 0; i < aggRexList.size(); i++) { + RexNode aggRex = aggRexList.get(i); + AggregateFunction aggFunc = extractAggregateFunction(aggExprList.get(i)); + if (aggFunc != null + && BuiltinFunctionName.ofAggregation(aggFunc.getFuncName()) + .filter(name -> name == BuiltinFunctionName.SUM) + .isPresent() + && isWidenedBigintSumType(aggRex.getType())) { + RexNode narrowed = + context.rexBuilder.makeCall(PPLBuiltinOperators.CHECKED_LONG_NARROW, aggRex); + // Wrapping in the narrowing UDF drops the aggregate's output name; restore it so downstream + // projection and the result schema keep the original "sum(...)" column name. + result.add( + aggRex instanceof RexInputRef ref + ? context.relBuilder.alias( + narrowed, + context.relBuilder.peek().getRowType().getFieldNames().get(ref.getIndex())) + : narrowed); + } else { + result.add(aggRex); + } + } + return result; + } + + /** + * Whether the type is the DECIMAL accumulator produced for {@code SUM(bigint)} (a scale-0 decimal + * at max precision, from the argument widening in {@code PPLFuncImpTable}), as opposed to a + * genuine user-supplied {@code SUM(decimal)} with a fractional scale. + */ + private static boolean isWidenedBigintSumType(RelDataType type) { + return type.getSqlTypeName() == SqlTypeName.DECIMAL + && type.getScale() == 0 + && type.getPrecision() == OpenSearchTypeSystem.MAX_PRECISION; + } + private static Function extractFunction(UnresolvedExpression expr) { if (expr instanceof Function f) return f; if (expr instanceof Alias alias) return extractFunction(alias.getDelegated()); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index d64f04bb9ad..de1fb1f6323 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -69,6 +69,7 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; import org.opensearch.sql.expression.function.udf.AutoConvertFunction; import org.opensearch.sql.expression.function.udf.CTimeConvertFunction; +import org.opensearch.sql.expression.function.udf.CheckedLongNarrowFunction; import org.opensearch.sql.expression.function.udf.CryptographicFunction; import org.opensearch.sql.expression.function.udf.Dur2SecConvertFunction; import org.opensearch.sql.expression.function.udf.MemkConvertFunction; @@ -448,6 +449,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); + public static final SqlOperator CHECKED_LONG_NARROW = + new CheckedLongNarrowFunction().toUDF("CHECKED_LONG_NARROW"); // PPL Convert command functions public static final SqlOperator AUTO = new AutoConvertFunction().toUDF("AUTO"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 151c4a96655..78b28796cf5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -291,6 +291,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; @@ -1609,10 +1610,68 @@ void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFuncti register(functionName, handler, typeChecker); } + /** + * Registers {@code SUM} with the extra behaviour that a BIGINT (long) column is summed in + * DECIMAL rather than long. A running long sum near 2^63 silently wraps to a negative value + * (the enumerable {@code SumImplementor} generates a plain {@code long + long}); accumulating + * in DECIMAL is exact. {@link CalciteRelNodeVisitor} casts the DECIMAL result back to BIGINT so + * the output type is unchanged and a genuine overflow surfaces as a client error instead of a + * wrong value. + * + *

Only a bare BIGINT column reference is widened. Expressions like {@code sum(field + 1)} + * are left as-is so {@link org.opensearch.sql.calcite.plan.rule.PPLAggregateConvertRule} can + * still rewrite them to pushdown-friendly {@code sum(field) OP literal} form. + */ + void registerSumOperator() { + SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(SqlStdOperatorTable.SUM); + PPLTypeChecker typeChecker = wrapSqlOperandTypeChecker(innerTypeChecker, SUM.name(), true); + AggHandler handler = + (distinct, field, argList, ctx) -> { + List newArgList = + argList.stream().map(PlanUtils::derefMapCall).collect(Collectors.toList()); + RexNode sumField = widenBigintColumnToDecimal(field, ctx); + return UserDefinedFunctionUtils.makeAggregateCall( + SqlStdOperatorTable.SUM, List.of(sumField), newArgList, ctx.relBuilder); + }; + register(SUM, handler, typeChecker); + } + + /** + * If {@code field} is a bare BIGINT column reference, cast it to DECIMAL so the aggregate that + * consumes it accumulates exactly instead of wrapping a long. Any other expression is returned + * unchanged. + */ + private static RexNode widenBigintColumnToDecimal(RexNode field, CalcitePlanContext ctx) { + if (field instanceof RexInputRef && field.getType().getSqlTypeName() == SqlTypeName.BIGINT) { + int maxPrecision = TYPE_FACTORY.getTypeSystem().getMaxNumericPrecision(); + RelDataType decimalType = + TYPE_FACTORY.createTypeWithNullability( + TYPE_FACTORY.createSqlType(SqlTypeName.DECIMAL, maxPrecision, 0), + field.getType().isNullable()); + return ctx.rexBuilder.makeCast(decimalType, field); + } + return field; + } + + /** + * If {@code field} is a bare BIGINT column reference, cast it to DOUBLE so that AVG's reduced + * intermediate SUM accumulates in double rather than a long that wraps near 2^63. Any other + * expression is returned unchanged. + */ + private static RexNode widenBigintColumnToDouble(RexNode field, CalcitePlanContext ctx) { + if (field instanceof RexInputRef && field.getType().getSqlTypeName() == SqlTypeName.BIGINT) { + RelDataType doubleType = + TYPE_FACTORY.createTypeWithNullability( + TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE), field.getType().isNullable()); + return ctx.rexBuilder.makeCast(doubleType, field); + } + return field; + } + void populate() { registerOperator(MAX, SqlStdOperatorTable.MAX); registerOperator(MIN, SqlStdOperatorTable.MIN); - registerOperator(SUM, SqlStdOperatorTable.SUM); + registerSumOperator(); registerOperator(VARSAMP, PPLBuiltinOperators.VAR_SAMP_NULLABLE); registerOperator(VARPOP, PPLBuiltinOperators.VAR_POP_NULLABLE); registerOperator(STDDEV_SAMP, PPLBuiltinOperators.STDDEV_SAMP_NULLABLE); @@ -1631,7 +1690,11 @@ void populate() { register( AVG, - (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field), + (distinct, field, argList, ctx) -> + // AVG reduces to SUM(field)/COUNT(field); over a bare BIGINT column the intermediate + // long SUM wraps near 2^63 (returning a wrong negative average). Averaging in DOUBLE + // avoids the wrap and keeps the DOUBLE output type unchanged. + ctx.relBuilder.avg(distinct, null, widenBigintColumnToDouble(field, ctx)), wrapSqlOperandTypeChecker( SqlStdOperatorTable.AVG.getOperandTypeChecker(), AVG.name(), false)); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CheckedLongNarrowFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CheckedLongNarrowFunction.java new file mode 100644 index 00000000000..fe86f2e4a65 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CheckedLongNarrowFunction.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.math.BigDecimal; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * Narrows a numeric SUM accumulator (widened to DECIMAL/double so it does not wrap) back to BIGINT, + * raising an error when the value overflowed the BIGINT range instead of wrapping to a negative + * value. + * + *

SUM over a BIGINT column is accumulated as a wider type to avoid the silent {@code long} + * two's-complement wrap near 2^63. This function restores the declared BIGINT output type. A value + * whose magnitude clearly exceeds {@code 2^63} is treated as overflow and surfaces as a client + * error. Because OpenSearch computes pushed-down sums in {@code double}, {@code (double) + * Long.MAX_VALUE} rounds to exactly {@code 2^63} and cannot be distinguished from a small overflow; + * to avoid erroring on a legitimate near-{@code Long.MAX_VALUE} sum, only magnitudes strictly + * beyond {@code 2^63} are rejected, and in-range values saturate on narrowing rather than wrap. + * This makes the pushdown and in-memory paths behave identically. + */ +public class CheckedLongNarrowFunction extends ImplementorUDF { + public CheckedLongNarrowFunction() { + super(new CheckedLongNarrowImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.BIGINT_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.NUMERIC; + } + + /** {@code 2^63}, the first magnitude beyond the signed BIGINT range. */ + private static final double TWO_POW_63 = 0x1p63; + + /** Narrow a widened sum value to a long, erroring when it overflowed the BIGINT range. */ + public static long narrow(Object value) { + double magnitude = ((Number) value).doubleValue(); + if (magnitude > TWO_POW_63 || magnitude < -TWO_POW_63) { + throw new ArithmeticException("BIGINT overflow in SUM"); + } + if (value instanceof BigDecimal decimal) { + // Exact for the in-memory (DECIMAL) path; clamps a value that rounds to exactly 2^63. + return decimal + .min(BigDecimal.valueOf(Long.MAX_VALUE)) + .max(BigDecimal.valueOf(Long.MIN_VALUE)) + .longValue(); + } + // double path (pushed-down sum): saturating narrow (JLS 5.1.3) clamps 2^63 to Long.MAX_VALUE. + return (long) magnitude; + } + + public static class CheckedLongNarrowImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call(CheckedLongNarrowFunction.class, "narrow", translatedOperands.get(0)); + } + } +} diff --git a/docs/user/ppl/functions/expressions.md b/docs/user/ppl/functions/expressions.md index 427a0334b58..6472c825793 100644 --- a/docs/user/ppl/functions/expressions.md +++ b/docs/user/ppl/functions/expressions.md @@ -13,7 +13,14 @@ Arithmetic expressions are formed by combining numeric literals and binary arith ### Overflow behavior -Integer and long arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. For example, `eval x = int_field + 1` where `int_field` is `2147483647` (integer max) returns an error rather than `-2147483648`. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. +Long (`BIGINT`) arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. Narrower integer operands are widened before arithmetic, so crossing the 32-bit integer boundary does not overflow. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. + +The accumulator used by `stats sum(bigint_field)` depends on whether Calcite pushdown is enabled: + +- With `plugins.calcite.pushdown.enabled=true` (the default), OpenSearch computes the sum as a `double` before the result is checked and narrowed to `BIGINT`. Large in-range sums can therefore lose low-order precision. A small overflow that rounds to a signed `BIGINT` boundary can also be indistinguishable from an in-range sum and may saturate at the boundary instead of returning an error. +- With `plugins.calcite.pushdown.enabled=false`, Calcite accumulates the sum as an exact `DECIMAL`, then narrows it to `BIGINT`. The exact result is returned when it fits; otherwise, the query returns an overflow error. + +For example, summing `4611686018427387904` (`2^62`) and `1` returns the exact `4611686018427387905` without pushdown. With pushdown enabled, the `double` accumulator cannot represent the low-order `1`, so the result is `4611686018427387904`. ### Precedence @@ -189,4 +196,3 @@ fetched rows / total rows = 2/2 | 28 | +-----+ ``` - \ No newline at end of file diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a2ab93b6599..40c31af50c6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -20,12 +20,16 @@ import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; +import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; +import static org.opensearch.sql.util.TestUtils.isIndexExist; +import static org.opensearch.sql.util.TestUtils.performRequest; import java.io.IOException; import java.util.Arrays; import java.util.List; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -93,6 +97,70 @@ public void testSumAvg() throws IOException { verifyDataRows(actual, rows(186973)); } + @Test + public void testSumAvgLongOverflow() throws IOException { + String overflowIndex = "test_sum_long_overflow"; + if (!isIndexExist(client(), overflowIndex)) { + createIndexByRestClient( + client(), overflowIndex, "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"long\"}}}}"); + Request bulk = new Request("POST", "/" + overflowIndex + "/_bulk?refresh=true"); + bulk.setJsonEntity( + "{\"index\":{}}\n" + + "{\"v\":9223372036854775807}\n" + + "{\"index\":{}}\n" + + "{\"v\":9223372036854775807}\n" + + "{\"index\":{}}\n" + + "{\"v\":9223372036854775807}\n"); + performRequest(client(), bulk); + } + String inRangeIndex = "test_sum_long_in_range"; + if (!isIndexExist(client(), inRangeIndex)) { + createIndexByRestClient( + client(), inRangeIndex, "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"long\"}}}}"); + Request bulk = new Request("POST", "/" + inRangeIndex + "/_bulk?refresh=true"); + bulk.setJsonEntity( + "{\"index\":{}}\n" + + "{\"v\":1000000000000}\n" + + "{\"index\":{}}\n" + + "{\"v\":2000000000000}\n" + + "{\"index\":{}}\n" + + "{\"v\":3000000000000}\n"); + performRequest(client(), bulk); + } + String boundaryIndex = "test_sum_long_boundary"; + if (!isIndexExist(client(), boundaryIndex)) { + createIndexByRestClient( + client(), boundaryIndex, "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"long\"}}}}"); + Request bulk = new Request("POST", "/" + boundaryIndex + "/_bulk?refresh=true"); + bulk.setJsonEntity("{\"index\":{}}\n" + "{\"v\":9223372036854775807}\n"); + performRequest(client(), bulk); + } + + // SUM overflows the BIGINT range (3 * (2^63 - 1)); surfaced as a client error rather than + // silently wrapping to a negative value. + Throwable error = + assertThrowsWithReplace( + RuntimeException.class, + () -> executeQuery(String.format("source=%s | stats sum(v)", overflowIndex))); + verifyErrorMessageContains(error, "verflow"); + + // AVG is averaged in DOUBLE, so it holds the true average (the shared value) without wrapping. + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows(9.223372036854776e18)); + + // A sum well within the BIGINT range returns the exact value with no error. + JSONObject inRange = executeQuery(String.format("source=%s | stats sum(v)", inRangeIndex)); + verifySchema(inRange, schema("sum(v)", "bigint")); + verifyDataRows(inRange, rows(6000000000000L)); + + // A single Long.MAX_VALUE is a valid (non-overflowing) sum and must not error, even though the + // pushed-down double result rounds to exactly 2^63 — the narrow saturates instead of erroring. + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(9223372036854775807L)); + } + @Test public void testAsExistedField() throws IOException { JSONObject actual = diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml index 19cd319c173..7258b1fc35f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_balance=[$2], age_range=[$0], state=[$1]) LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) - LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,1],"DIGESTS":["age",35,"u35","email"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,1],"DIGESTS":["age",35,"u35","email"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml index 6d12ca7285e..fc219f5b554 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$3], count()=[$4], age_range=[$0], state=[$1], gender=[$2]) LogicalAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(balance)=AVG($3),count()=COUNT()), PROJECT->[avg(balance), count(), age_range, state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(balance)=AVG($3),count()=COUNT()), PROJECT->[avg(balance), count(), age_range, state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml index 4bbafd2928a..dc059c231a4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_balance=[$3], age_range=[$0], balance_range=[$1], state=[$2]) LogicalAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) - LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg_balance=AVG($3)), PROJECT->[avg_balance, age_range, balance_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u35","to":35.0},{"key":"a35","from":35.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg_balance=AVG($3)), PROJECT->[avg_balance, age_range, balance_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u35","to":35.0},{"key":"a35","from":35.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml index 605111dfc86..38649be18f4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$2], state=[$0], age_range=[$1]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance=[$7]) + LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml index f3d749487c0..5821910b1a9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_metric_complex_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$1], age_range=[$0]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"30-40 or >=80","from":35.0,"to":40.0},{"key":"30-40 or >=80","from":80.0},{"key":"null","from":30.0,"to":35.0},{"key":"null","from":40.0,"to":80.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), age_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"30-40 or >=80","from":35.0,"to":40.0},{"key":"30-40 or >=80","from":80.0},{"key":"null","from":30.0,"to":35.0},{"key":"null","from":40.0,"to":80.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml index 5b44ebfdc68..e1156c404e9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_range_range_metric_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_balance=[$2], age_range=[$0], balance_range=[$1]) LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, balance_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, balance_range]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"u40","from":30.0,"to":40.0},{"key":"u100","from":40.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_auto_date_histo_with_metrics.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_auto_date_histo_with_metrics.yaml index b29b215e612..63ce257fe0d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/range_auto_date_histo_with_metrics.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/range_auto_date_histo_with_metrics.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(tmin=[$2], tavg=[$3], tmax=[$4], range_bucket=[$0], @timestamp=[$1]) - LogicalAggregate(group=[{0, 1}], tmin=[MIN($2)], tavg=[AVG($3)], tmax=[MAX($3)]) - LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)], @timestamp=[WIDTH_BUCKET($17, 10, -(MAX($17) OVER (), MIN($17) OVER ()), MAX($17) OVER ())], metrics.tmin=[$29], metrics.size=[$28]) + LogicalAggregate(group=[{0, 1}], tmin=[MIN($2)], tavg=[AVG($4)], tmax=[MAX($3)]) + LogicalProject(range_bucket=[CASE(<($28, 100), 'range_1':VARCHAR, SEARCH($28, Sarg[[100..1000)]), 'range_2':VARCHAR, SEARCH($28, Sarg[[1000..2000)]), 'range_3':VARCHAR, >=($28, 2000), 'range_4':VARCHAR, null:NULL)], @timestamp=[WIDTH_BUCKET($17, 10, -(MAX($17) OVER (), MIN($17) OVER ()), MAX($17) OVER ())], metrics.tmin=[$29], metrics.size=[$28], metrics.size1=[CAST($28):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},tmin=MIN($2),tavg=AVG($3),tmax=MAX($3)), PROJECT->[tmin, tavg, tmax, range_bucket, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":10,"minimum_interval":null},"aggregations":{"tmin":{"min":{"field":"metrics.tmin"}},"tavg":{"avg":{"field":"metrics.size"}},"tmax":{"max":{"field":"metrics.size"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},tmin=MIN($2),tavg=AVG($4),tmax=MAX($3)), PROJECT->[tmin, tavg, tmax, range_bucket, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"range_bucket":{"range":{"field":"metrics.size","ranges":[{"key":"range_1","to":100.0},{"key":"range_2","from":100.0,"to":1000.0},{"key":"range_3","from":1000.0,"to":2000.0},{"key":"range_4","from":2000.0}],"keyed":true},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":10,"minimum_interval":null},"aggregations":{"tmin":{"min":{"field":"metrics.tmin"}},"tavg":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["metrics.size"]}}}},"tmax":{"max":{"field":"metrics.size"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml index 63c7ea798df..a7e19364311 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml @@ -7,7 +7,7 @@ calcite: LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) + LogicalProject(gender=[$4], age=[$10], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) @@ -15,7 +15,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) + LogicalProject(gender=[$4], age=[$10], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | @@ -26,10 +26,10 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 726eeedc429..6a9127c733d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -6,16 +6,16 @@ calcite: LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age0=[SPAN($5, 10, null:NULL)], balance0=[CAST($3):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age0=[SPAN($5, 10, null:NULL)], balance0=[CAST($3):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | @@ -25,15 +25,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t9]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], expr#6=[CAST($t0):DOUBLE], gender=[$t1], age0=[$t5], balance0=[$t6]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], expr#10=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t9], $condition=[$t10]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], expr#6=[CAST($t0):DOUBLE], gender=[$t1], age0=[$t5], balance0=[$t6]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml index c21e8c468e9..f67a59b8ac5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) + LogicalProject(gender=[$4], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml index de69b108758..075fd706be1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) + LogicalProject(state=[$9], gender=[$4], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"state","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"state","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml index 36063b37709..ff9e1d8107b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q4.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], avg(UserID)=[AVG($0)]) - LogicalProject(UserID=[$84]) + LogicalProject(UserID0=[CAST($84):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},avg(UserID)=AVG($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"avg(UserID)":{"avg":{"field":"UserID"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},avg(UserID)=AVG($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"avg(UserID)":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["UserID"]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml index 4f57bbf4cd4..e1bb9ae82f3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml @@ -4,9 +4,9 @@ calcite: LogicalFilter(condition=[OR(>($3, 1000), >($4, 1))]) LogicalProject(avg=[$1], cnt=[$2], state=[$0], new_avg=[+($1, 1000)], new_cnt=[+($2, 1)]) LogicalAggregate(group=[{0}], avg=[AVG($1)], cnt=[COUNT()]) - LogicalProject(state=[$7], balance=[$3]) + LogicalProject(state=[$7], balance0=[CAST($3):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1:BIGINT], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[1], expr#9=[>($t6, $t8)], expr#10=[OR($t7, $t9)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml index 9c41efa9139..05c6666fc04 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml @@ -2,10 +2,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) - LogicalProject(sum=[$1], state=[$0]) + LogicalProject(sum=[CHECKED_LONG_NARROW($1)], state=[$0]) LogicalAggregate(group=[{0}], sum=[SUM($1)]) - LogicalProject(state=[$7], balance=[$3]) + LogicalProject(state=[$7], balance0=[CAST($3):DECIMAL(38, 0)]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableTopK(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CHECKED_LONG_NARROW($t1)], sum=[$t2], state=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum=SUM($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"sum":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml index f2105ce0d3c..81f792a6d5e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -2,11 +2,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) - LogicalProject(sum(balance)=[$1], span(age,5)=[$0]) - LogicalAggregate(group=[{1}], sum(balance)=[SUM($0)]) - LogicalProject(balance=[$7], span(age,5)=[SPAN($10, 5, null:NULL)]) + LogicalProject(sum(balance)=[CHECKED_LONG_NARROW($1)], span(age,5)=[$0]) + LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)]) + LogicalProject(span(age,5)=[SPAN($10, 5, null:NULL)], balance0=[CAST($7):DECIMAL(38, 0)]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableTopK(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CHECKED_LONG_NARROW($t1)], sum(balance)=[$t2], span(age,5)=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"sum(balance)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml index cd0355241fe..7fed9590a2a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml @@ -2,10 +2,11 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last]) - LogicalProject(sum(balance)=[$1], c=[$2], dc(employer)=[$3], state=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) - LogicalProject(state=[$7], balance=[$3], employer=[$6]) + LogicalProject(sum(balance)=[CHECKED_LONG_NARROW($1)], c=[$2], dc(employer)=[$3], state=[$0]) + LogicalAggregate(group=[{0}], sum(balance)=[SUM($2)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $1)]) + LogicalProject(state=[$7], employer=[$6], balance0=[CAST($3):DECIMAL(38, 0)]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CHECKED_LONG_NARROW($t1)], sum(balance)=[$t4], c=[$t2], dc(employer)=[$t3], state=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($2),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml index 59cd137ca59..c23c6a6ee16 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml @@ -2,11 +2,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) - LogicalProject(sum(balance)=[$2], count()=[$3], d=[$4], gender=[$0], new_state=[$1]) - LogicalAggregate(group=[{0, 1}], sum(balance)=[SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) - LogicalProject(gender=[$4], new_state=[$17], balance=[$3], employer=[$6]) + LogicalProject(sum(balance)=[CHECKED_LONG_NARROW($2)], count()=[$3], d=[$4], gender=[$0], new_state=[$1]) + LogicalAggregate(group=[{0, 1}], sum(balance)=[SUM($3)], count()=[COUNT()], d=[COUNT(DISTINCT $2)]) + LogicalProject(gender=[$4], new_state=[$17], employer=[$6], balance0=[CAST($3):DECIMAL(38, 0)]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($17))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CHECKED_LONG_NARROW($t2)], sum(balance)=[$t5], count()=[$t3], d=[$t4], gender=[$t0], new_state=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($3),count()=COUNT(),d=COUNT(DISTINCT $2)), SORT_AGG_METRICS->[4 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml index 68cb12a49dd..417cd022480 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml @@ -2,11 +2,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) - LogicalProject(c=[$2], s=[$3], span(age,5)=[$1], state=[$0]) - LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[SUM($1)]) - LogicalProject(state=[$7], balance=[$3], span(age,5)=[SPAN($8, 5, null:NULL)]) + LogicalProject(c=[$2], s=[CHECKED_LONG_NARROW($3)], span(age,5)=[$1], state=[$0]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], s=[SUM($2)]) + LogicalProject(state=[$7], span(age,5)=[SPAN($8, 5, null:NULL)], balance0=[CAST($3):DECIMAL(38, 0)]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[CHECKED_LONG_NARROW($t3)], c=[$t2], s=[$t4], span(age,5)=[$t1], state=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},c=COUNT(),s=SUM($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index 1d664d5cd43..c1c5b3c2cfc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -1,10 +1,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) + LogicalProject(sum(balance)=[CHECKED_LONG_NARROW($1)], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) - LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) + LogicalProject(gender=[$4], balance4=[CAST($7):DECIMAL(38, 0)], $f7=[+($7, 100)], $f8=[-($7, 100)], $f9=[*($7, 100)], $f10=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CHECKED_LONG_NARROW($t1)], expr#6=[100:BIGINT], expr#7=[*($t3, $t6)], expr#8=[+($t2, $t7)], expr#9=[-($t2, $t7)], expr#10=[*($t2, $t6)], sum(balance)=[$t5], sum(balance + 100)=[$t8], sum(balance - 100)=[$t9], sum(balance * 100)=[$t10], sum(balance / 100)=[$t4], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_SUM=SUM($3),sum(balance + 100)_COUNT=COUNT($3),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}},"sum(balance + 100)_SUM":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json index 064aa294a2d..f265a37f292 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml index c3db9d4f6b9..cceb536d3a3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["age"]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml index 3767a38b3a9..79a15b46e06 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml @@ -2,11 +2,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(c=[$1], s=[$2], state=[$0]) + LogicalProject(c=[$1], s=[CHECKED_LONG_NARROW($2)], state=[$0]) LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) - LogicalProject(state=[$7], balance=[$3]) + LogicalProject(state=[$7], balance0=[CAST($3):DECIMAL(38, 0)]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CHECKED_LONG_NARROW($t2)], c=[$t1], s=[$t3], state=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT(),s=SUM($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml index 520f729b7f9..201af27e53a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml @@ -2,11 +2,12 @@ calcite: logical: | LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(c=[$1], s=[$2], state=[$0]) + LogicalProject(c=[$1], s=[CHECKED_LONG_NARROW($2)], state=[$0]) LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) - LogicalProject(state=[$7], balance=[$3]) + LogicalProject(state=[$7], balance0=[CAST($3):DECIMAL(38, 0)]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[CHECKED_LONG_NARROW($t2)], c=[$t1], s=[$t3], state=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT(),s=SUM($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBOHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDM4LAogICAgInNjYWxlIjogMAogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_nested_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_nested_agg_push.yaml index 233773bf0d1..e8f0f283533 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_nested_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_nested_agg_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(count_area=[$1], min_area=[$2], max_area=[$3], avg_area=[$4], avg_age=[$5], name=[$0]) LogicalAggregate(group=[{0}], count_area=[COUNT($1)], min_area=[MIN($1)], max_area=[MAX($1)], avg_area=[AVG($1)], avg_age=[AVG($2)]) - LogicalProject(name=[$0], address.area=[$2], age=[$8]) + LogicalProject(name=[$0], address.area=[$2], age0=[CAST($8):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count_area=COUNT($1),min_area=MIN($1),max_area=MAX($1),avg_area=AVG($1),avg_age=AVG($2)), PROJECT->[count_area, min_area, max_area, avg_area, avg_age, name], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"name":{"terms":{"field":"name.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"nested_count_area":{"nested":{"path":"address"},"aggregations":{"count_area":{"value_count":{"field":"address.area"}}}},"nested_min_area":{"nested":{"path":"address"},"aggregations":{"min_area":{"min":{"field":"address.area"}}}},"nested_max_area":{"nested":{"path":"address"},"aggregations":{"max_area":{"max":{"field":"address.area"}}}},"nested_avg_area":{"nested":{"path":"address"},"aggregations":{"avg_area":{"avg":{"field":"address.area"}}}},"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count_area=COUNT($1),min_area=MIN($1),max_area=MAX($1),avg_area=AVG($1),avg_age=AVG($2)), PROJECT->[count_area, min_area, max_area, avg_area, avg_age, name], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"name":{"terms":{"field":"name.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"nested_count_area":{"nested":{"path":"address"},"aggregations":{"count_area":{"value_count":{"field":"address.area"}}}},"nested_min_area":{"nested":{"path":"address"},"aggregations":{"min_area":{"min":{"field":"address.area"}}}},"nested_max_area":{"nested":{"path":"address"},"aggregations":{"max_area":{"max":{"field":"address.area"}}}},"nested_avg_area":{"nested":{"path":"address"},"aggregations":{"avg_area":{"avg":{"field":"address.area"}}}},"avg_age":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["age"]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml index b8e0f7ed6d5..7818690c70a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml @@ -10,7 +10,7 @@ calcite: LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | @@ -19,4 +19,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["age"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json index 6daddb8ee76..915084f2e31 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165293.75 rows, 36190.23815546412 cpu, 0.0 io}, id = 4117\n LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165237.5 rows, 35290.23815546412 cpu, 0.0 io}, id = 4116\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 56.25, cumulative cost = {165181.25 rows, 35233.98815546412 cpu, 0.0 io}, id = 4115\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2]): rowcount = 56.25, cumulative cost = {165125.0 rows, 29793.75 cpu, 0.0 io}, id = 4114\n LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165068.75 rows, 29625.0 cpu, 0.0 io}, id = 4113\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)]): rowcount = 225.0, cumulative cost = {165012.5 rows, 29400.0 cpu, 0.0 io}, id = 4112\n LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {164787.5 rows, 28500.0 cpu, 0.0 io}, id = 4111\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {164562.5 rows, 28000.0 cpu, 0.0 io}, id = 4110\n LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = 4106\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = 4105\n LogicalProject(state=[$7], city=[$5], age=[$8]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = 4104\n LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = 4103\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = 4102\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1]): rowcount = 225.0, cumulative cost = {3201.75 rows, 39096.942171903866 cpu, 0.0 io}, id = 4910\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]): rowcount = 225.0, cumulative cost = {2976.75 rows, 38196.942171903866 cpu, 0.0 io}, id = 4902\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]): rowcount = 225.0, cumulative cost = {2531.25 rows, 8950.0 cpu, 0.0 io}, id = 4906\n EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2306.25 rows, 4900.0 cpu, 0.0 io}, id = 4898\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]): rowcount = 450.0, cumulative cost = {1856.25 rows, 4000.0 cpu, 0.0 io}, id = 4914\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#4218:LogicalAggregate.NONE.[](input=RelSubset#4186,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1406.25 rows, 0.0 cpu, 0.0 io}, id = 4505\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165293.75 rows, 36190.23815546412 cpu, 0.0 io}, id = *\n LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165237.5 rows, 35290.23815546412 cpu, 0.0 io}, id = *\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 56.25, cumulative cost = {165181.25 rows, 35233.98815546412 cpu, 0.0 io}, id = *\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2]): rowcount = 56.25, cumulative cost = {165125.0 rows, 29793.75 cpu, 0.0 io}, id = *\n LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165068.75 rows, 29625.0 cpu, 0.0 io}, id = *\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)]): rowcount = 225.0, cumulative cost = {165012.5 rows, 29400.0 cpu, 0.0 io}, id = *\n LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {164787.5 rows, 28500.0 cpu, 0.0 io}, id = *\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {164562.5 rows, 28000.0 cpu, 0.0 io}, id = *\n LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = *\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = *\n LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = *\n LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = *\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = *\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1]): rowcount = 225.0, cumulative cost = {3252.375032186508 rows, 39096.942171903866 cpu, 0.0 io}, id = *\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]): rowcount = 225.0, cumulative cost = {3027.375032186508 rows, 38196.942171903866 cpu, 0.0 io}, id = *\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]): rowcount = 225.0, cumulative cost = {2581.875032186508 rows, 8950.0 cpu, 0.0 io}, id = *\n EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2356.875032186508 rows, 4900.0 cpu, 0.0 io}, id = *\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]): rowcount = 450.0, cumulative cost = {1906.8750321865082 rows, 4000.0 cpu, 0.0 io}, id = *\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0],\"DIGESTS\":[\"age\"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1456.8750321865082 rows, 0.0 cpu, 0.0 io}, id = *\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml index c760955d27b..e9789abea85 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_cost.yaml @@ -1,22 +1,22 @@ calcite: logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165293.75 rows, 36190.23815546412 cpu, 0.0 io}, id = 3303 - LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165237.5 rows, 35290.23815546412 cpu, 0.0 io}, id = 3302 - LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 56.25, cumulative cost = {165181.25 rows, 35233.98815546412 cpu, 0.0 io}, id = 3301 - LogicalProject(avg_age=[$0], state=[$1], age2=[$2]): rowcount = 56.25, cumulative cost = {165125.0 rows, 29793.75 cpu, 0.0 io}, id = 3300 - LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165068.75 rows, 29625.0 cpu, 0.0 io}, id = 3299 - LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)]): rowcount = 225.0, cumulative cost = {165012.5 rows, 29400.0 cpu, 0.0 io}, id = 3298 - LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {164787.5 rows, 28500.0 cpu, 0.0 io}, id = 3297 - LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {164562.5 rows, 28000.0 cpu, 0.0 io}, id = 3296 - LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = 3292 - LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = 3291 - LogicalProject(state=[$7], city=[$5], age=[$8]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = 3290 - LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = 3289 - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = 3288 + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 56.25, cumulative cost = {165293.75 rows, 36190.23815546412 cpu, 0.0 io}, id = * + LogicalProject(age2=[$2]): rowcount = 56.25, cumulative cost = {165237.5 rows, 35290.23815546412 cpu, 0.0 io}, id = * + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]): rowcount = 56.25, cumulative cost = {165181.25 rows, 35233.98815546412 cpu, 0.0 io}, id = * + LogicalProject(avg_age=[$0], state=[$1], age2=[$2]): rowcount = 56.25, cumulative cost = {165125.0 rows, 29793.75 cpu, 0.0 io}, id = * + LogicalFilter(condition=[<=($3, 1)]): rowcount = 56.25, cumulative cost = {165068.75 rows, 29625.0 cpu, 0.0 io}, id = * + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)]): rowcount = 225.0, cumulative cost = {165012.5 rows, 29400.0 cpu, 0.0 io}, id = * + LogicalFilter(condition=[IS NOT NULL($2)]): rowcount = 225.0, cumulative cost = {164787.5 rows, 28500.0 cpu, 0.0 io}, id = * + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]): rowcount = 500.0, cumulative cost = {164562.5 rows, 28000.0 cpu, 0.0 io}, id = * + LogicalProject(avg_age=[$2], state=[$0], city=[$1]): rowcount = 500.0, cumulative cost = {164062.5 rows, 26500.0 cpu, 0.0 io}, id = * + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]): rowcount = 500.0, cumulative cost = {163562.5 rows, 25000.0 cpu, 0.0 io}, id = * + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]): rowcount = 5000.0, cumulative cost = {163000.0 rows, 25000.0 cpu, 0.0 io}, id = * + LogicalFilter(condition=[>($8, 30)]): rowcount = 5000.0, cumulative cost = {158000.0 rows, 10000.0 cpu, 0.0 io}, id = * + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]): rowcount = 10000.0, cumulative cost = {153000.0 rows, 0.0 cpu, 0.0 io}, id = * physical: | - EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1]): rowcount = 225.0, cumulative cost = {3201.75 rows, 39096.942171903866 cpu, 0.0 io}, id = 4096 - CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]): rowcount = 225.0, cumulative cost = {2976.75 rows, 38196.942171903866 cpu, 0.0 io}, id = 4088 - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]): rowcount = 225.0, cumulative cost = {2531.25 rows, 8950.0 cpu, 0.0 io}, id = 4092 - EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2306.25 rows, 4900.0 cpu, 0.0 io}, id = 4084 - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]): rowcount = 450.0, cumulative cost = {1856.25 rows, 4000.0 cpu, 0.0 io}, id = 4100 - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#3404:LogicalAggregate.NONE.[](input=RelSubset#3372,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1406.25 rows, 0.0 cpu, 0.0 io}, id = 3691 + EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1]): rowcount = 225.0, cumulative cost = {3252.375032186508 rows, 39096.942171903866 cpu, 0.0 io}, id = * + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]): rowcount = 225.0, cumulative cost = {3027.375032186508 rows, 38196.942171903866 cpu, 0.0 io}, id = * + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]): rowcount = 225.0, cumulative cost = {2581.875032186508 rows, 8950.0 cpu, 0.0 io}, id = * + EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]): rowcount = 450.0, cumulative cost = {2356.875032186508 rows, 4900.0 cpu, 0.0 io}, id = * + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]): rowcount = 450.0, cumulative cost = {1906.8750321865082 rows, 4000.0 cpu, 0.0 io}, id = * + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["age"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]): rowcount = 500.0, cumulative cost = {1456.8750321865082 rows, 0.0 cpu, 0.0 io}, id = * diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json index 45ec4243593..b67311a45d7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.json @@ -1,7 +1,7 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1])\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#2590:LogicalAggregate.NONE.[](input=RelSubset#2558,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n", + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1])\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"CAST\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"CAST\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"SPECIAL\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"DOUBLE\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0],\"DIGESTS\":[\"age\"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n", "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable = v1stashed.scan();\n final org.apache.calcite.linq4j.AbstractEnumerable source = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if ((Double) ((Object[]) inputEnumerator.current())[1] != null) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n final Object[] current = (Object[]) inputEnumerator.current();\n final Double input_value0 = (Double) current[1];\n return new Object[] {\n current[0],\n input_value0 == null ? null : Double.valueOf(input_value0.doubleValue() + (double) 2)};\n }\n\n };\n }\n\n };\n int prevStart;\n int prevEnd;\n final java.util.Comparator comparator = new java.util.Comparator(){\n public int compare(Object[] v0, Object[] v1) {\n final int c;\n c = org.apache.calcite.runtime.Utilities.compareNullsFirst((Double) v0[1], (Double) v1[1]);\n if (c != 0) {\n return c;\n }\n return 0;\n }\n\n public int compare(Object o0, Object o1) {\n return this.compare((Object[]) o0, (Object[]) o1);\n }\n\n };\n final org.apache.calcite.runtime.SortedMultiMap multiMap = new org.apache.calcite.runtime.SortedMultiMap();\n source.foreach(new org.apache.calcite.linq4j.function.Function1() {\n public Object apply(Object[] v) {\n Double key = (Double) v[1];\n multiMap.putMulti(key, v);\n return null;\n }\n public Object apply(Object v) {\n return apply(\n (Object[]) v);\n }\n }\n );\n final java.util.Iterator iterator = multiMap.arrays(comparator);\n final java.util.ArrayList _list = new java.util.ArrayList(\n multiMap.size());\n Long a0w0 = (Long) null;\n while (iterator.hasNext()) {\n final Object[] _rows = (Object[]) iterator.next();\n prevStart = -1;\n prevEnd = 2147483647;\n for (int i = 0; i < _rows.length; (++i)) {\n final Object[] row = (Object[]) _rows[i];\n if (i != prevEnd) {\n int actualStart = i < prevEnd ? 0 : prevEnd + 1;\n prevEnd = i;\n a0w0 = Long.valueOf(((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown((i - 0 + 1))).longValue());\n }\n _list.add(new Object[] {\n row[0],\n row[1],\n a0w0});\n }\n }\n multiMap.clear();\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable0 = org.apache.calcite.linq4j.Linq4j.asEnumerable(_list);\n final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable0.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n while (inputEnumerator.moveNext()) {\n if (org.apache.calcite.runtime.SqlFunctions.toLong(((Object[]) inputEnumerator.current())[2]) <= $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b) {\n return true;\n }\n }\n return false;\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n final Object[] current = (Object[]) inputEnumerator.current();\n final Object input_value = current[0];\n final Object input_value0 = current[1];\n final Object input_value1 = current[2];\n return new Object[] {\n input_value,\n input_value0,\n input_value1};\n }\n\n static final long $L4J$C$_Number_org_apache_calcite_linq4j_tree_Primitive_of_long_class_358aa52b = ((Number)org.apache.calcite.linq4j.tree.Primitive.of(long.class).numberValueRoundDown(1)).longValue();\n };\n }\n\n };\n final org.apache.calcite.linq4j.Enumerable _inputEnumerable1 = org.apache.calcite.linq4j.EnumerableDefaults.orderBy(child, new org.apache.calcite.linq4j.function.Function1() {\n public String apply(Object[] v) {\n return v[0] == null ? null : v[0].toString();\n }\n public Object apply(Object v) {\n return apply(\n (Object[]) v);\n }\n }\n , org.apache.calcite.linq4j.function.Functions.nullsComparator(true, false), 0, 10000);\n return new org.apache.calcite.linq4j.AbstractEnumerable(){\n public org.apache.calcite.linq4j.Enumerator enumerator() {\n return new org.apache.calcite.linq4j.Enumerator(){\n public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable1.enumerator();\n public void reset() {\n inputEnumerator.reset();\n }\n\n public boolean moveNext() {\n return inputEnumerator.moveNext();\n }\n\n public void close() {\n inputEnumerator.close();\n }\n\n public Object current() {\n return (Double) ((Object[]) inputEnumerator.current())[1];\n }\n\n };\n }\n\n };\n}\n\n\npublic Class getElementType() {\n return java.lang.Double.class;\n}\n\n\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml index f91ea4ba4d8..2efc86730c5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_extended.yaml @@ -10,7 +10,7 @@ calcite: LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | @@ -19,7 +19,7 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#1776:LogicalAggregate.NONE.[](input=RelSubset#1744,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"CAST\\\",\\n \\\"kind\\\": \\\"CAST\\\",\\n \\\"syntax\\\": \\\"SPECIAL\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"DOUBLE\\\",\\n \\\"nullable\\\": true\\n }\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["age"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) extended: |+ public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) { final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get("v1stashed"); @@ -186,4 +186,6 @@ calcite: public Class getElementType() { return java.lang.Double.class; - } \ No newline at end of file + } + + diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json index 1874fc81ec7..ac1c9b71e61 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1])\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $1 NULLS FIRST)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1])\n CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0],\"DIGESTS\":[\"age\"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml index b8e0f7ed6d5..7818690c70a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output_standard.yaml @@ -10,7 +10,7 @@ calcite: LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | @@ -19,4 +19,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[2], expr#3=[+($t1, $t2)], expr#4=[IS NOT NULL($t1)], state=[$t0], age2=[$t3], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["age"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json index 9eea9f67a53..80d55307008 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXQBEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNUIiwKICAgICJraW5kIjogIkNBU1QiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0],\"DIGESTS\":[\"age\"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json index 0d6b8c4b9cf..0a1fa221c10 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},avg(balance)=AVG($0)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance0=[CAST($3):DOUBLE])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], avg(balance)=[$t7], state=[$t0])\n EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):DOUBLE], state=[$t0], balance0=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, state, age], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}, {\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], PROJECT->[state, balance]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"state\",\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 0478b24369c..301309e5fd6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -3,19 +3,21 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) - LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($18)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __r_age__0=[CAST($20):DOUBLE]) + LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:DOUBLE], expr#23=[CASE($t21, $t22, $t18)], expr#24=[/($t23, $t19)], proj#0..10=[{exprs}], avg_age=[$t24]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..2=[{inputs}], __r_seq__=[$t2], __r_gender__=[$t0], __r_age__=[$t1]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($18)], agg#1=[COUNT($18)]) + EnumerableCalc(expr#0..20=[{inputs}], expr#21=[CAST($t20):DOUBLE], proj#0..17=[{exprs}], __r_age__0=[$t21]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..2=[{inputs}], __r_seq__=[$t2], __r_gender__=[$t0], __r_age__=[$t1]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml index a1cf6ae00e9..a8826392cfe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml @@ -3,21 +3,23 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) - LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), =($4, $19))], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($18)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __r_age__0=[CAST($20):DOUBLE]) + LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), =($4, $19))], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:DOUBLE], expr#23=[CASE($t21, $t22, $t18)], expr#24=[/($t23, $t19)], proj#0..10=[{exprs}], avg_age=[$t24]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) - EnumerableSort(sort0=[$4], dir0=[ASC]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], __r_seq__=[$t2], __r_gender__=[$t0], __r_age__=[$t1]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($18)], agg#1=[COUNT($18)]) + EnumerableCalc(expr#0..20=[{inputs}], expr#21=[CAST($t20):DOUBLE], proj#0..17=[{exprs}], __r_age__0=[$t21]) + EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) + EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], __r_seq__=[$t2], __r_gender__=[$t0], __r_age__=[$t1]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index 324960f28dd..bd569295468 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) + LogicalProject(age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), OR(=($4, $cor0.gender), AND(IS NULL($4), IS NULL($cor0.gender))))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) @@ -22,17 +22,18 @@ calcite: EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:DOUBLE], expr#9=[CASE($t7, $t8, $t4)], expr#10=[/($t9, $t5)], proj#0..3=[{exprs}], avg_age=[$t10]) + EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DOUBLE], proj#0..3=[{exprs}], age0=[$t8]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml index 42b50e7eb5f..5ae183cc6ad 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) + LogicalProject(age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), =($4, $cor0.gender))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) @@ -24,17 +24,18 @@ calcite: EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t3)], expr#9=[/($t8, $t4)], proj#0..2=[{exprs}], avg_age=[$t9]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[CAST($t4):DOUBLE], proj#0..2=[{exprs}], age0=[$t7]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml index 059caa2e2d2..501c5085737 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_balance=[$2], age_range=[$0], state=[$1]) LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) - LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], state=[$t1]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], avg_balance=[$t8], age_range=[$t0], state=[$t1]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=[CASE($t20, $t21, $t11)], age_range=[$t22], state=[$t9], balance=[$t7]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=[CASE($t20, $t21, $t11)], expr#23=[CAST($t7):DOUBLE], age_range=[$t22], state=[$t9], balance0=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml index 43e27cd2d5d..0e083fb4b00 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$3], count()=[$4], age_range=[$0], state=[$1], gender=[$2]) LogicalAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t4, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t3)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t4)], avg(balance)=[$t11], count()=[$t5], age_range=[$t0], state=[$t1], gender=[$t2]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t4, $t6)], expr#8=[null:DOUBLE], expr#9=[CASE($t7, $t8, $t3)], expr#10=[/($t9, $t4)], avg(balance)=[$t10], count()=[$t5], age_range=[$t0], state=[$t1], gender=[$t2]) EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)], count()=[COUNT()]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], age_range=[$t23], state=[$t9], gender=[$t4], balance=[$t7]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[CAST($t7):DOUBLE], age_range=[$t23], state=[$t9], gender=[$t4], balance0=[$t24]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml index 6dfa7cd65a3..f6e8b92ffb7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_balance=[$3], age_range=[$0], balance_range=[$1], state=[$2]) LogicalAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) - LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], avg_balance=[$t10], age_range=[$t0], balance_range=[$t1], state=[$t2]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t3)], expr#9=[/($t8, $t4)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1], state=[$t2]) EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=['a35':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[20000], expr#25=[<($t7, $t24)], expr#26=['medium':VARCHAR], expr#27=['high':VARCHAR], expr#28=[CASE($t25, $t26, $t27)], age_range=[$t23], balance_range=[$t28], state=[$t9], balance=[$t7]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=['a35':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[20000], expr#25=[<($t7, $t24)], expr#26=['medium':VARCHAR], expr#27=['high':VARCHAR], expr#28=[CASE($t25, $t26, $t27)], expr#29=[CAST($t7):DOUBLE], age_range=[$t23], balance_range=[$t28], state=[$t9], balance0=[$t29]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml index 41ed8ba61fc..e18853d30ba 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$2], state=[$0], age_range=[$1]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance=[$7]) + LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t0], age_range=[$t1]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], avg(balance)=[$t8], state=[$t0], age_range=[$t1]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], state=[$t9], age_range=[$t23], balance=[$t7]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[CAST($t7):DOUBLE], state=[$t9], age_range=[$t23], balance0=[$t24]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml index 10ead7ad449..c1071fda5ac 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg(balance)=[$1], age_range=[$0]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[35..40), [80..+∞)]), '30-40 or >=80':VARCHAR, null:NULL)], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], age_range=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], avg(balance)=[$t7], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[CAST($t7):DOUBLE], age_range=[$t26], balance0=[$t27]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml index 404726f6083..ef12f55a9ae 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_balance=[$2], age_range=[$0], balance_range=[$1]) LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) - LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], balance=[$7]) + LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, <($10, 40), 'u40':VARCHAR, 'u100':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], balance0=[CAST($7):DOUBLE]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], avg_balance=[$t8], age_range=[$t0], balance_range=[$t1]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], balance=[$t7]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], expr#32=[CAST($t7):DOUBLE], age_range=[$t26], balance_range=[$t31], balance0=[$t32]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index fe925e0a80a..83c7a6b4495 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -7,7 +7,7 @@ calcite: LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) + LogicalProject(gender=[$4], age=[$10], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) @@ -15,7 +15,7 @@ calcite: LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(gender=[$4], age=[$10], balance=[$7]) + LogicalProject(gender=[$4], age=[$10], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t9]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t7):DOUBLE], expr#20=[IS NOT NULL($t4)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], gender=[$t4], age=[$t10], balance0=[$t19], $condition=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[SAFE_CAST($t10)], expr#22=[IS NOT NULL($t21)], expr#23=[AND($t19, $t20, $t22)], proj#0..18=[{exprs}], $condition=[$t23]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], age=[$t4], avg(balance)=[$t9]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t7):DOUBLE], expr#20=[IS NOT NULL($t4)], expr#21=[IS NOT NULL($t7)], expr#22=[SAFE_CAST($t10)], expr#23=[IS NOT NULL($t22)], expr#24=[AND($t20, $t21, $t23)], gender=[$t4], age=[$t10], balance0=[$t19], $condition=[$t24]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index beb3275a6c6..b3dd139a431 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -6,16 +6,16 @@ calcite: LogicalProject(gender=[$0], age=[CASE(IS NULL($1), 'nil', <=($5, 10), $1, 'OTHER')], avg(balance)=[$2]) LogicalJoin(condition=[=($1, $3)], joinType=[left]) LogicalProject(gender=[$0], age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age0=[SPAN($5, 10, null:NULL)], balance0=[CAST($3):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) LogicalProject(age=[$0], __grand_total__=[$1], _row_number_chart_=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS LAST)]) LogicalAggregate(group=[{0}], __grand_total__=[SUM($1)]) LogicalFilter(condition=[IS NOT NULL($0)]) LogicalProject(age=[SAFE_CAST($1)], avg(balance)=[$2]) - LogicalAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$3], age0=[SPAN($5, 10, null:NULL)]) + LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) + LogicalProject(gender=[$4], age0=[SPAN($5, 10, null:NULL)], balance0=[CAST($3):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($3))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) physical: | @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t9]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[CAST($t3):DOUBLE], expr#17=[IS NOT NULL($t4)], expr#18=[IS NOT NULL($t3)], expr#19=[AND($t17, $t18)], gender=[$t4], age0=[$t15], balance0=[$t16], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t20]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t2)], expr#9=[/($t8, $t3)], age=[$t4], avg(balance)=[$t9]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[CAST($t3):DOUBLE], expr#17=[IS NOT NULL($t4)], expr#18=[IS NOT NULL($t3)], expr#19=[SAFE_CAST($t15)], expr#20=[IS NOT NULL($t19)], expr#21=[AND($t17, $t18, $t20)], gender=[$t4], age0=[$t15], balance0=[$t16], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml index 8224f075819..79917e39762 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml @@ -3,13 +3,13 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)]) - LogicalProject(gender=[$4], balance=[$7]) + LogicalProject(gender=[$4], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) - EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], gender=[$t0], avg(balance)=[$t7]) + EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t7):DOUBLE], expr#20=[IS NOT NULL($t4)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], gender=[$t4], balance0=[$t19], $condition=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml index 16aa3871687..fbaced2bd72 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml @@ -3,13 +3,13 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC]) LogicalAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) - LogicalProject(state=[$9], gender=[$4], balance=[$7]) + LogicalProject(state=[$9], gender=[$4], balance0=[CAST($7):DOUBLE]) LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) - EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(balance)=[$t8]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t7):DOUBLE], expr#20=[IS NOT NULL($t9)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], state=[$t9], gender=[$t4], balance0=[$t19], $condition=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index 655e16839ed..0addd887dd4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -1,13 +1,13 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) + LogicalProject(sum(balance)=[CHECKED_LONG_NARROW($1)], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) - LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) + LogicalProject(gender=[$4], balance4=[CAST($7):DECIMAL(38, 0)], $f7=[+($7, 100)], $f8=[-($7, 100)], $f9=[*($7, 100)], $f10=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CHECKED_LONG_NARROW($t1)], expr#6=[100:BIGINT], expr#7=[*($t3, $t6)], expr#8=[+($t2, $t7)], expr#9=[-($t2, $t7)], expr#10=[*($t2, $t6)], sum(balance)=[$t5], sum(balance + 100)=[$t8], sum(balance - 100)=[$t9], sum(balance * 100)=[$t10], sum(balance / 100)=[$t4], gender=[$t0]) + EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_SUM=[SUM($3)], sum(balance + 100)_COUNT=[COUNT($3)], sum(balance / 100)=[SUM($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t7):DECIMAL(38, 0)], expr#20=[100], expr#21=[DIVIDE($t7, $t20)], gender=[$t4], balance=[$t19], $f5=[$t21], balance0=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json index a31d2acfc61..0b9e873d52c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml index ac3728eacb9..bfabab136c1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -3,12 +3,12 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], avg_age=[$t8], state=[$t0], city=[$t1]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CAST($t8):DOUBLE], expr#18=[30], expr#19=[>($t8, $t18)], state=[$t7], city=[$t5], age0=[$t17], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index f781995261c..013b71f7c79 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -10,7 +10,7 @@ calcite: LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) LogicalProject(avg_age=[$2], state=[$0], city=[$1]) LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) - LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | @@ -19,7 +19,7 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t11)], state=[$t1], age2=[$t11], $condition=[$t12]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[2], expr#10=[+($t8, $t9)], expr#11=[IS NOT NULL($t10)], state=[$t0], age2=[$t10], $condition=[$t11]) + EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CAST($t8):DOUBLE], expr#18=[30], expr#19=[>($t8, $t18)], state=[$t7], city=[$t5], age0=[$t17], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json index 028a56cb020..6e3f6db20ef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age0=[CAST($8):DOUBLE])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], avg_age=[$t8], state=[$t0], city=[$t1])\n EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CAST($t8):DOUBLE], state=[$t7], city=[$t5], age0=[$t17])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json index 9d64b554b18..55e564cd0b3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], state=[$t0])\n EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance0=[CAST($3):DOUBLE])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], avg(balance)=[$t7], state=[$t0])\n EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[CAST($t3):DOUBLE], state=[$t7], balance0=[$t17])\n EnumerableSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index 0bf9a2c50ce..7ae49b5ed27 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -3,20 +3,22 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) - LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($18)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __r_age__0=[CAST($20):DOUBLE]) + LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:DOUBLE], expr#23=[CASE($t21, $t22, $t18)], expr#24=[/($t23, $t19)], proj#0..10=[{exprs}], avg_age=[$t24]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..17=[{inputs}], __r_seq__=[$t17], __r_gender__=[$t4], __r_age__=[$t8]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($18)], agg#1=[COUNT($18)]) + EnumerableCalc(expr#0..20=[{inputs}], expr#21=[CAST($t20):DOUBLE], proj#0..17=[{exprs}], __r_age__0=[$t21]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..17=[{inputs}], __r_seq__=[$t17], __r_gender__=[$t4], __r_age__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml index d72bf7b429f..cefe37921c4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml @@ -3,22 +3,24 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], avg_age=[$18]) LogicalSort(sort0=[$17], dir0=[ASC]) - LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) - LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), =($4, $19))], joinType=[left]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($18)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __r_age__0=[CAST($20):DOUBLE]) + LogicalJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), =($4, $19))], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:DOUBLE], expr#23=[CASE($t21, $t22, $t18)], expr#24=[/($t23, $t19)], proj#0..10=[{exprs}], avg_age=[$t24]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) - EnumerableSort(sort0=[$4], dir0=[ASC]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..17=[{inputs}], __r_seq__=[$t17], __r_gender__=[$t4], __r_age__=[$t8]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($18)], agg#1=[COUNT($18)]) + EnumerableCalc(expr#0..20=[{inputs}], expr#21=[CAST($t20):DOUBLE], proj#0..17=[{exprs}], __r_age__0=[$t21]) + EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) + EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..17=[{inputs}], __r_seq__=[$t17], __r_gender__=[$t4], __r_age__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 3ec98ba9382..10f066db682 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) + LogicalProject(age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), OR(=($4, $cor0.gender), AND(IS NULL($4), IS NULL($cor0.gender))))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) @@ -23,17 +23,18 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:DOUBLE], expr#9=[CASE($t7, $t8, $t4)], expr#10=[/($t9, $t5)], proj#0..3=[{exprs}], avg_age=[$t10]) + EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) + EnumerableCalc(expr#0..7=[{inputs}], expr#8=[CAST($t5):DOUBLE], proj#0..3=[{exprs}], age0=[$t8]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml index 40fb4087001..59f7de9e8d1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml @@ -8,7 +8,7 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) LogicalAggregate(group=[{}], avg_age=[AVG($0)]) - LogicalProject(age=[$8]) + LogicalProject(age0=[CAST($8):DOUBLE]) LogicalFilter(condition=[AND(<($17, $cor0.__stream_seq__), =($20, $cor0.__seg_id__), =($4, $cor0.gender))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[$17], __reset_before_flag__=[$18], __reset_after_flag__=[$19], __seg_id__=[+(SUM($18) OVER (ROWS UNBOUNDED PRECEDING), COALESCE(SUM($19) OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], __stream_seq__=[ROW_NUMBER() OVER ()], __reset_before_flag__=[CASE(>($8, 34), 1, 0)], __reset_after_flag__=[CASE(<($8, 25), 1, 0)]) @@ -23,17 +23,18 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:DOUBLE], expr#8=[CASE($t6, $t7, $t3)], expr#9=[/($t8, $t4)], proj#0..2=[{exprs}], avg_age=[$t9]) + EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[CAST($t4):DOUBLE], proj#0..2=[{exprs}], age0=[$t7]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml new file mode 100644 index 00000000000..f429a7870da --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml @@ -0,0 +1,123 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# SUM/AVG over a BIGINT (long) column near 2^63 must not silently wrap to a negative value. +# +# The enumerable SUM accumulator for a long argument is a plain long, so a running sum past 2^63 +# wraps to a negative result (e.g. SUM(long) returned -5594372458244005145). AVG reduces to +# SUM(field)/COUNT(field), so its intermediate long SUM wraps the same way (AVG returned a negative +# average). SUM(long) is now accumulated in DECIMAL and cast back to BIGINT (in-memory) or checked +# when narrowing the pushed-down double result, turning a genuine overflow into a 4xx client error +# while keeping the bigint output type; AVG(long) is averaged in DOUBLE, which holds the true +# average without wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_agg_overflow + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_overflow + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - do: + indices.create: + index: test_agg_in_range + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_in_range + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 1000000000000}' + - '{"index": {}}' + - '{"long_field": 2000000000000}' + - '{"index": {}}' + - '{"long_field": 3000000000000}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_agg_overflow + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_in_range + ignore_unavailable: true + +--- +"SUM of large longs overflowing BIGINT throws error": + - skip: + features: + - headers + # 3 * (2^63 - 1) far exceeds the BIGINT range; historically this wrapped to a negative value. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"AVG of large longs does not overflow or error": + - skip: + features: + - headers + # The average of three identical values is that value; averaging in DOUBLE avoids the wrap that + # a long intermediate SUM would cause. 9223372036854775807 is returned as a double (9.223...E18). + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats avg(long_field) + - match: { total: 1 } + - match: { datarows: [[9.223372036854776e18]] } + +--- +"SUM of longs within BIGINT range does not error": + - skip: + features: + - headers + # 1e12 + 2e12 + 3e12 = 6e12, well within long range; must return the exact sum, no error. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_in_range | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[6000000000000]] }