diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java index de167cd08e4f..8f193a4acede 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollect.java @@ -51,7 +51,16 @@ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, *

Use {@link #create} unless you know what you're doing. */ public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode child, boolean withOrdinality) { - super(cluster, traitSet, child, withOrdinality, Collections.emptyList()); + this(cluster, traitSet, child, withOrdinality, true); + } + + /** Creates an EnumerableUncollect. + * + *

Use {@link #create} unless you know what you're doing. */ + public EnumerableUncollect(RelOptCluster cluster, RelTraitSet traitSet, + RelNode child, boolean withOrdinality, boolean expandStructFields) { + super(cluster, traitSet, child, withOrdinality, Collections.emptyList(), + expandStructFields); assert getConvention() instanceof EnumerableConvention; assert getConvention() == child.getConvention(); } @@ -72,10 +81,27 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, return new EnumerableUncollect(cluster, traitSet, input, withOrdinality); } + /** + * Creates an EnumerableUncollect. + * + * @param traitSet Trait set + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, + boolean withOrdinality, boolean expandStructFields) { + final RelOptCluster cluster = input.getCluster(); + return new EnumerableUncollect(cluster, traitSet, input, withOrdinality, + expandStructFields); + } + @Override public EnumerableUncollect copy(RelTraitSet traitSet, RelNode newInput) { return new EnumerableUncollect(getCluster(), traitSet, newInput, - withOrdinality); + withOrdinality, expandStructFields); } @Override public Result implement(EnumerableRelImplementor implementor, Prefer pref) { @@ -105,7 +131,7 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, inputTypes.add(FlatProductInputType.MAP); } else { final RelDataType elementType = getComponentTypeOrThrow(type); - if (elementType.isStruct()) { + if (elementType.isStruct() && expandStructFields) { if (elementType.getFieldCount() == 1 && child.getRowType().getFieldList().size() == 1 && !withOrdinality) { // Solves CALCITE-4063: if we are processing a single field, which is a struct with a @@ -116,6 +142,12 @@ public static EnumerableUncollect create(RelTraitSet traitSet, RelNode input, fieldCounts.add(elementType.getFieldCount()); inputTypes.add(FlatProductInputType.LIST); } + } else if (elementType.isStruct()) { + // A struct element kept whole occupies a single output column, + // like a scalar element, but its row value must be converted from + // the collection's internal list representation to Object[]. + fieldCounts.add(-1); + inputTypes.add(FlatProductInputType.STRUCT); } else { fieldCounts.add(-1); inputTypes.add(FlatProductInputType.SCALAR); diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java index 9079964897dc..95a9237c2222 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableUncollectRule.java @@ -49,6 +49,6 @@ protected EnumerableUncollectRule(Config config) { convert(input, input.getTraitSet().replace(EnumerableConvention.INSTANCE)); return EnumerableUncollect.create(traitSet, newInput, - uncollect.withOrdinality); + uncollect.withOrdinality, uncollect.expandStructFields); } } diff --git a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java index 2d4c3620a74c..e607509ddc18 100644 --- a/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/core/Uncollect.java @@ -50,10 +50,21 @@ *

Like its inverse operation {@link Collect}, Uncollect is generally * invoked in a nested loop, driven by * {@link org.apache.calcite.rel.logical.LogicalCorrelate} or similar. + * + *

{@code expandStructFields} controls the shape of the element columns: + * if {@code true} a collection whose element type is a struct produces one + * output column per struct field; if {@code false} it produces a single + * column typed as the whole element (Trino semantics). Maps always expand + * into a key and a value column, regardless of this flag. */ public class Uncollect extends SingleRel { public final boolean withOrdinality; + /** If true, a collection whose element type is a struct expands into one + * output column per struct field; if false, it produces a single column + * typed as the whole element. */ + public final boolean expandStructFields; + // To alias the items in Uncollect list, // i.e., "UNNEST(a, b, c) as T(d, e, f)" // outputs as row type Record(d, e, f) where the field "d" has element type of "a", @@ -74,12 +85,30 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, /** Creates an Uncollect. * *

Use {@link #create} unless you know what you're doing. */ - @SuppressWarnings("method.invocation.invalid") public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, boolean withOrdinality, List itemAliases) { + // Non-empty item aliases historically implied that struct elements are not + // expanded (Presto dialect), so this constructor derives + // {@code expandStructFields} from their absence. + this(cluster, traitSet, input, withOrdinality, itemAliases, itemAliases.isEmpty()); + } + + /** Creates an Uncollect. + * + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param itemAliases Aliases for the operand items + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + @SuppressWarnings("method.invocation.invalid") + public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, + boolean withOrdinality, List itemAliases, boolean expandStructFields) { super(cluster, traitSet, input); this.withOrdinality = withOrdinality; this.itemAliases = ImmutableList.copyOf(itemAliases); + this.expandStructFields = expandStructFields; requireNonNull(deriveRowType(), "invalid child rowType"); } @@ -88,7 +117,8 @@ public Uncollect(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, */ public Uncollect(RelInput input) { this(input.getCluster(), input.getTraitSet(), input.getInput(), - input.getBoolean("withOrdinality", false), Collections.emptyList()); + input.getBoolean("withOrdinality", false), Collections.emptyList(), + input.getBoolean("expandStructFields", true)); } /** @@ -111,6 +141,28 @@ public static Uncollect create( return new Uncollect(cluster, traitSet, input, withOrdinality, itemAliases); } + /** + * Creates an Uncollect. + * + * @param traitSet Trait set + * @param input Input relational expression + * @param withOrdinality Whether output should contain an ORDINALITY column + * @param itemAliases Aliases for the operand items + * @param expandStructFields If true, a collection whose element type is a struct + * produces one output column per struct field; if false, + * a single column typed as the whole element + */ + public static Uncollect create( + RelTraitSet traitSet, + RelNode input, + boolean withOrdinality, + List itemAliases, + boolean expandStructFields) { + final RelOptCluster cluster = input.getCluster(); + return new Uncollect(cluster, traitSet, input, withOrdinality, itemAliases, + expandStructFields); + } + //~ Methods ---------------------------------------------------------------- @Override public RelNode accept(RelShuttle shuttle) { @@ -119,7 +171,8 @@ public static Uncollect create( @Override public RelWriter explainTerms(RelWriter pw) { return super.explainTerms(pw) - .itemIf("withOrdinality", withOrdinality, withOrdinality); + .itemIf("withOrdinality", withOrdinality, withOrdinality) + .itemIf("expandStructFields", expandStructFields, !expandStructFields); } @Override public final RelNode copy(RelTraitSet traitSet, @@ -129,34 +182,47 @@ public static Uncollect create( public RelNode copy(RelTraitSet traitSet, RelNode input) { assert traitSet.containsIfApplicable(Convention.NONE); - return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases); - } - - @Override protected RelDataType deriveRowType() { - return deriveUncollectRowType(input, withOrdinality, itemAliases); + return new Uncollect(getCluster(), traitSet, input, withOrdinality, itemAliases, + expandStructFields); } /** * Returns the row type returned by applying the 'UNNEST' operation to a * relational expression. * - *

Each column in the relational expression must be a multiset of - * structs or an array. The return type is the combination of expanding - * element types from each column, plus an ORDINALITY column if {@code - * withOrdinality}. If {@code itemAliases} is not empty, the element types - * would not expand, each column element outputs as a whole (the return - * type has same column types as input type). + * @deprecated Construct an {@link Uncollect} and call + * {@link #getRowType()} instead. */ + @Deprecated // to be removed before 2.0 public static RelDataType deriveUncollectRowType(RelNode rel, boolean withOrdinality, List itemAliases) { - RelDataType inputType = rel.getRowType(); + return new Uncollect(rel.getCluster(), rel.getTraitSet(), rel, + withOrdinality, itemAliases).getRowType(); + } + + /** + * Returns the row type of the 'UNNEST' operation. + * + *

Each column in the input relational expression must be a multiset of + * structs or an array. The return type is the combination of expanding + * element types from each column, plus an ORDINALITY column if {@code + * withOrdinality}. + * + *

{@code expandStructFields} controls the expansion of struct element + * types: if {@code true}, one output column per struct field; if {@code + * false}, a single column typed as the whole element. Maps always expand + * into a key and a value column. {@code itemAliases}, when not empty, + * names the non-expanded element columns. + */ + @Override protected RelDataType deriveRowType() { + RelDataType inputType = input.getRowType(); assert inputType.isStruct() : inputType + " is not a struct"; boolean requireAlias = !itemAliases.isEmpty(); assert !requireAlias || itemAliases.size() == inputType.getFieldCount(); final List fields = inputType.getFieldList(); - final RelDataTypeFactory typeFactory = rel.getCluster().getTypeFactory(); + final RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); final RelDataTypeFactory.Builder builder = typeFactory.builder(); if (fields.size() == 1 @@ -192,12 +258,7 @@ public static RelDataType deriveUncollectRowType(RelNode rel, throw RESOURCE.unnestArgument().ex(); } boolean isNullable = componentType.isNullable() || padNullable; - if (requireAlias) { - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) - : componentType; - builder.add(itemAliases.get(i), colType); - } else if (componentType.isStruct()) { + if (expandStructFields && componentType.isStruct()) { for (RelDataTypeField fieldInfo : componentType.getFieldList()) { RelDataType fieldType = fieldInfo.getType(); if (isNullable) { @@ -206,11 +267,18 @@ public static RelDataType deriveUncollectRowType(RelNode rel, builder.add(fieldInfo.getName(), fieldType); } } else { - // Element type is not a record, use the field name of the element directly - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) + // A single column typed as the whole element, named by the item + // alias when present, otherwise by the collection field's name. + RelDataType elementType = componentType.isStruct() + ? typeFactory.builder().kind(componentType.getStructKind()) + .addAll(componentType.getFieldList()).build() : componentType; - builder.add(field.getName(), colType); + // A NULL collection element becomes a NULL value in this column, so + // the column is nullable whenever the element type is. + RelDataType colType = isNullable + ? typeFactory.enforceTypeWithNullability(elementType, true) + : elementType; + builder.add(requireAlias ? itemAliases.get(i) : field.getName(), colType); } } } diff --git a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java index d7cee2dae7ac..4ff564f1fd54 100644 --- a/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java +++ b/core/src/main/java/org/apache/calcite/rel/logical/ToLogicalConverter.java @@ -41,8 +41,6 @@ import org.apache.calcite.rel.core.Window; import org.apache.calcite.tools.RelBuilder; -import java.util.Collections; - /** * Shuttle to convert any rel plan to a plan with all logical nodes. */ @@ -191,7 +189,8 @@ public ToLogicalConverter(RelBuilder relBuilder) { final Uncollect uncollect = (Uncollect) relNode; final RelNode input = visit(uncollect.getInput()); return Uncollect.create(input.getTraitSet(), input, - uncollect.withOrdinality, Collections.emptyList()); + uncollect.withOrdinality, uncollect.getItemAliases(), + uncollect.expandStructFields); } throw new AssertionError("Need to implement logical converter for " diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java index ed509b6d60b6..176be5cfec66 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java @@ -257,7 +257,7 @@ public static RelNode fromMutable(MutableRel node, RelBuilder relBuilder) { final MutableUncollect uncollect = (MutableUncollect) node; final RelNode child = fromMutable(uncollect.getInput(), relBuilder); return Uncollect.create(child.getTraitSet(), child, uncollect.withOrdinality, - Collections.emptyList()); + Collections.emptyList(), uncollect.expandStructFields); } case WINDOW: { final MutableWindow window = (MutableWindow) node; @@ -378,7 +378,8 @@ public static MutableRel toMutable(RelNode rel) { if (rel instanceof Uncollect) { final Uncollect uncollect = (Uncollect) rel; final MutableRel input = toMutable(uncollect.getInput()); - return MutableUncollect.of(uncollect.getRowType(), input, uncollect.withOrdinality); + return MutableUncollect.of(uncollect.getRowType(), input, + uncollect.withOrdinality, uncollect.expandStructFields); } if (rel instanceof Window) { final Window window = (Window) rel; diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java index 594d109b5d69..bae3854f6948 100644 --- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java +++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableUncollect.java @@ -25,15 +25,17 @@ /** Mutable equivalent of {@link org.apache.calcite.rel.core.Uncollect}. */ public class MutableUncollect extends MutableSingleRel { public final boolean withOrdinality; + public final boolean expandStructFields; private MutableUncollect(RelDataType rowType, - MutableRel input, boolean withOrdinality) { + MutableRel input, boolean withOrdinality, boolean expandStructFields) { super(MutableRelType.UNCOLLECT, rowType, input); this.withOrdinality = withOrdinality; + this.expandStructFields = expandStructFields; } /** - * Creates a MutableUncollect. + * Creates a MutableUncollect that expands struct elements. * * @param rowType Row type * @param input Input relational expression @@ -42,26 +44,47 @@ private MutableUncollect(RelDataType rowType, */ public static MutableUncollect of(RelDataType rowType, MutableRel input, boolean withOrdinality) { - return new MutableUncollect(rowType, input, withOrdinality); + return of(rowType, input, withOrdinality, true); + } + + /** + * Creates a MutableUncollect. + * + * @param rowType Row type + * @param input Input relational expression + * @param withOrdinality Whether the output contains an extra + * {@code ORDINALITY} column + * @param expandStructFields If true, a collection whose element type + * is a struct produces one output column per + * struct field; if false, a single column + * typed as the whole element + */ + public static MutableUncollect of(RelDataType rowType, + MutableRel input, boolean withOrdinality, boolean expandStructFields) { + return new MutableUncollect(rowType, input, withOrdinality, + expandStructFields); } @Override public boolean equals(@Nullable Object obj) { return obj == this || obj instanceof MutableUncollect && withOrdinality == ((MutableUncollect) obj).withOrdinality + && expandStructFields == ((MutableUncollect) obj).expandStructFields && input.equals(((MutableUncollect) obj).input); } @Override public int hashCode() { - return Objects.hash(input, withOrdinality); + return Objects.hash(input, withOrdinality, expandStructFields); } @Override public StringBuilder digest(StringBuilder buf) { - return buf.append("Uncollect(withOrdinality: ") - .append(withOrdinality).append(")"); + return buf.append("Uncollect(withOrdinality: ").append(withOrdinality) + .append(", expandStructFields: ").append(expandStructFields) + .append(")"); } @Override public MutableRel clone() { - return MutableUncollect.of(rowType, input.clone(), withOrdinality); + return MutableUncollect.of(rowType, input.clone(), withOrdinality, + expandStructFields); } } diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index c6e1a4dbdcc5..c5047574a3d0 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -337,6 +337,9 @@ ExInst invalidCompare(String a0, String a1, String a2, @BaseMessage("Cannot specify condition (NATURAL keyword, or ON or USING clause) following CROSS JOIN") ExInst crossJoinDisallowsCondition(); + @BaseMessage("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not ''{0}''") + ExInst unnestInvalidJoinType(String a0); + @BaseMessage("Cannot specify NATURAL keyword with ON or USING clause") ExInst naturalDisallowsOnOrUsing(); diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 56e6bdb42298..4b9b48041fad 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -209,6 +209,23 @@ public class SqlFunctions { private static final Function1, Enumerable> LIST_AS_ENUMERABLE = a0 -> a0 == null ? Linq4j.emptyEnumerable() : Linq4j.asEnumerable(a0); + /** Like {@link #LIST_AS_ENUMERABLE}, for a collection whose struct elements + * are kept whole: each element is converted to an Object[] struct value. */ + private static final Function1, Enumerable<@Nullable Object>> + STRUCT_LIST_AS_ENUMERABLE = + a0 -> a0 == null ? Linq4j.emptyEnumerable() + : Linq4j.asEnumerable(a0).<@Nullable Object>select(SqlFunctions::structValue); + + /** Converts one element of a collection of structs to its Object[] struct + * value. Elements arrive as List or as Object[]; null elements stay null. */ + @SuppressWarnings("rawtypes") + private static @Nullable Object structValue(@Nullable Object element) { + if (element == null || element instanceof Object[]) { + return element; + } + return ((List) element).toArray(); + } + @SuppressWarnings("unused") private static final Function1> ARRAY_CARTESIAN_PRODUCT = SqlFunctions::arrayCartesianProduct; @@ -7590,8 +7607,15 @@ public static Function1>> flatZip( // Simple unnest without ordinality //noinspection unchecked return (Function1) LIST_AS_ENUMERABLE; + } else if (!withOrdinality && inputTypes[0] == FlatProductInputType.STRUCT) { + // A single collection of structs kept whole, without ordinality: the + // output row type has a single (ROW-typed) column, so PhysTypeImpl + // optimizes the row format down to SCALAR, under which rows are bare + // struct values rather than singleton lists. + //noinspection unchecked + return (Function1) STRUCT_LIST_AS_ENUMERABLE; } else { - // unnest with ordinality for a single scalar column + // unnest with ordinality for a single column return row -> z2(new Object[] { row }, fieldCounts, withOrdinality, inputTypes); } } @@ -7604,9 +7628,10 @@ public static Function1>> flatZip( * padding shorter collections with {@code NULL}. * * @param lists one element per collection (scalar list, struct list, or map) - * @param fieldCounts output column count for each collection (-1 for a collection of scalars) + * @param fieldCounts output column count for each collection (-1 for a collection + * of scalars or of structs kept whole) * @param withOrdinality whether to append a 1-based ordinality column - * @param inputTypes type of elements in each collection (SCALAR, LIST, or MAP) + * @param inputTypes type of elements in each collection (SCALAR, LIST, STRUCT, or MAP) */ @SuppressWarnings("rawtypes") private static Enumerable> z2( @@ -7626,6 +7651,17 @@ private static Enumerable> z2( enumerators.add(Linq4j.transform(Linq4j.enumerator(list), FlatLists::of)); widths[i] = 1; break; + case STRUCT: + // A struct element kept whole occupies a single output column, like a + // scalar element, but its value must be converted to Object[]. + @SuppressWarnings("unchecked") List structList = + (List) inputObject; + @SuppressWarnings("unchecked") Enumerator> structEnumerator = + (Enumerator) Linq4j.transform(Linq4j.enumerator(structList), + (Object e) -> FlatLists.ofSingle(structValue(e))); + enumerators.add(structEnumerator); + widths[i] = 1; + break; case LIST: @SuppressWarnings("unchecked") List> listList = (List>) inputObject; @@ -7766,7 +7802,10 @@ private static class ZipPaddedEnumerator int width = widths[i]; if (!endOfCollection[i]) { final Object elemRow = enumerators.get(i).current(); - if (elemRow instanceof Object[]) { + if (elemRow == null) { + // A NULL struct element expands to a row of NULLs, one per field. + Arrays.fill(flatElements, column, column + width, null); + } else if (elemRow instanceof Object[]) { final Object[] arr = (Object[]) elemRow; for (int p = 0; p < width; p++) { flatElements[column + p] = p < arr.length ? arr[p] : null; @@ -7900,7 +7939,7 @@ public enum JsonScope { /** Type of argument passed into {@link #flatZip}. */ public enum FlatProductInputType { - SCALAR, LIST, MAP + SCALAR, LIST, MAP, STRUCT } /** Type of part to extract passed into {@link ParseUrlFunction#parseUrl}. */ diff --git a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java index af1753f8e8b5..61ada978234f 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlUnnestOperator.java @@ -99,6 +99,9 @@ public SqlUnnestOperator(boolean withOrdinality) { } else { RelDataType componentType = requireNonNull(type.getComponentType(), "componentType"); boolean isNullable = componentType.isNullable() || padNullable; + // Whether a struct element expands into one column per field depends + // on the SQL conformance; allowAliasUnnestItems describes how + // collections of ROW values are expanded. if (!allowAliasUnnestItems(opBinding) && componentType.isStruct()) { for (RelDataTypeField field : componentType.getFieldList()) { RelDataType fieldType = field.getType(); @@ -108,9 +111,15 @@ public SqlUnnestOperator(boolean withOrdinality) { builder.add(field.getName(), fieldType); } } else { - RelDataType colType = padNullable - ? typeFactory.enforceTypeWithNullability(componentType, true) + RelDataType elementType = componentType.isStruct() + ? typeFactory.builder().kind(componentType.getStructKind()) + .addAll(componentType.getFieldList()).build() : componentType; + // A NULL collection element becomes a NULL value in this column, so + // the column is nullable whenever the element type is. + RelDataType colType = isNullable + ? typeFactory.enforceTypeWithNullability(elementType, true) + : elementType; builder.add(SqlUtil.deriveAliasFromOrdinal(operand), colType); } } diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index 023de1b77f77..7021696b679c 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -4116,6 +4116,20 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { } } + // UNNEST on the right side is only meaningful with INNER, LEFT, CROSS, or COMMA. + if (isUnnestNode(right)) { + switch (joinType) { + case INNER: + case LEFT: + case CROSS: + case COMMA: + break; + default: + throw newValidationError(join.getJoinTypeNode(), + RESOURCE.unnestInvalidJoinType(joinType.name())); + } + } + // Which join types require/allow a ON/USING condition, or allow // a NATURAL keyword? switch (joinType) { @@ -4192,6 +4206,22 @@ protected void validateJoin(SqlJoin join, SqlValidatorScope scope) { } } + /** + * Returns whether {@code node} is (or wraps, via AS or LATERAL) an + * {@code UNNEST} call. + */ + private static boolean isUnnestNode(SqlNode node) { + switch (node.getKind()) { + case UNNEST: + return true; + case AS: + case LATERAL: + return isUnnestNode(((SqlCall) node).operand(0)); + default: + return false; + } + } + /** * Shuttle which determines whether all SqlCalls that are * comparisons are comparing columns from both namespaces. diff --git a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index 182ed7a254a3..3ca7ae44f854 100644 --- a/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -2888,7 +2888,8 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f // so Uncollect's row type stays aligned with the validator. List itemAliases; if (fieldNames != null) { - itemAliases = fieldNames; + // do not include the ordinality column name + itemAliases = fieldNames.subList(0, nodes.size()); } else { itemAliases = new ArrayList<>(nodes.size()); for (int i = 0; i < nodes.size(); i++) { @@ -2899,6 +2900,7 @@ private void convertUnnest(Blackboard bb, SqlCall call, @Nullable List f .push(child) .project(exprs) .uncollect(itemAliases, operator.withOrdinality) + .let(r -> fieldNames == null ? r : r.rename(fieldNames)) .build(); } else { // REVIEW danny 2020-04-26: should we unify the normal field aliases and diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index f4f16d73266a..a90099d7cb91 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -115,6 +115,7 @@ AliasListDuplicate=Duplicate name ''{0}'' in column alias list JoinRequiresCondition=INNER, LEFT, RIGHT, FULL, or ASOF join requires a condition (NATURAL keyword or ON or USING clause) DisallowsQualifyingCommonColumn=Cannot qualify common column ''{0}'' CrossJoinDisallowsCondition=Cannot specify condition (NATURAL keyword, or ON or USING clause) following CROSS JOIN +UnnestInvalidJoinType=UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not ''{0}'' NaturalDisallowsOnOrUsing=Cannot specify NATURAL keyword with ON or USING clause ColumnInUsingNotUnique=Column name ''{0}'' in NATURAL join or USING clause is not unique on one side of join NaturalOrUsingColumnNotCompatible=Column ''{0}'' matched using NATURAL keyword or USING clause has incompatible types: cannot compare ''{1}'' to ''{2}'' diff --git a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java index b878bfb085e0..bbb547b85833 100644 --- a/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java +++ b/core/src/test/java/org/apache/calcite/test/CoreQuidemTest.java @@ -185,6 +185,13 @@ protected Collection data() { .with(CalciteAssert.SchemaSpec.STEELWHEELS) .with(Lex.BIG_QUERY)) .connect(); + case "hr-presto": + // Same as "hr", but uses PRESTO conformance, under which + // UNNEST(array) AS alias does not expand struct elements. + return customize(CalciteAssert.hr() + .with(CalciteConnectionProperty.CONFORMANCE, + SqlConformanceEnum.PRESTO)) + .connect(); default: return super.connect(name, reference); } diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 9231437cf8ea..e3827c12530e 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -46,6 +46,7 @@ import static org.apache.calcite.avatica.util.DateTimeUtils.timestampStringToUnixDate; import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.LIST; import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.SCALAR; +import static org.apache.calcite.runtime.SqlFunctions.FlatProductInputType.STRUCT; import static org.apache.calcite.runtime.SqlFunctions.arraysOverlap; import static org.apache.calcite.runtime.SqlFunctions.charLength; import static org.apache.calcite.runtime.SqlFunctions.concat; @@ -2238,4 +2239,85 @@ private static List> zipScalars( assertThat(rows.get(0), is(list(1, 2, 10, 20))); assertThat(rows.get(1), is(Arrays.asList(3, 4, null, null))); } + + /** The runtime representation of {@code ARRAY[ROW(1, 'x'), ROW(2, 'y')]}: + * a list whose elements are the field lists of each ROW. */ + private static List> rowArray() { + return Arrays.asList(FlatLists.of(1, "x"), FlatLists.of(2, "y")); + } + + @Test void testZipPaddedWholeStructElements() { + // Models the Trino semantics of + // UNNEST(ARRAY[ROW(1, 'x'), ROW(2, 'y')], ARRAY[10, 20]) AS t(s, i): + // the STRUCT collection keeps each ROW element whole, so column s holds + // the element as an Object[]; the scalar column i zips alongside. + @SuppressWarnings({"rawtypes", "unchecked"}) + final Function1>> fn = + SqlFunctions.flatZip( + new int[]{-1, -1}, // one output column per collection + false, // no ordinality + new SqlFunctions.FlatProductInputType[]{STRUCT, SCALAR}); + + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{rowArray(), Arrays.asList(10, 20)})) { + rows.add(new ArrayList<>(row)); + } + + // Expected rows: ({1, 'x'}, 10) and ({2, 'y'}, 20). + assertThat(rows, hasSize(2)); + assertArrayEquals(new Object[]{1, "x"}, (Object[]) rows.get(0).get(0)); + assertThat(rows.get(0).get(1), is(10)); + assertArrayEquals(new Object[]{2, "y"}, (Object[]) rows.get(1).get(0)); + assertThat(rows.get(1).get(1), is(20)); + } + + @Test void testZipPaddedNullStructElement() { + // A null element of an expanded ROW ARRAY is a null List, which + // must be expanded to one null per ROW field rather than dereferencing the list. + @SuppressWarnings({"rawtypes", "unchecked"}) + final Function1>> fn = + SqlFunctions.flatZip( + new int[]{2, -1}, // two columns from the struct, one scalar column + false, // no ordinality + new SqlFunctions.FlatProductInputType[]{LIST, SCALAR}); + + final List> rows = new ArrayList<>(); + for (FlatLists.ComparableList row + : fn.apply(new Object[]{ + Arrays.asList(FlatLists.of(1, "x"), null), + Arrays.asList(10, 20)})) { + rows.add(new ArrayList<>(row)); + } + + assertThat(rows, hasSize(2)); + assertThat(rows.get(0), is(Arrays.asList(1, "x", 10))); + assertThat(rows.get(1), is(Arrays.asList(null, null, 20))); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test void testFlatZipSingleWholeStructCollection() { + // Models the Trino semantics of + // UNNEST(ARRAY[ROW(1, 'x'), ROW(2, 'y')]) AS t(s): + // the output has the single ROW-typed column s, which PhysTypeImpl stores + // in SCALAR row format, so each output row is the bare Object[] struct + // value rather than a singleton list. + final Function1>> fn = + SqlFunctions.flatZip( + new int[]{-1}, false, + new SqlFunctions.FlatProductInputType[]{STRUCT}); + + final List rows = new ArrayList<>(); + for (Object row : (Enumerable) fn.apply(rowArray())) { + rows.add(row); + } + + // Expected rows: {1, 'x'} and {2, 'y'}. + assertThat(rows, hasSize(2)); + assertArrayEquals(new Object[]{1, "x"}, (Object[]) rows.get(0)); + assertArrayEquals(new Object[]{2, "y"}, (Object[]) rows.get(1)); + + // UNNEST of a null array yields no rows. + assertThat(((Enumerable) fn.apply(null)).any(), is(false)); + } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 8f95099ab952..018c27e898f1 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -9855,6 +9855,30 @@ void testGroupExpressionEquivalenceParams() { .fails("Column 'ORDINALITY' not found in any table"); } + /** UNNEST is valid with INNER, LEFT, CROSS, and COMMA joins; + * all other join kinds must be rejected by the validator. */ + @Test void testUnnestJoinType() { + // Allowed join kinds — these must all validate without error. + sql("select * from dept inner join unnest(array[1, 2]) as u(x) on true").ok(); + sql("select * from dept left join unnest(array[1, 2]) as u(x) on true").ok(); + sql("select * from dept cross join unnest(array[1, 2]) as u(x)").ok(); + sql("select * from dept, unnest(array[1, 2]) as u(x)").ok(); + + // LATERAL wrapping must also be allowed for valid join kinds. + sql("select * from dept cross join lateral unnest(array[1, 2]) as u(x)").ok(); + sql("select * from dept left join lateral unnest(array[1, 2]) as u(x) on true").ok(); + + // Disallowed join kinds — validator must reject these. + sql("select * from dept right ^join^ unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'RIGHT'"); + sql("select * from dept full ^join^ unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'FULL'"); + + // LATERAL wrapping must also be rejected for invalid join kinds. + sql("select * from dept right ^join^ lateral unnest(array[1, 2]) as u(x) on true") + .fails("UNNEST is only supported with INNER, LEFT, CROSS, or COMMA join, not 'RIGHT'"); + } + @Test void unnestMapMustNameColumnsKeyAndValueWhenNotAliased() { sql("select * from unnest(map[1, 12, 2, 22])") .type("RecordType(INTEGER NOT NULL KEY, INTEGER NOT NULL VALUE) NOT NULL"); diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index aafc9c11efd3..217b2bbc03b0 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -336,7 +336,7 @@ from UNNEST(ARRAY[1, 2, 3]) as t]]> @@ -348,7 +348,7 @@ LogicalProject(T=[$0]) LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2, 3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS], EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -365,7 +365,7 @@ from dept_nested_expanded as d CROSS JOIN LogicalProject(DEPTNO=[$0], E=[$5], EMPNO=[$6.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2, 3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS], EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -382,7 +382,7 @@ from dept_nested_expanded as d CROSS JOIN LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -399,7 +399,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -421,7 +421,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], EMPNO=[$5.EMPNO]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{2}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(EMPLOYEES=[$cor0.EMPLOYEES]) LogicalValues(tuples=[[{ 0 }]]) ]]> @@ -438,7 +438,7 @@ from dept_nested_expanded as d, LogicalProject(DEPTNO=[$0], A=[$5]) LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{3}]) LogicalTableScan(table=[[CATALOG, SALES, DEPT_NESTED_EXPANDED]]) - Uncollect + Uncollect(expandStructFields=[false]) LogicalProject(ADMINS=[$cor0.ADMINS]) LogicalValues(tuples=[[{ 0 }]]) ]]> diff --git a/core/src/test/resources/sql/unnest.iq b/core/src/test/resources/sql/unnest.iq index 8defd3b01ce6..054234fcbed3 100644 --- a/core/src/test/resources/sql/unnest.iq +++ b/core/src/test/resources/sql/unnest.iq @@ -626,4 +626,193 @@ WHERE ( !ok +!use scott + +# Standard UNNEST semantics: struct elements expand into one column per field, so a +# NULL element yields a row of NULLs. +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | x | +| | | ++---+---+ +(2 rows) + +!ok + +# Same as previous WITH ORDINALITY +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) WITH ORDINALITY AS t(a, b, o); ++---+---+---+ +| A | B | O | ++---+---+---+ +| 1 | x | 1 | +| | | 2 | ++---+---+---+ +(2 rows) + +!ok + +# A result column is nullable whenever the ROW is nullable or the field is. +# Here no element is NULL, so the ROW is NOT NULL and each column keeps its +# own field nullability. +SELECT * FROM UNNEST(ARRAY[ROW(1, CAST(NULL AS INTEGER))]) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | | ++---+---+ +(1 row) + +!ok +A INTEGER(10) NOT NULL +B INTEGER(10) +!type + +# Same fields, but a NULL element makes the ROW nullable, so column A is also nullable +SELECT * FROM UNNEST(ARRAY[ + ROW(1, CAST(NULL AS INTEGER)), + CAST(NULL AS ROW(a INTEGER, b INTEGER))]) AS t(a, b); ++---+---+ +| A | B | ++---+---+ +| 1 | | +| | | ++---+---+ +(2 rows) + +!ok +A INTEGER(10) +B INTEGER(10) +!type + +# Tests for [CALCITE-7669] Uncollect should support the Trino semantics of UNNEST +# PRESTO conformance: UNNEST(array) AS t(col) does not expand a struct +# element into its fields; col holds the whole struct, accessed by dot +# notation. +!use hr-presto + +# INNER comma-join: Marketing (0 employees) is dropped. +select d."name" as dept, e.emp."name" as ename, e.emp."empid" as empid +from "hr"."depts" as d, +UNNEST(d."employees") as e(emp); ++-------+-----------+-------+ +| DEPT | ENAME | EMPID | ++-------+-----------+-------+ +| HR | Eric | 200 | +| Sales | Bill | 100 | +| Sales | Sebastian | 150 | ++-------+-----------+-------+ +(3 rows) + +!ok + +# WITH ORDINALITY: the struct column stays whole; ordinality still expands. +select e.emp."name" as ename, e.rn +from "hr"."depts" as d, +UNNEST(d."employees") WITH ORDINALITY as e(emp, rn); ++-----------+----+ +| ENAME | RN | ++-----------+----+ +| Eric | 1 | +| Bill | 1 | +| Sebastian | 2 | ++-----------+----+ +(3 rows) + +!ok + +# Output a whole struct column +select d."name" as dept, e.emp as emp +from "hr"."depts" as d, +UNNEST(d."employees") as e(emp); ++-------+------------------------------------+ +| DEPT | EMP | ++-------+------------------------------------+ +| HR | {200, 20, Eric, 8000.0, 500} | +| Sales | {100, 10, Bill, 10000.0, 1000} | +| Sales | {150, 10, Sebastian, 7000.0, null} | ++-------+------------------------------------+ +(3 rows) + +!ok + +WITH data AS ( + SELECT ARRAY[ + ROW(1, 'Alice'), + ROW(2, 'Bob'), + ROW(3, 'Carol'), + ROW(NULL, 'Dan'), + NULL + ] AS people +) +SELECT p.* +FROM data, UNNEST(people) AS p(p); ++-------------+ +| P | ++-------------+ +| {1, Alice} | +| {2, Bob} | +| {3, Carol} | +| {null, Dan} | +| | ++-------------+ +(5 rows) + +!ok + +# Nested records: a NULL nested-record field, and a bare NULL array element. +SELECT * FROM UNNEST(ARRAY[ + ROW(ROW(1, 'a'), 10), + ROW(ROW(NULL, 'b'), 20), + ROW(NULL, 30), + NULL]) AS p(p); ++-----------------+ +| P | ++-----------------+ +| {null, 30} | +| {{1, a}, 10} | +| {{null, b}, 20} | +| | ++-----------------+ +(4 rows) + +!ok + +# Only one value is emitted per element, so a NULL element is a single NULL +# rather than a row of NULLs. +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) AS t(r); ++--------+ +| R | ++--------+ +| {1, x} | +| | ++--------+ +(2 rows) + +!ok +# The column is nullable: a NULL element yields NULL here. +R STRUCT +!type + +# Same, WITH ORDINALITY: exercises z2 rather than the single-collection path. +SELECT * FROM UNNEST(ARRAY[ + ROW(1, 'x'), + CAST(NULL AS ROW(a INTEGER, b CHAR(1)))]) WITH ORDINALITY AS t(r, o); ++--------+---+ +| R | O | ++--------+---+ +| {1, x} | 1 | +| | 2 | ++--------+---+ +(2 rows) + +!ok + # End unnest.iq