From b2be7e1dacaefd93c6aaccc1192694610e4d2457 Mon Sep 17 00:00:00 2001 From: Paul Jackson Date: Wed, 1 Nov 2023 15:42:23 -0400 Subject: [PATCH] [CALCITE-6044] RelMetadataQuery should regard single-row relational expressions as unique A single-row relation can result from a LIMIT 1 or an aggregation without GROUP BY. Every column in one of these relations should be unique by virtue of having a max row count of 1. When joining with a single-row relation on a key field, the join result should no longer require that key field for uniqueness. For example, suppose the emp table had a composite key (empno,hiredate). If we join on hiredate=max(hiredate) then empno alone should be a unique column. Close apache/calcite#3495 --- .../calcite/rel/metadata/BuiltInMetadata.java | 10 +- .../rel/metadata/RelMdColumnUniqueness.java | 145 ++++++-- .../calcite/rel/metadata/RelMdPredicates.java | 43 ++- .../calcite/rel/metadata/RelMdUniqueKeys.java | 202 ++++++++++- .../rel/metadata/RelMetadataQuery.java | 4 +- .../apache/calcite/test/RelMetadataTest.java | 330 +++++++++++++++++- .../apache/calcite/test/RelOptRulesTest.java | 2 +- .../GeneratedMetadata_PredicatesHandler.java | 2 + .../GeneratedMetadata_UniqueKeysHandler.java | 2 + .../apache/calcite/test/RelOptRulesTest.xml | 2 +- .../calcite/test/SqlToRelConverterTest.xml | 9 +- core/src/test/resources/sql/conditions.iq | 7 +- core/src/test/resources/sql/sub-query.iq | 26 +- 13 files changed, 718 insertions(+), 66 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java index cbd7f38f210b..be83c9bf8420 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/BuiltInMetadata.java @@ -83,13 +83,21 @@ public interface UniqueKeys extends Metadata { * represented as an {@link org.apache.calcite.util.ImmutableBitSet}, where * each bit position represents a 0-based output column ordinal. * + *

Note that a unique key plus other columns is still unique. + * Therefore, all columns are unique in a table with a unique key + * consisting of the empty set, as is the case for zero-row and + * single-row tables. The converse is not true: a table with all + * columns unique does necessary have the empty set as a key - + * that is never true with multi-row tables. + * *

Nulls can be ignored if the relational expression has filtered out * null values. * * @param ignoreNulls if true, ignore null values when determining * whether the keys are unique * @return set of keys, or null if this information cannot be determined - * (whereas empty set indicates definitely no keys at all) + * (whereas empty set indicates definitely no keys at all, and a set + * containing the empty set implies every column is unique) */ @Nullable Set getUniqueKeys(boolean ignoreNulls); diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java index bb38c3dbe72f..e4e0b3282252 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdColumnUniqueness.java @@ -19,6 +19,7 @@ import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.volcano.RelSubset; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttleImpl; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.convert.Converter; import org.apache.calcite.rel.core.Aggregate; @@ -37,6 +38,7 @@ import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Values; +import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; @@ -44,12 +46,19 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; +import org.apache.calcite.rex.RexSlot; +import org.apache.calcite.rex.RexSubQuery; +import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; +import org.apache.commons.lang3.mutable.MutableBoolean; + import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.checkerframework.checker.nullness.qual.Nullable; @@ -57,6 +66,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * RelMdColumnUniqueness supplies a default implementation of @@ -68,6 +78,15 @@ public class RelMdColumnUniqueness ReflectiveRelMetadataProvider.reflectiveSource( new RelMdColumnUniqueness(), BuiltInMetadata.ColumnUniqueness.Handler.class); + /** + * The set of aggregate functions A such that if x is unique then A(x) will also be unique. + * An aggregate function with this property is called 'passthrough'. This quality is not + * guaranteed in the presence of an OVER clause. NOTE: if a multi-argument function is added, + * methods that use this Set must be enhanced to select the appropriate column to pass through. + */ + static final Set PASSTHROUGH_AGGREGATIONS = + ImmutableSet.of(SqlKind.MIN, SqlKind.MAX, SqlKind.ANY_VALUE); + //~ Constructors ----------------------------------------------------------- private RelMdColumnUniqueness() {} @@ -151,6 +170,10 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, public @Nullable Boolean areColumnsUnique(Sort rel, RelMetadataQuery mq, ImmutableBitSet columns, boolean ignoreNulls) { + Double maxRowCount = mq.getMaxRowCount(rel); + if (maxRowCount != null && maxRowCount <= 1.0d) { + return true; + } columns = decorateWithConstantColumnsFromPredicates(columns, rel, mq); return mq.areColumnsUnique(rel.getInput(), columns, ignoreNulls); } @@ -266,11 +289,6 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, } } - // If no columns can affect uniqueness, then return unknown - if (childColumns.cardinality() == 0) { - return null; - } - return mq.areColumnsUnique(rel.getInput(), childColumns.build(), ignoreNulls); } @@ -278,9 +296,6 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, public @Nullable Boolean areColumnsUnique(Join rel, RelMetadataQuery mq, ImmutableBitSet columns, boolean ignoreNulls) { columns = decorateWithConstantColumnsFromPredicates(columns, rel, mq); - if (columns.cardinality() == 0) { - return false; - } final RelNode left = rel.getLeft(); final RelNode right = rel.getRight(); @@ -290,13 +305,13 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, return mq.areColumnsUnique(left, columns, ignoreNulls); } + final int leftColumnCount = rel.getLeft().getRowType().getFieldCount(); // Divide up the input column mask into column masks for the left and // right sides of the join final Pair leftAndRightColumns = - splitLeftAndRightColumns(rel.getLeft().getRowType().getFieldCount(), - columns); - final ImmutableBitSet leftColumns = leftAndRightColumns.left; - final ImmutableBitSet rightColumns = leftAndRightColumns.right; + splitLeftAndRightColumns(leftColumnCount, columns); + ImmutableBitSet leftColumns = leftAndRightColumns.left; + ImmutableBitSet rightColumns = leftAndRightColumns.right; // for FULL OUTER JOIN if columns contain column from both inputs it is not // guaranteed that the result will be unique @@ -305,6 +320,18 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, return false; } + final JoinInfo joinInfo = rel.analyzeCondition(); + + // Joining with a singleton constrains the keys on the other table + final Double rightMaxRowCount = mq.getMaxRowCount(right); + if (rightMaxRowCount != null && rightMaxRowCount <= 1.0) { + leftColumns = leftColumns.union(joinInfo.leftSet()); + } + final Double leftMaxRowCount = mq.getMaxRowCount(left); + if (leftMaxRowCount != null && leftMaxRowCount <= 1.0) { + rightColumns = rightColumns.union(joinInfo.rightSet()); + } + // If the original column mask contains columns from both the left and // right hand side, then the columns are unique if and only if they're // unique for their respective join inputs @@ -325,7 +352,6 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, // the columns are unique for the entire join if they're unique for // the corresponding join input, provided that input is not null // generating. - final JoinInfo joinInfo = rel.analyzeCondition(); if (leftColumns.cardinality() > 0) { if (rel.getJoinType().generatesNullsOnLeft()) { return false; @@ -348,11 +374,15 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, return leftJoinColsUnique && rightUnique; } - throw new AssertionError(); + return false; } public @Nullable Boolean areColumnsUnique(Aggregate rel, RelMetadataQuery mq, ImmutableBitSet columns, boolean ignoreNulls) { + Double maxRowCount = mq.getMaxRowCount(rel); + if (maxRowCount != null && maxRowCount <= 1.0d) { + return true; + } if (Aggregate.isSimple(rel) || ignoreNulls) { columns = decorateWithConstantColumnsFromPredicates(columns, rel, mq); // group by keys form a unique key @@ -364,6 +394,22 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, return false; } + if (Aggregate.isSimple(rel)) { + // Map columns to input columns + ImmutableBitSet inputCols = ImmutableBitSet.builder() + .addAll(columns.intersect(rel.getGroupSet())) + .addAll(columns.toList() + .stream() + .map(col -> col - rel.getGroupSet().length()) + .filter(col -> col >= 0) + .map(col -> rel.getAggCallList().get(col)) + .filter(call -> PASSTHROUGH_AGGREGATIONS.contains(call.getAggregation().getKind())) + .map(call -> call.getArgList().get(0)) + .collect(Collectors.toSet())) + .build(); + return mq.areColumnsUnique(rel.getInput(), inputCols, ignoreNulls); + } + final ImmutableBitSet commonKeys = columns.intersect(groupKey); if (commonKeys.isEmpty()) { return false; @@ -380,10 +426,10 @@ public Boolean areColumnsUnique(Intersect rel, RelMetadataQuery mq, public Boolean areColumnsUnique(Values rel, RelMetadataQuery mq, ImmutableBitSet columns, boolean ignoreNulls) { - columns = decorateWithConstantColumnsFromPredicates(columns, rel, mq); if (rel.tuples.size() < 2) { return true; } + columns = decorateWithConstantColumnsFromPredicates(columns, rel, mq); final Set> set = new HashSet<>(); final List values = new ArrayList<>(columns.cardinality()); for (ImmutableList tuple : rel.tuples) { @@ -478,12 +524,7 @@ private static ImmutableBitSet decorateWithConstantColumnsFromPredicates( ImmutableBitSet checkingColumns, RelNode rel, RelMetadataQuery mq) { final RelOptPredicateList predicates = mq.getPulledUpPredicates(rel); if (!RelOptPredicateList.isEmpty(predicates)) { - final Set constantIndexes = new HashSet<>(); - predicates.constantMap.keySet().forEach(rex -> { - if (rex instanceof RexInputRef) { - constantIndexes.add(((RexInputRef) rex).getIndex()); - } - }); + ImmutableBitSet constantIndexes = getConstantColumnSet(predicates); if (!constantIndexes.isEmpty()) { return checkingColumns.union(ImmutableBitSet.of(constantIndexes)); } @@ -491,4 +532,66 @@ private static ImmutableBitSet decorateWithConstantColumnsFromPredicates( // If no constant columns deduced, return the original "checkingColumns". return checkingColumns; } + + /** + * Returns the set of columns that are set to a constant literal or a scalar query (as + * in a correlated subquery). Examples of constants are {@code x} in the following: + * 

SELECT x FROM table WHERE x = 5
+ * and + * 
SELECT x, y FROM table WHERE x = (SELECT MAX(x) FROM table)
+ * + *

NOTE: Subqueries that reference correlating variables are not considered constant: + * 

SELECT x, y FROM table A WHERE x = (SELECT MAX(x) FROM table B WHERE A.y = B.y)
+ */ + static ImmutableBitSet getConstantColumnSet(RelOptPredicateList relOptPredicateList) { + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + relOptPredicateList.constantMap.keySet() + .stream() + .filter(RexInputRef.class::isInstance) + .map(RexInputRef.class::cast) + .map(RexSlot::getIndex) + .forEach(builder::set); + + relOptPredicateList.pulledUpPredicates.forEach(rex -> { + if (rex.getKind() == SqlKind.EQUALS + || rex.getKind() == SqlKind.IS_NOT_DISTINCT_FROM) { + List ops = ((RexCall) rex).getOperands(); + RexNode op0 = ops.get(0); + RexNode op1 = ops.get(1); + addInputRefIfOtherConstant(builder, op0, op1); + addInputRefIfOtherConstant(builder, op1, op0); + } + }); + + return builder.build(); + } + + private static void addInputRefIfOtherConstant(ImmutableBitSet.Builder builder, RexNode inputRef, + RexNode other) { + if (inputRef instanceof RexInputRef + && (other.getKind() == SqlKind.LITERAL || isConstantScalarQuery(other))) { + builder.set(((RexInputRef) inputRef).getIndex()); + } + } + + /** + * Returns whether the supplied {@link RexNode} is a constant scalar subquery - one that does not + * reference any correlating variables. + */ + private static boolean isConstantScalarQuery(RexNode rexNode) { + if (rexNode.getKind() == SqlKind.SCALAR_QUERY) { + MutableBoolean hasCorrelatingVars = new MutableBoolean(false); + ((RexSubQuery) rexNode).rel.accept(new RelShuttleImpl() { + @Override public RelNode visit(final LogicalFilter filter) { + if (RexUtil.containsCorrelation(filter.getCondition())) { + hasCorrelatingVars.setTrue(); + return filter; + } + return super.visit(filter); + } + }); + return hasCorrelatingVars.isFalse(); + } + return false; + } } diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java index e25d30686f6b..dd10eedcd0dc 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdPredicates.java @@ -39,6 +39,7 @@ import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.core.Values; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexExecutor; @@ -76,10 +77,12 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static java.util.Objects.requireNonNull; @@ -365,7 +368,7 @@ public RelOptPredicateList getPredicates(Aggregate agg, RelMetadataQuery mq) { // it is not valid to pull up predicates. In particular, consider the // predicate "false": it is valid on all input rows (trivially - there are // no rows!) but not on the output (there is one row). - return RelOptPredicateList.EMPTY; + return RelOptPredicateList.of(rexBuilder, aggPullUpPredicates); } Mapping m = Mappings.create(MappingType.PARTIAL_FUNCTION, @@ -533,6 +536,44 @@ public RelOptPredicateList getPredicates(Exchange exchange, return mq.getPulledUpPredicates(input); } + /** + * Infers predicates for a Values. + * + *

The predicates on {@code T (x, y, z)} with rows + * {@code (1, 2, null), (1, 2, null), (5, 2, null)} are {@code 'y = 2'} and {@code 'z is null'}. + */ + public RelOptPredicateList getPredicates(Values values, RelMetadataQuery mq) { + ImmutableList> tuples = values.tuples; + if (tuples.size() > 0) { + Set constants = new HashSet<>(); + IntStream.range(0, tuples.size()).boxed().forEach(constants::add); + List firstTuple = new ArrayList<>(tuples.get(0)); + for (ImmutableList tuple : tuples) { + if (constants.isEmpty()) { + break; + } + for (int i = 0; i < tuple.size(); i++) { + if (!Objects.equals(tuple.get(i), firstTuple.get(i))) { + constants.remove(i); + } + } + } + RexBuilder rexBuilder = values.getCluster().getRexBuilder(); + List predicates = new ArrayList<>(); + for (int i = 0; i < firstTuple.size(); i++) { + if (constants.contains(i)) { + RexLiteral literal = firstTuple.get(i); + predicates.add( + rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + rexBuilder.makeInputRef(literal.getType(), i), + literal)); + } + } + return RelOptPredicateList.of(rexBuilder, predicates); + } + return RelOptPredicateList.EMPTY; + } + // CHECKSTYLE: IGNORE 1 /** * Returns the diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java index 447619cef2f8..392c431da13c 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdUniqueKeys.java @@ -17,9 +17,11 @@ package org.apache.calcite.rel.metadata; import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Calc; import org.apache.calcite.rel.core.Correlate; import org.apache.calcite.rel.core.Filter; @@ -32,24 +34,34 @@ import org.apache.calcite.rel.core.TableModify; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.core.Values; import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexProgram; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.ImmutableIntList; import org.apache.calcite.util.Util; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.checkerframework.checker.nullness.qual.Nullable; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.apache.calcite.rel.metadata.RelMdColumnUniqueness.PASSTHROUGH_AGGREGATIONS; +import static org.apache.calcite.rel.metadata.RelMdColumnUniqueness.getConstantColumnSet; + import static java.util.Objects.requireNonNull; /** @@ -74,11 +86,28 @@ private RelMdUniqueKeys() {} public @Nullable Set getUniqueKeys(Filter rel, RelMetadataQuery mq, boolean ignoreNulls) { - return mq.getUniqueKeys(rel.getInput(), ignoreNulls); + Set uniqueKeys = mq.getUniqueKeys(rel.getInput(), ignoreNulls); + if (uniqueKeys == null) { + return null; + } + // Remove constant columns from each key + RelOptPredicateList predicates = mq.getPulledUpPredicates(rel); + if (RelOptPredicateList.isEmpty(predicates)) { + return uniqueKeys; + } else { + ImmutableBitSet constantColumns = getConstantColumnSet(predicates); + return uniqueKeys.stream() + .map(key -> key.except(constantColumns)) + .collect(Collectors.toSet()); + } } public @Nullable Set getUniqueKeys(Sort rel, RelMetadataQuery mq, boolean ignoreNulls) { + Double maxRowCount = mq.getMaxRowCount(rel); + if (maxRowCount != null && maxRowCount <= 1.0d) { + return ImmutableSet.of(ImmutableBitSet.of()); + } return mq.getUniqueKeys(rel.getInput(), ignoreNulls); } @@ -234,7 +263,11 @@ private static Set getProjectUniqueKeys(SingleRel rel, RelMetad && rightUnique && (leftSet != null) && !rel.getJoinType().generatesNullsOnLeft()) { - retSet.addAll(leftSet); + ImmutableBitSet leftConstants = + getConstantJoinKeys(joinInfo.leftKeys, joinInfo.rightKeys, right, mq); + leftSet.stream() + .map(set -> set.except(leftConstants)) + .forEach(retSet::add); } // same as above except left and right are reversed @@ -242,26 +275,82 @@ private static Set getProjectUniqueKeys(SingleRel rel, RelMetad && leftUnique && (rightSet != null) && !rel.getJoinType().generatesNullsOnRight()) { - retSet.addAll(rightSet); + ImmutableBitSet rightConstants = + getConstantJoinKeys(joinInfo.rightKeys, joinInfo.leftKeys, left, mq); + rightSet.stream() + .map(set -> set.except(rightConstants)) + .forEach(retSet::add); + } + + // Remove sets that are supersets of other sets + final Set reducedSet = new HashSet<>(); + for (ImmutableBitSet bigger : retSet) { + if (retSet.stream() + .filter(smaller -> !bigger.equals(smaller)) + .noneMatch(bigger::contains)) { + reducedSet.add(bigger); + } } - return retSet; + return reducedSet; + } + + /** + * Return the keys that are constant by virtue of equality with a constant + * (literal or scalar query result) on the other side of a join. + */ + private ImmutableBitSet getConstantJoinKeys(ImmutableIntList keys, ImmutableIntList otherKeys, + RelNode otherRel, RelMetadataQuery mq) { + Double maxRowCount = mq.getMaxRowCount(otherRel); + ImmutableBitSet otherConstants; + if (maxRowCount != null && maxRowCount <= 1.0d) { + // In a single row solution, every column is constant + int size = otherRel.getRowType().getFieldList().size(); + otherConstants = ImmutableBitSet.range(size); + } else { + otherConstants = getConstantColumnSet(mq.getPulledUpPredicates(otherRel)); + } + ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + for (int i = 0; i < keys.size(); i++) { + if (otherConstants.get(otherKeys.get(i))) { + builder.set(keys.get(i)); + } + } + return builder.build(); } public Set getUniqueKeys(Aggregate rel, RelMetadataQuery mq, boolean ignoreNulls) { if (Aggregate.isSimple(rel)) { final ImmutableBitSet groupKeys = rel.getGroupSet(); - final Set inputUniqueKeys = mq - .getUniqueKeys(rel.getInput(), ignoreNulls); + RelOptPredicateList pulledUpPredicates = mq.getPulledUpPredicates(rel); + ImmutableBitSet reducedGroupKeys = + groupKeys.except(getConstantColumnSet(pulledUpPredicates)); + + final Set preciseUniqueKeys; + final Set inputUniqueKeys = + mq.getUniqueKeys(rel.getInput(), ignoreNulls); if (inputUniqueKeys == null) { - return ImmutableSet.of(groupKeys); + preciseUniqueKeys = ImmutableSet.of(reducedGroupKeys); + } else { + // Try to find more precise unique keys. + final Set keysInGroupBy = inputUniqueKeys.stream() + .filter(reducedGroupKeys::contains).collect(Collectors.toSet()); + preciseUniqueKeys = keysInGroupBy.isEmpty() + ? ImmutableSet.of(reducedGroupKeys) + : keysInGroupBy; } - // Try to find more precise unique keys. - final Set preciseUniqueKeys = inputUniqueKeys.stream() - .filter(groupKeys::contains).collect(Collectors.toSet()); - return preciseUniqueKeys.isEmpty() ? ImmutableSet.of(groupKeys) : preciseUniqueKeys; + // If an input's unique column(s) value is returned (passed through) by an aggregation + // function, then the result of the function(s) is also unique. + final ImmutableSet.Builder keysBuilder = ImmutableSet.builder(); + if (inputUniqueKeys != null) { + for (ImmutableBitSet inputKey : inputUniqueKeys) { + keysBuilder.addAll(getPassedThroughCols(inputKey, rel)); + } + } + + return filterSupersets(Sets.union(preciseUniqueKeys, keysBuilder.build())); } else if (ignoreNulls) { // group by keys form a unique key return ImmutableSet.of(rel.getGroupSet()); @@ -272,6 +361,69 @@ public Set getUniqueKeys(Aggregate rel, RelMetadataQuery mq, } } + /** + * Returns the subset of the supplied keys that are not a superset of any of the other keys. + * Given {@code {0},{1},{1,2}}, returns {@code {0},{1}} + */ + private Set filterSupersets(Set uniqueKeys) { + Set minimalKeys = new HashSet<>(); + outer: + for (ImmutableBitSet candidateKey : uniqueKeys) { + for (ImmutableBitSet possibleSubset : uniqueKeys) { + if (!candidateKey.equals(possibleSubset) && candidateKey.contains(possibleSubset)) { + continue outer; + } + } + minimalKeys.add(candidateKey); + } + return minimalKeys; + } + + /** + * Given a set of columns in the input of an Aggregate rel, returns the set of mappings from the + * input columns to the output of the aggregations. A mapping for a particular column exists if + * it is part of a simple group by and/or it is "passed through" unmodified by a + * {@link RelMdColumnUniqueness#PASSTHROUGH_AGGREGATIONS pass-through aggregation function}. + */ + private Set getPassedThroughCols(ImmutableBitSet inputColumns, Aggregate rel) { + Preconditions.checkArgument(Aggregate.isSimple(rel)); + Set conbinations = new HashSet<>(); + conbinations.add(ImmutableBitSet.of()); + for (Integer inputColumn : inputColumns.asSet()) { + final ImmutableBitSet passedThroughCols = getPassedThroughCols(inputColumn, rel); + final Set crossProduct = new HashSet<>(); + for (ImmutableBitSet set : conbinations) { + for (Integer passedThroughCol : passedThroughCols) { + crossProduct.add(set.rebuild().set(passedThroughCol).build()); + } + } + conbinations = crossProduct; + } + return conbinations; + } + + /** + * Given a column in the input of an Aggregate rel, returns the mappings from the input column to + * the output of the aggregations. A mapping for the column exists if it is part of a simple + * group by and/or it is "passed through" unmodified by a + * {@link RelMdColumnUniqueness#PASSTHROUGH_AGGREGATIONS pass-through aggregation function}. + */ + private ImmutableBitSet getPassedThroughCols(Integer inputColumn, Aggregate rel) { + final ImmutableBitSet.Builder builder = ImmutableBitSet.builder(); + if (rel.getGroupSet().get(inputColumn)) { + builder.set(inputColumn); + } + final int aggCallsOffset = rel.getGroupSet().length(); + for (int i = 0, size = rel.getAggCallList().size(); i < size; i++) { + AggregateCall call = rel.getAggCallList().get(i); + if (PASSTHROUGH_AGGREGATIONS.contains(call.getAggregation().getKind()) + && call.getArgList().get(0).equals(inputColumn)) { + builder.set(i + aggCallsOffset); + } + } + return builder.build(); + } + public Set getUniqueKeys(Union rel, RelMetadataQuery mq, boolean ignoreNulls) { if (!rel.all) { @@ -340,6 +492,34 @@ public Set getUniqueKeys(Minus rel, return ImmutableSet.copyOf(keys); } + public @Nullable Set getUniqueKeys(Values rel, RelMetadataQuery mq, + boolean ignoreNulls) { + ImmutableList> tuples = rel.tuples; + if (tuples.size() <= 1) { + return ImmutableSet.of(ImmutableBitSet.of()); + } + // Identify the single-column keys - a subset of all composite keys + List> ranges = new ArrayList<>(); + int rowSize = tuples.get(0).size(); + for (int i = 0; i < rowSize; i++) { + ranges.add(new HashSet<>()); + } + for (ImmutableList tuple : tuples) { + for (int i = 0; i < rowSize; i++) { + ranges.get(i).add(tuple.get(i)); + } + } + + ImmutableSet.Builder keySetBuilder = ImmutableSet.builder(); + for (int i = 0; i < ranges.size(); i++) { + final Set range = ranges.get(i); + if (range.size() == tuples.size()) { + keySetBuilder.add(ImmutableBitSet.of(i)); + } + } + return keySetBuilder.build(); + } + // Catch-all rule when none of the others apply. public @Nullable Set getUniqueKeys(RelNode rel, RelMetadataQuery mq, boolean ignoreNulls) { diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java index d2c7b7f26523..27d1bdefff2d 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMetadataQuery.java @@ -366,7 +366,7 @@ public static RelMetadataQuery instance() { * @param rel the relational expression * @param column 0-based ordinal for output column of interest * @return set of origin columns, or null if this information cannot be - * determined (whereas empty set indicates Handler.classinitely no origin columns at + * determined (whereas empty set indicates definitely no origin columns at * all) */ public @Nullable Set getColumnOrigins(RelNode rel, int column) { @@ -476,7 +476,7 @@ public static RelMetadataQuery instance() { * * @param rel the relational expression * @return set of keys, or null if this information cannot be determined - * (whereas empty set indicates Handler.classinitely no keys at all) + * (whereas empty set indicates definitely no keys at all) */ public @Nullable Set getUniqueKeys(RelNode rel) { return getUniqueKeys(rel, false); diff --git a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java index 37b0a477fb8b..24155414e69c 100644 --- a/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelMetadataTest.java @@ -1141,6 +1141,94 @@ private void checkColumnUniquenessForFilterWithConstantColumns(String sql) { .assertThatAreColumnsUnique(bitSetOf(0, 1), is(false)); } + @Test void testColumnUniquenessForLimit1() { + final String sql = "" + + "select *\n" + + "from emp\n" + + "limit 1"; + sql(sql) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + + @Test void testColumnUniquenessForJoinOnLimit1() { + final String sql = "" + + "select *\n" + + "from emp A\n" + + "join (\n" + + " select * from emp\n" + + " limit 1) B\n" + + "on A.empno = B.empno"; + sql(sql) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(9), is(true)) + .assertThatAreColumnsUnique(bitSetOf(10), is(true)) + .assertThatAreColumnsUnique(bitSetOf(), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + + @Test void testColumnUniquenessForJoinOnAggregation() { + final String sql = "" + + "select *\n" + + "from emp A\n" + + "join (\n" + + " select max(empno) AS maxno from emp) B\n" + + "on A.empno = B.maxno"; + sql(sql) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(9), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1, 9), is(true)) + .assertThatAreColumnsUnique(bitSetOf(), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + + @Test void testColumnUniquenessForConstantKey() { + final String sql = "" + + "select *\n" + + "from emp A\n" + + "where empno = 1010"; + sql(sql) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + + @Test void testColumnUniquenessForCorrelatedSubquery() { + final String sql = "" + + "select *\n" + + "from emp A\n" + + "where empno = (\n" + + " select max(empno) from emp)"; + sql(sql) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + + @Test void testColumnUniquenessForSubqueryWithCorrelatingVars() { + final String sql = "" + + "select empno, deptno, slacker\n" + + "from emp A\n" + + "where empno = (\n" + + " select max(empno)\n" + + " from emp B\n" + + " where A.deptno = B.deptno\n" + + ")"; + sql(sql) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + // This requires drilling into the subquery +// .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(), is(false)) + .assertThatUniqueKeysAre(bitSetOf(0)); + } + @Test void testColumnUniquenessForAggregateWithConstantColumns() { final String sql = "" + "select deptno, ename, sum(sal)\n" @@ -1148,7 +1236,11 @@ private void checkColumnUniquenessForFilterWithConstantColumns(String sql) { + "where deptno=1010\n" + "group by deptno, ename"; sql(sql) - .assertThatAreColumnsUnique(bitSetOf(1), is(true)); + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(), is(false)) + .assertThatUniqueKeysAre(bitSetOf(1)); } @Test void testColumnUniquenessForExchangeWithConstantColumns() { @@ -1210,6 +1302,65 @@ private void checkColumnUniquenessForFilterWithConstantColumns(String sql) { .assertThatUniqueKeysAre(bitSetOf(0)); } + /** + * The group by columns constitute a key, and the keys of the relation we are + * aggregating over are retained. + */ + @Test void testGroupByNonKey() { + sql("select sal, max(deptno), max(empno) from emp group by sal") + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0), bitSetOf(2)); + } + + /** + * All columns are unique. Should not include () because there are multiple rows. + */ + @Test void testGroupByNonKeyNoAggs() { + sql("select sal from emp group by sal") + .assertThatAreColumnsUnique(bitSetOf(), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + } + +// TODO: Enable when CALCITE-6126 fixed +/* + @Test void testOverByNonKey() { + sql("select sal,\n" + + "max(deptno) over (partition BY sal rows between 2 preceding and 0 following) maxDept,\n" + + "max(empno) over (partition BY sal rows between 2 preceding and 0 following) maxEmp\n" + + "from emp") + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatUniqueKeysAre(); + } +*/ + +// TODO: Enable when CALCITE-6126 fixed +/* + @Test void testOverNoPartitioning() { + sql("select max(empno) over (rows between 2 preceding and 0 following) maxEmp from emp") + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatUniqueKeysAre(); + } +*/ + + @Test void testNoGroupBy() { + sql("select max(sal), count(*) from emp") + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + + @Test void testGroupByNothing() { + sql("select max(sal), count(*) from emp group by ()") + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + } + @Test void testGroupingSets() { sql("select deptno, sal, count(*) from emp\n" + "group by GROUPING SETS (deptno, sal)") @@ -1258,6 +1409,11 @@ private void checkColumnUniquenessForFilterWithConstantColumns(String sql) { // all columns, contain composite keys sql("select * from s.composite_keys_table") .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0, 1, 2), is(true)) .assertThatUniqueKeysAre(bitSetOf(0, 1)); // only contain composite keys @@ -1274,6 +1430,173 @@ private void checkColumnUniquenessForFilterWithConstantColumns(String sql) { sql("select value1 from s.composite_keys_table") .withCatalogReaderFactory(factory) .assertThatUniqueKeysAre(); + + // One key set to constant + sql("select key1, key2, value1 from s.composite_keys_table t\n" + + "where t.key2 = 'constant'") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // One key set to a value from correlated subquery + sql("select * from s.composite_keys_table where key2 = (" + + "select max(key2) from s.composite_keys_table)") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // One key set to table-wide aggregation in join expression + sql("select * from s.composite_keys_table t1\n" + + "inner join (\n" + + " select max(key2) max_key2 from s.composite_keys_table) t2\n" + + "on t1.key2 = t2.max_key2") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // One key set to single value by limit in join expression + sql("select * from s.composite_keys_table t1\n" + + "inner join (\n" + + " select * from s.composite_keys_table limit 1) t2\n" + + "on t1.key2 = t2.key2") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // One key set to single constant by select in join expression + sql("select * from s.composite_keys_table t1\n" + + "inner join (\n" + + " select CAST('constant' AS VARCHAR) c) t2\n" + + "on t1.key2 = t2.c") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // One key set joined with single-row constant + sql("select * from s.composite_keys_table t1\n" + + "inner join (\n" + + "values (CAST('constant' AS VARCHAR))) as t2 (c)\n" + + "on t1.key2 = t2.c") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // One key set joined with multi-row constant + sql("select * from s.composite_keys_table t1\n" + + "inner join (\n" + + "values (CAST('constant' AS VARCHAR)),(CAST('constant' AS VARCHAR))) as t2 (c)\n" + + "on t1.key2 = t2.c") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatUniqueKeysAre(); + } + + @Test void testCompositeKeysAggregationUniqueKeys() { + SqlTestFactory.CatalogReaderFactory factory = (typeFactory, caseSensitive) -> { + CompositeKeysCatalogReader catalogReader = + new CompositeKeysCatalogReader(typeFactory, false); + catalogReader.init(); + return catalogReader; + }; + + // both keys in passthrough functions, no group by (single row) + sql("select any_value(key1), any_value(key2) from s.composite_keys_table") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + + // one key in mutating function, no group by (single row) + sql("select min(key1), avg(key2) from s.composite_keys_table") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) + .assertThatUniqueKeysAre(bitSetOf()); + + // both keys in passthrough functions, group by non-key + sql("select value1, min(key1), max(key2) from s.composite_keys_table group by value1") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1, 2), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0, 1, 2), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0), bitSetOf(1, 2)); + + // keys passed through multiple functions, group by non-key + sql("select min(key1), max(key1), avg(key1), min(key2), max(key2), avg(key2), value1\n" + + "from s.composite_keys_table group by value1") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(3), is(false)) + .assertThatAreColumnsUnique(bitSetOf(4), is(false)) + .assertThatAreColumnsUnique(bitSetOf(5), is(false)) + .assertThatAreColumnsUnique(bitSetOf(6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 3), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0, 4), is(true)) + .assertThatAreColumnsUnique(bitSetOf(0, 5), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(1, 2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1, 3), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1, 4), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1, 5), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1, 6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(2, 3), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2, 4), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2, 5), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2, 6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(3, 4), is(false)) + .assertThatAreColumnsUnique(bitSetOf(3, 5), is(false)) + .assertThatAreColumnsUnique(bitSetOf(3, 6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(4, 5), is(false)) + .assertThatAreColumnsUnique(bitSetOf(4, 6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(5, 6), is(true)) // group by + .assertThatAreColumnsUnique(bitSetOf(0, 1, 2, 3), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1, 3, 4, 5), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0, 3), bitSetOf(0, 4), bitSetOf(1, 3), bitSetOf(1, 4), + bitSetOf(6)); + + // one key in mutating function, group by non-key + sql("select value1, min(key1), count(key2) from s.composite_keys_table group by value1") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) + .assertThatAreColumnsUnique(bitSetOf(1, 2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1, 2), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0)); + + // one key part of group by, one in passthrough function + sql("select key1, min(key2), value1 from s.composite_keys_table group by key1, value1") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) // passthroughs + .assertThatAreColumnsUnique(bitSetOf(0, 2), is(true)) // group bys + .assertThatAreColumnsUnique(bitSetOf(1, 2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1, 2), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0, 1), bitSetOf(0, 2)); + + // one key part of group by, one in mutating function + sql("select key1, value1, count(key2) from s.composite_keys_table group by key1, value1") + .withCatalogReaderFactory(factory) + .assertThatAreColumnsUnique(bitSetOf(0), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1), is(false)) + .assertThatAreColumnsUnique(bitSetOf(2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1), is(true)) // group bys + .assertThatAreColumnsUnique(bitSetOf(0, 2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(1, 2), is(false)) + .assertThatAreColumnsUnique(bitSetOf(0, 1, 2), is(true)) + .assertThatUniqueKeysAre(bitSetOf(0, 1)); } private static ImmutableBitSet bitSetOf(int... bits) { @@ -1302,7 +1625,7 @@ private static ImmutableBitSet bitSetOf(int... bits) { sql("select a1.empno, a2.empno\n" + "from emp a1 join emp a2 on (a1.empno=a2.empno)") .convertingProjectAsCalc() - .assertThatUniqueKeysAre(bitSetOf(0), bitSetOf(1), bitSetOf(0, 1)); + .assertThatUniqueKeysAre(bitSetOf(0), bitSetOf(1)); } @Test void calcMultipleColumnsAreUniqueCalc3() { @@ -1310,8 +1633,7 @@ private static ImmutableBitSet bitSetOf(int... bits) { + " from emp a1 join emp a2\n" + " on (a1.empno=a2.empno)") .convertingProjectAsCalc() - .assertThatUniqueKeysAre(bitSetOf(0), bitSetOf(0, 1), bitSetOf(0, 1, 2), - bitSetOf(0, 2), bitSetOf(1), bitSetOf(1, 2), bitSetOf(2)); + .assertThatUniqueKeysAre(bitSetOf(0), bitSetOf(1), bitSetOf(1, 2), bitSetOf(2)); } @Test void calcColumnsAreNonUniqueCalc() { diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index eb8b70010d2e..2243b04b5820 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -4615,7 +4615,7 @@ private void checkEmptyJoin(RelOptFixture f) { final RelDataType type = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT); RelNode left = b - .values(new String[]{"x", "y"}, 1, 2).build(); + .values(new String[]{"x", "y"}, 1, 2, 2, 1).build(); RexNode ref = rexBuilder.makeInputRef(left, 0); RexLiteral literal1 = rexBuilder.makeLiteral(1, type); RexLiteral literal2 = rexBuilder.makeLiteral(2, type); diff --git a/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_PredicatesHandler.java b/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_PredicatesHandler.java index c0bc003c418b..597a7d4380dd 100644 --- a/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_PredicatesHandler.java +++ b/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_PredicatesHandler.java @@ -88,6 +88,8 @@ private org.apache.calcite.plan.RelOptPredicateList getPredicates_( return provider0.getPredicates((org.apache.calcite.rel.core.TableScan) r, mq); } else if (r instanceof org.apache.calcite.rel.core.Union) { return provider0.getPredicates((org.apache.calcite.rel.core.Union) r, mq); + } else if (r instanceof org.apache.calcite.rel.core.Values) { + return provider0.getPredicates((org.apache.calcite.rel.core.Values) r, mq); } else if (r instanceof org.apache.calcite.rel.RelNode) { return provider0.getPredicates((org.apache.calcite.rel.RelNode) r, mq); } else { diff --git a/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_UniqueKeysHandler.java b/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_UniqueKeysHandler.java index 44b0be215e7f..743416457778 100644 --- a/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_UniqueKeysHandler.java +++ b/core/src/test/resources/org/apache/calcite/rel/metadata/janino/GeneratedMetadata_UniqueKeysHandler.java @@ -88,6 +88,8 @@ private java.util.Set getUniqueKeys_( return provider0.getUniqueKeys((org.apache.calcite.rel.core.TableScan) r, mq, a2); } else if (r instanceof org.apache.calcite.rel.core.Union) { return provider0.getUniqueKeys((org.apache.calcite.rel.core.Union) r, mq, a2); + } else if (r instanceof org.apache.calcite.rel.core.Values) { + return provider0.getUniqueKeys((org.apache.calcite.rel.core.Values) r, mq, a2); } else if (r instanceof org.apache.calcite.rel.RelNode) { return provider0.getUniqueKeys((org.apache.calcite.rel.RelNode) r, mq, a2); } else { diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index 87b2c2fdce07..e51ecb62579b 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -11729,7 +11729,7 @@ LogicalProject(QX=[CAST(CASE(=($0, 1), 1, 2)):INTEGER]) diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml index 9db58746c396..0dac4bc8fd31 100644 --- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml @@ -3835,11 +3835,10 @@ LogicalProject(DEPTNO=[$9], SAL=[$7]) LogicalTableScan(table=[[CATALOG, SALES, DEPT]]) LogicalJoin(condition=[true], joinType=[left]) LogicalTableScan(table=[[CATALOG, SALES, EMP]]) - LogicalAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)]) - LogicalProject(EXPR$0=[>($0, 0)]) - LogicalAggregate(group=[{}], agg#0=[AVG($0)]) - LogicalProject(SAL=[$5]) - LogicalTableScan(table=[[CATALOG, SALES, EMP]]) + LogicalProject(EXPR$0=[>($0, 0)]) + LogicalAggregate(group=[{}], agg#0=[AVG($0)]) + LogicalProject(SAL=[$5]) + LogicalTableScan(table=[[CATALOG, SALES, EMP]]) ]]> diff --git a/core/src/test/resources/sql/conditions.iq b/core/src/test/resources/sql/conditions.iq index a5bf3d29d2b7..0cec3ce4dfd4 100644 --- a/core/src/test/resources/sql/conditions.iq +++ b/core/src/test/resources/sql/conditions.iq @@ -395,10 +395,9 @@ EnumerableCalc(expr#0..2=[{inputs}], EMPNO=[$t0]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..2=[{inputs}], DEPTNO=[$t0]) EnumerableTableScan(table=[[scott, DEPT]]) - EnumerableAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[0], expr#2=[>($t0, $t1)], EXPR$0=[$t2]) - EnumerableAggregate(group=[{}], agg#0=[COUNT()]) - EnumerableTableScan(table=[[scott, EMP]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[0], expr#2=[>($t0, $t1)], EXPR$0=[$t2]) + EnumerableAggregate(group=[{}], agg#0=[COUNT()]) + EnumerableTableScan(table=[[scott, EMP]]) !plan # sub-query return true with Equal condition diff --git a/core/src/test/resources/sql/sub-query.iq b/core/src/test/resources/sql/sub-query.iq index 0d1b165705c0..cb39a2a5fbc0 100644 --- a/core/src/test/resources/sql/sub-query.iq +++ b/core/src/test/resources/sql/sub-query.iq @@ -1044,9 +1044,8 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], SAL=[$t1], EXPR$ EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], cs=[$t3], $condition=[$t5]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], cs=[$t3], $condition=[$t5]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan # Test project literal IN nullable @@ -1294,9 +1293,8 @@ EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NULL($t2)], SAL=[$t1], EXPR$1=[$ EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], cs=[$t3], $condition=[$t5]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], cs=[$t3], $condition=[$t5]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan # Test project literal NOT IN nullable @@ -1464,9 +1462,8 @@ EnumerableCalc(expr#0..2=[{inputs}], SAL=[$t1]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], cs=[$t3], $condition=[$t5]) - EnumerableTableScan(table=[[scott, DEPT]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[=($t3, $t0)], DEPTNO=[$t0], $condition=[$t4]) + EnumerableTableScan(table=[[scott, DEPT]]) !plan # Test filter literal IN nullable @@ -1904,12 +1901,12 @@ select sal from "scott".emp e (11 rows) !ok -EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t4)], expr#6=[IS NOT NULL($t4)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], SAL=[$t1], $condition=[$t8]) - EnumerableMergeJoin(condition=[=($2, $3)], joinType=[left]) +EnumerableCalc(expr#0..4=[{inputs}], expr#5=[NOT($t3)], expr#6=[IS NOT NULL($t3)], expr#7=[OR($t5, $t6)], expr#8=[IS NOT TRUE($t7)], SAL=[$t1], $condition=[$t8]) + EnumerableMergeJoin(condition=[=($2, $4)], joinType=[left]) EnumerableSort(sort0=[$2], dir0=[ASC]) EnumerableCalc(expr#0..7=[{inputs}], EMPNO=[$t0], SAL=[$t5], DEPTNO=[$t7]) EnumerableTableScan(table=[[scott, EMP]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], DEPTNO1=[$t0], $f1=[$t3], $condition=[$t5]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[true], expr#4=[10], expr#5=[=($t4, $t0)], cs=[$t3], DEPTNO1=[$t0], $condition=[$t5]) EnumerableTableScan(table=[[scott, DEPT]]) !plan @@ -3515,9 +3512,8 @@ select (select (1, 2)); EnumerableCalc(expr#0..1=[{inputs}], EXPR$0=[$t1]) EnumerableNestedLoopJoin(condition=[true], joinType=[left]) EnumerableValues(tuples=[[{ 0 }]]) - EnumerableAggregate(group=[{}], agg#0=[SINGLE_VALUE($0)]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[ROW($t1, $t2)], EXPR$0=[$t3]) - EnumerableValues(tuples=[[{ 0 }]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[2], expr#3=[ROW($t1, $t2)], EXPR$0=[$t3]) + EnumerableValues(tuples=[[{ 0 }]]) !plan # Test case for correlated sub-query