From 08f685683e8e6e97b47189d95603ccccaf17051d Mon Sep 17 00:00:00 2001 From: Mihai Budiu Date: Tue, 5 Dec 2023 15:06:57 -0800 Subject: [PATCH] [CALCITE-6074] The size of REAL, DOUBLE, and FLOAT is not consistent Signed-off-by: Mihai Budiu --- .../calcite/interpreter/AggregateNode.java | 4 ++-- .../calcite/plan/VisitorDataContext.java | 1 + .../apache/calcite/rel/metadata/RelMdSize.java | 2 +- .../rel/type/RelDataTypeFactoryImpl.java | 3 +-- .../org/apache/calcite/rex/RexBuilder.java | 5 +++-- .../org/apache/calcite/rex/RexLiteral.java | 4 ++++ .../org/apache/calcite/sql/SqlLiteral.java | 2 ++ .../sql/dialect/BigQuerySqlDialect.java | 2 ++ .../sql/dialect/ClickHouseSqlDialect.java | 3 ++- .../sql/dialect/JethroDataSqlDialect.java | 2 +- .../sql/type/SqlTypeAssignmentRule.java | 3 +++ .../type/SqlTypeExplicitPrecedenceList.java | 10 ++++++++-- .../apache/calcite/sql/type/SqlTypeUtil.java | 18 ------------------ .../calcite/sql/validate/SqlValidatorImpl.java | 2 ++ .../org/apache/calcite/rex/RexBuilderTest.java | 2 +- .../calcite/adapter/druid/DruidType.java | 2 +- .../calcite/adapter/file/CsvEnumerator.java | 3 ++- .../org/apache/calcite/piglet/PigTypes.java | 2 +- .../org/apache/calcite/test/PigRelExTest.java | 8 ++++---- 19 files changed, 41 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java index e461b238de8b..7c908cc0e8dc 100644 --- a/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java +++ b/core/src/main/java/org/apache/calcite/interpreter/AggregateNode.java @@ -218,10 +218,10 @@ private static Class maxMinClass(AggregateCall call) { switch (call.getType().getSqlTypeName()) { case INTEGER: return max ? MaxInt.class : MinInt.class; - case FLOAT: + case REAL: return max ? MaxFloat.class : MinFloat.class; + case FLOAT: case DOUBLE: - case REAL: return max ? MaxDouble.class : MinDouble.class; case DECIMAL: return max ? MaxBigDecimal.class : MinBigDecimal.class; diff --git a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java index 758695b9b444..1b48b8c64f82 100644 --- a/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java +++ b/core/src/main/java/org/apache/calcite/plan/VisitorDataContext.java @@ -126,6 +126,7 @@ public VisitorDataContext(@Nullable Object[] values) { switch (type.getSqlTypeName()) { case INTEGER: return Pair.of(index, rexLiteral.getValueAs(Integer.class)); + case FLOAT: case DOUBLE: return Pair.of(index, rexLiteral.getValueAs(Double.class)); case REAL: diff --git a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java index ca91d5147832..903a65f48a98 100644 --- a/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java +++ b/core/src/main/java/org/apache/calcite/rel/metadata/RelMdSize.java @@ -360,7 +360,6 @@ public double typeValueSize(RelDataType type, @Nullable Comparable value) { case SMALLINT: return 2d; case INTEGER: - case FLOAT: case REAL: case DATE: case TIME: @@ -370,6 +369,7 @@ public double typeValueSize(RelDataType type, @Nullable Comparable value) { case INTERVAL_MONTH: return 4d; case BIGINT: + case FLOAT: // sic case DOUBLE: case TIMESTAMP: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: diff --git a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java index 7beb8b9f9df8..805b1cc9f32c 100644 --- a/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java +++ b/core/src/main/java/org/apache/calcite/rel/type/RelDataTypeFactoryImpl.java @@ -590,8 +590,7 @@ private RelDataType decimalOf2(RelDataType type) { return createSqlType(SqlTypeName.DECIMAL, 38, 0); case REAL: return createSqlType(SqlTypeName.DECIMAL, 14, 7); - case FLOAT: - return createSqlType(SqlTypeName.DECIMAL, 14, 7); + case FLOAT: // sic case DOUBLE: // the default max precision is 19, so this is actually DECIMAL(19, 15) // but derived system can override the max precision/scale. diff --git a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java index 7d778a399f8b..23463f75e9de 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexBuilder.java +++ b/core/src/main/java/org/apache/calcite/rex/RexBuilder.java @@ -595,6 +595,7 @@ public RexNode makeCast( case INTEGER: case SMALLINT: case TINYINT: + case DOUBLE: case FLOAT: case REAL: case DECIMAL: @@ -1783,13 +1784,13 @@ public RexNode makeLiteral(@Nullable Object value, RelDataType type, o.getClass().getCanonicalName(), type.getSqlTypeName()); return new BigDecimal(((Number) o).longValue()); - case FLOAT: + case REAL: if (o instanceof BigDecimal) { return o; } return new BigDecimal(((Number) o).doubleValue(), MathContext.DECIMAL32) .stripTrailingZeros(); - case REAL: + case FLOAT: case DOUBLE: if (o instanceof BigDecimal) { return o; diff --git a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java index b5231b66b71e..4bf9073082bf 100644 --- a/core/src/main/java/org/apache/calcite/rex/RexLiteral.java +++ b/core/src/main/java/org/apache/calcite/rex/RexLiteral.java @@ -406,6 +406,7 @@ public static SqlTypeName strictTypeName(RelDataType type) { return SqlTypeName.DECIMAL; case REAL: case FLOAT: + case DOUBLE: return SqlTypeName.DOUBLE; case VARBINARY: return SqlTypeName.BINARY; @@ -653,6 +654,7 @@ private static void appendAsJava(@Nullable Comparable value, StringBuilder sb, sb.append(value.toString()); break; case DOUBLE: + case FLOAT: assert value instanceof BigDecimal; sb.append(Util.toScientificNotation((BigDecimal) value)); break; @@ -806,6 +808,8 @@ private static RexLiteral toLiteral(RelDataType type, Comparable value) { return new RexLiteral(b, type, typeName); case DECIMAL: case DOUBLE: + case REAL: + case FLOAT: BigDecimal d = new BigDecimal(literal); return new RexLiteral(d, type, typeName); case BINARY: diff --git a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java index 0ecfd05acd3f..4cc41317ed38 100644 --- a/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java +++ b/core/src/main/java/org/apache/calcite/sql/SqlLiteral.java @@ -199,6 +199,8 @@ public static boolean valueMatchesType( return value == null; case DECIMAL: case DOUBLE: + case FLOAT: + case REAL: return value instanceof BigDecimal; case DATE: return value instanceof DateString; diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java index 15ae0bad0b40..c36e25a1da3b 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/BigQuerySqlDialect.java @@ -340,6 +340,8 @@ private static TimeUnit validate(TimeUnit timeUnit) { case FLOAT: case DOUBLE: return createSqlDataTypeSpecByName("FLOAT64", typeName); + case REAL: + return createSqlDataTypeSpecByName("FLOAT32", typeName); case DECIMAL: return createSqlDataTypeSpecByName("NUMERIC", typeName); case BOOLEAN: diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java index 802f562c3d65..b6616d0c94e5 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/ClickHouseSqlDialect.java @@ -88,8 +88,9 @@ public ClickHouseSqlDialect(Context context) { return createSqlDataTypeSpecByName("Int32", typeName); case BIGINT: return createSqlDataTypeSpecByName("Int64", typeName); - case FLOAT: + case REAL: return createSqlDataTypeSpecByName("Float32", typeName); + case FLOAT: case DOUBLE: return createSqlDataTypeSpecByName("Float64", typeName); case DATE: diff --git a/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java b/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java index 40790acb57b8..0dddb6de0522 100644 --- a/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java +++ b/core/src/main/java/org/apache/calcite/sql/dialect/JethroDataSqlDialect.java @@ -148,7 +148,7 @@ private static SqlTypeName parse(String strType) { case "double": return SqlTypeName.DOUBLE; case "float": - return SqlTypeName.FLOAT; + return SqlTypeName.REAL; case "string": return SqlTypeName.VARCHAR; case "timestamp": diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeAssignmentRule.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeAssignmentRule.java index 650481435519..6167c4df0167 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeAssignmentRule.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeAssignmentRule.java @@ -99,6 +99,7 @@ private SqlTypeAssignmentRule( rule.add(SqlTypeName.BIGINT); rule.add(SqlTypeName.DECIMAL); rule.add(SqlTypeName.FLOAT); + rule.add(SqlTypeName.DOUBLE); rules.add(SqlTypeName.FLOAT, rule); // REAL (32 bit floating point) is assignable from... @@ -109,6 +110,7 @@ private SqlTypeAssignmentRule( rule.add(SqlTypeName.BIGINT); rule.add(SqlTypeName.DECIMAL); rule.add(SqlTypeName.FLOAT); + rule.add(SqlTypeName.DOUBLE); rule.add(SqlTypeName.REAL); rules.add(SqlTypeName.REAL, rule); @@ -204,6 +206,7 @@ private SqlTypeAssignmentRule( rule.add(SqlTypeName.BIGINT); rule.add(SqlTypeName.DECIMAL); rule.add(SqlTypeName.FLOAT); + rule.add(SqlTypeName.DOUBLE); rule.add(SqlTypeName.REAL); rule.add(SqlTypeName.TIME); rule.add(SqlTypeName.DATE); diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java index 92593288a9dc..da87c5166ce4 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeExplicitPrecedenceList.java @@ -65,8 +65,14 @@ public class SqlTypeExplicitPrecedenceList .put(SqlTypeName.BIGINT, numeric(SqlTypeName.BIGINT)) .put(SqlTypeName.DECIMAL, numeric(SqlTypeName.DECIMAL)) .put(SqlTypeName.REAL, numeric(SqlTypeName.REAL)) - .put(SqlTypeName.FLOAT, list(SqlTypeName.FLOAT, SqlTypeName.REAL, SqlTypeName.DOUBLE)) - .put(SqlTypeName.DOUBLE, list(SqlTypeName.DOUBLE, SqlTypeName.DECIMAL)) + .put( + SqlTypeName.FLOAT, list( + SqlTypeName.FLOAT, SqlTypeName.REAL, + SqlTypeName.DOUBLE, SqlTypeName.DECIMAL)) + .put( + SqlTypeName.DOUBLE, list( + SqlTypeName.FLOAT, SqlTypeName.REAL, + SqlTypeName.DOUBLE, SqlTypeName.DECIMAL)) .put(SqlTypeName.CHAR, list(SqlTypeName.CHAR, SqlTypeName.VARCHAR)) .put(SqlTypeName.VARCHAR, list(SqlTypeName.VARCHAR)) .put(SqlTypeName.BINARY, diff --git a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java index ff1becd2bbd5..f8ccdb92e20a 100644 --- a/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java +++ b/core/src/main/java/org/apache/calcite/sql/type/SqlTypeUtil.java @@ -491,24 +491,6 @@ public static boolean isDecimal(RelDataType type) { return typeName == SqlTypeName.DECIMAL; } - /** Returns whether a type is DOUBLE. */ - public static boolean isDouble(RelDataType type) { - SqlTypeName typeName = type.getSqlTypeName(); - if (typeName == null) { - return false; - } - return typeName == SqlTypeName.DOUBLE; - } - - /** Returns whether a type is BIGINT. */ - public static boolean isBigint(RelDataType type) { - SqlTypeName typeName = type.getSqlTypeName(); - if (typeName == null) { - return false; - } - return typeName == SqlTypeName.BIGINT; - } - /** Returns whether a type is numeric with exact precision. */ public static boolean isExactNumeric(RelDataType type) { SqlTypeName typeName = type.getSqlTypeName(); diff --git a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index eee83f3cfb65..36c59bc42ff3 100644 --- a/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -3330,6 +3330,8 @@ private void registerOperandSubQueries( break; case DOUBLE: + case FLOAT: + case REAL: validateLiteralAsDouble(literal); break; diff --git a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java index a1cac491fad0..bcba21687d93 100644 --- a/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java +++ b/core/src/test/java/org/apache/calcite/rex/RexBuilderTest.java @@ -762,7 +762,7 @@ private void checkDate(RexLiteral literal) { final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); final RexBuilder rexBuilder = new RexBuilder(typeFactory); - final RelDataType floatType = typeFactory.createSqlType(SqlTypeName.FLOAT); + final RelDataType floatType = typeFactory.createSqlType(SqlTypeName.REAL); RexNode left = rexBuilder.makeInputRef(floatType, 0); final RexNode literal1 = rexBuilder.makeLiteral(1.0f, floatType); final RexNode literal2 = rexBuilder.makeLiteral(2.0f, floatType); diff --git a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java index adb4d8541a50..212d2c898682 100644 --- a/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java +++ b/druid/src/main/java/org/apache/calcite/adapter/druid/DruidType.java @@ -21,7 +21,7 @@ /** Druid type. */ public enum DruidType { LONG(SqlTypeName.BIGINT), - FLOAT(SqlTypeName.FLOAT), + FLOAT(SqlTypeName.REAL), DOUBLE(SqlTypeName.DOUBLE), STRING(SqlTypeName.VARCHAR), COMPLEX(SqlTypeName.OTHER), diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java index effba113661e..ef99349ea894 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java @@ -328,11 +328,12 @@ abstract static class RowConverter { return null; } return Long.parseLong(string); - case FLOAT: + case REAL: if (string.length() == 0) { return null; } return Float.parseFloat(string); + case FLOAT: case DOUBLE: if (string.length() == 0) { return null; diff --git a/piglet/src/main/java/org/apache/calcite/piglet/PigTypes.java b/piglet/src/main/java/org/apache/calcite/piglet/PigTypes.java index 15418cf65eeb..c629336a98fa 100644 --- a/piglet/src/main/java/org/apache/calcite/piglet/PigTypes.java +++ b/piglet/src/main/java/org/apache/calcite/piglet/PigTypes.java @@ -116,7 +116,7 @@ static RelDataType convertSchemaField(LogicalSchema.LogicalFieldSchema pigField, case DataType.LONG: return TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT, nullable); case DataType.FLOAT: - return TYPE_FACTORY.createSqlType(SqlTypeName.FLOAT, nullable); + return TYPE_FACTORY.createSqlType(SqlTypeName.REAL, nullable); case DataType.DOUBLE: return TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE, nullable); case DataType.DATETIME: diff --git a/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java b/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java index ec7f61bb6bd2..b36f684fa59e 100644 --- a/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java +++ b/piglet/src/test/java/org/apache/calcite/test/PigRelExTest.java @@ -188,7 +188,7 @@ public void testMatch() { @Test void testCast() { checkTranslation("(int) b", inTree("CAST($1):INTEGER")); checkTranslation("(long) a", inTree("CAST($0):BIGINT")); - checkTranslation("(float) b", inTree("CAST($1):FLOAT")); + checkTranslation("(float) b", inTree("CAST($1):REAL")); checkTranslation("(double) b", inTree("CAST($1):DOUBLE")); checkTranslation("(chararray) b", inTree("CAST($1):VARCHAR")); checkTranslation("(bytearray) b", inTree("CAST($1):BINARY")); @@ -197,18 +197,18 @@ public void testMatch() { checkTranslation("(bigdecimal) b", inTree("CAST($1):DECIMAL(19, 0)")); checkTranslation("(tuple()) b", inTree("CAST($1):(DynamicRecordRow[])")); checkTranslation("(tuple(int, float)) b", - inTree("CAST($1):RecordType(INTEGER $0, FLOAT $1)")); + inTree("CAST($1):RecordType(INTEGER $0, REAL $1)")); checkTranslation("(bag{}) b", inTree("CAST($1):(DynamicRecordRow[]) NOT NULL MULTISET")); checkTranslation("(bag{tuple(int)}) b", inTree("CAST($1):RecordType(INTEGER $0) MULTISET")); checkTranslation("(bag{tuple(int, float)}) b", - inTree("CAST($1):RecordType(INTEGER $0, FLOAT $1) MULTISET")); + inTree("CAST($1):RecordType(INTEGER $0, REAL $1) MULTISET")); checkTranslation("(map[]) b", inTree("CAST($1):(VARCHAR NOT NULL, BINARY(1) NOT NULL) MAP")); checkTranslation("(map[int]) b", inTree("CAST($1):(VARCHAR NOT NULL, INTEGER")); checkTranslation("(map[tuple(int, float)]) b", - inTree("CAST($1):(VARCHAR NOT NULL, RecordType(INTEGER val_0, FLOAT val_1)) MAP")); + inTree("CAST($1):(VARCHAR NOT NULL, RecordType(INTEGER val_0, REAL val_1)) MAP")); } @Test void testPigBuiltinFunctions() {