diff --git a/Makefile b/Makefile index 5d16475..14ddbda 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,8 @@ lint-bandit: ## Run bandit @echo "\n${BLUE}Running bandit...${NC}\n" @${POETRY_RUN} bandit -r ${PROJ} -lint-base: lint-flake8 lint-bandit ## Just run the linters without autolinting +#lint-base: lint-flake8 lint-bandit ## Just run the linters without autolinting +lint-base: lint-flake8 # TODO: Can we drop bandit? lint: autolint lint-base lint-mypy ## Autolint and code linting diff --git a/dat/generated_tables.py b/dat/generated_tables.py index cd15c68..79e7a35 100644 --- a/dat/generated_tables.py +++ b/dat/generated_tables.py @@ -1,14 +1,14 @@ -from decimal import Decimal import os +import random from datetime import date, datetime, timedelta +from decimal import Decimal from pathlib import Path -import random from typing import Callable, List, Tuple -from delta.tables import DeltaTable import pyspark.sql -from pyspark.sql import SparkSession import pyspark.sql.types as types +from delta.tables import DeltaTable +from pyspark.sql import SparkSession from dat.models import TableVersionMetadata, TestCaseInfo from dat.spark_builder import get_spark_session @@ -158,16 +158,17 @@ def create_multi_partitioned(case: TestCaseInfo, spark: SparkSession): @reference_table( - name="multi_partitioned_2", - description="Multiple levels of partitioning, with boolean, timestamp, and decimal partition columns" + name='multi_partitioned_2', + description=('Multiple levels of partitioning, with boolean, timestamp, and ' + 'decimal partition columns') ) def create_multi_partitioned_2(case: TestCaseInfo, spark: SparkSession): columns = ['bool', 'time', 'amount', 'int'] partition_columns = ['bool', 'time', 'amount'] data = [ - (True, datetime(1970, 1, 1), Decimal("200.00"), 1), - (True, datetime(1970, 1, 1, 12, 30), Decimal("200.00"), 2), - (False, datetime(1970, 1, 2, 8, 45), Decimal("12.00"), 3) + (True, datetime(1970, 1, 1), Decimal('200.00'), 1), + (True, datetime(1970, 1, 1, 12, 30), Decimal('200.00'), 2), + (False, datetime(1970, 1, 2, 8, 45), Decimal('12.00'), 3) ] df = spark.createDataFrame(data, schema=columns) df.repartition(1).write.format('delta').partitionBy( @@ -194,24 +195,25 @@ def with_schema_change(case: TestCaseInfo, spark: SparkSession): case.delta_root) save_expected(case) + @reference_table( name='all_primitive_types', description='Table containing all non-nested types', ) def create_all_primitive_types(case: TestCaseInfo, spark: SparkSession): schema = types.StructType([ - types.StructField("utf8", types.StringType()), - types.StructField("int64", types.LongType()), - types.StructField("int32", types.IntegerType()), - types.StructField("int16", types.ShortType()), - types.StructField("int8", types.ByteType()), - types.StructField("float32", types.FloatType()), - types.StructField("float64", types.DoubleType()), - types.StructField("bool", types.BooleanType()), - types.StructField("binary", types.BinaryType()), - types.StructField("decimal", types.DecimalType(5, 3)), - types.StructField("date32", types.DateType()), - types.StructField("timestamp", types.TimestampType()), + types.StructField('utf8', types.StringType()), + types.StructField('int64', types.LongType()), + types.StructField('int32', types.IntegerType()), + types.StructField('int16', types.ShortType()), + types.StructField('int8', types.ByteType()), + types.StructField('float32', types.FloatType()), + types.StructField('float64', types.DoubleType()), + types.StructField('bool', types.BooleanType()), + types.StructField('binary', types.BinaryType()), + types.StructField('decimal', types.DecimalType(5, 3)), + types.StructField('date32', types.DateType()), + types.StructField('timestamp', types.TimestampType()), ]) df = spark.createDataFrame([ @@ -225,7 +227,7 @@ def create_all_primitive_types(case: TestCaseInfo, spark: SparkSession): float(i), i % 2 == 0, bytes(i), - Decimal("10.000") + i, + Decimal('10.000') + i, date(1970, 1, 1) + timedelta(days=i), datetime(1970, 1, 1) + timedelta(hours=i) ) @@ -240,20 +242,25 @@ def create_all_primitive_types(case: TestCaseInfo, spark: SparkSession): description='Table containing various nested types', ) def create_nested_types(case: TestCaseInfo, spark: SparkSession): - schema = types.StructType([ - types.StructField("struct", types.StructType([ - types.StructField("float64", types.DoubleType()), - types.StructField("bool", types.BooleanType()), - ])), - types.StructField("array", types.ArrayType(types.ShortType())), - types.StructField("map", types.MapType(types.StringType(), types.IntegerType())), - ]) + schema = types.StructType([types.StructField( + 'struct', types.StructType( + [types.StructField( + 'float64', types.DoubleType()), + types.StructField( + 'bool', types.BooleanType()), ])), + types.StructField( + 'array', types.ArrayType( + types.ShortType())), + types.StructField( + 'map', types.MapType( + types.StringType(), + types.IntegerType())), ]) df = spark.createDataFrame([ ( - { "float64": float(i), "bool": i % 2 == 0 }, + {'float64': float(i), 'bool': i % 2 == 0}, list(range(i + 1)), - { str(i): i for i in range(i) } + {str(i): i for i in range(i)} ) for i in range(5) ], schema=schema) @@ -261,17 +268,18 @@ def create_nested_types(case: TestCaseInfo, spark: SparkSession): df.repartition(1).write.format('delta').save(case.delta_root) -def get_sample_data(spark: SparkSession, seed: int=42, nrows: int=5) -> pyspark.sql.DataFrame: +def get_sample_data( + spark: SparkSession, seed: int = 42, nrows: int = 5) -> pyspark.sql.DataFrame: # Use seed to get consistent data between runs, for reproducibility random.seed(seed) return spark.createDataFrame([ ( - random.choice(["a", "b", "c", None]), + random.choice(['a', 'b', 'c', None]), random.randint(0, 1000), date(random.randint(1970, 2020), random.randint(1, 12), 1) ) for i in range(nrows) - ], schema=["letter", "int", "date"]) + ], schema=['letter', 'int', 'date']) @reference_table( @@ -279,24 +287,95 @@ def get_sample_data(spark: SparkSession, seed: int=42, nrows: int=5) -> pyspark. description='Table with a checkpoint', ) def create_with_checkpoint(case: TestCaseInfo, spark: SparkSession): - spark.conf.set("spark.databricks.delta.retentionDurationCheck.enabled", "false") + df = get_sample_data(spark) + + (DeltaTable.create(spark) + .location(str(Path(case.delta_root).absolute())) + .addColumns(df.schema) + .property('delta.checkpointInterval', '2') + .execute()) + + for i in range(3): + df = get_sample_data(spark, seed=i, nrows=5) + df.repartition(1).write.format('delta').mode( + 'overwrite').save(case.delta_root) + + assert any(path.suffixes == ['.checkpoint', '.parquet'] + for path in (Path(case.delta_root) / '_delta_log').iterdir()) + + +def remove_log_file(delta_root: str, version: int): + os.remove(os.path.join(delta_root, '_delta_log', f'{version:0>20}.json')) + + +@reference_table( + name='no_replay', + description='Table with a checkpoint and prior commits cleaned up', +) +def create_no_replay(case: TestCaseInfo, spark: SparkSession): + spark.conf.set( + 'spark.databricks.delta.retentionDurationCheck.enabled', 'false') df = get_sample_data(spark) - + table = (DeltaTable.create(spark) + .location(str(Path(case.delta_root).absolute())) + .addColumns(df.schema) + .property('delta.checkpointInterval', '2') + .execute()) + + for i in range(3): + df = get_sample_data(spark, seed=i, nrows=5) + df.repartition(1).write.format('delta').mode( + 'overwrite').save(case.delta_root) + + table.vacuum(retentionHours=0) + + remove_log_file(case.delta_root, version=0) + remove_log_file(case.delta_root, version=1) + + files_in_log = list((Path(case.delta_root) / '_delta_log').iterdir()) + assert any(path.suffixes == ['.checkpoint', '.parquet'] + for path in files_in_log) + assert not any(path.name == f'{0:0>20}.json' for path in files_in_log) + + +@reference_table( + name='stats_as_struct', + description='Table with stats only written as struct (not JSON) with Checkpoint', +) +def create_stats_as_struct(case: TestCaseInfo, spark: SparkSession): + df = get_sample_data(spark) + (DeltaTable.create(spark) .location(str(Path(case.delta_root).absolute())) .addColumns(df.schema) - .property("delta.checkpointInterval", "2") - .property("delta.logRetentionDuration", "0 days") + .property('delta.checkpointInterval', '2') + .property('delta.checkpoint.writeStatsAsStruct', 'true') + .property('delta.checkpoint.writeStatsAsJson', 'false') .execute()) - for i in range(5): + for i in range(3): df = get_sample_data(spark, seed=i, nrows=5) - df.repartition(1).write.format('delta').mode('overwrite').save(case.delta_root) - - assert any(path.suffixes == [".checkpoint", ".parquet"] - for path in (Path(case.delta_root) / "_delta_log").iterdir()) + df.repartition(1).write.format('delta').mode( + 'overwrite').save(case.delta_root) - table.vacuum(retentionHours=0) +@reference_table( + name='no_stats', + description='Table with no stats', +) +def create_no_stats(case: TestCaseInfo, spark: SparkSession): + df = get_sample_data(spark) + (DeltaTable.create(spark) + .location(str(Path(case.delta_root).absolute())) + .addColumns(df.schema) + .property('delta.checkpointInterval', '2') + .property('delta.checkpoint.writeStatsAsStruct', 'false') + .property('delta.checkpoint.writeStatsAsJson', 'false') + .property('delta.dataSkippingNumIndexedCols', '0') + .execute()) + for i in range(3): + df = get_sample_data(spark, seed=i, nrows=5) + df.repartition(1).write.format('delta').mode( + 'overwrite').save(case.delta_root) diff --git a/dat/main.py b/dat/main.py index 1980db7..8d30b73 100644 --- a/dat/main.py +++ b/dat/main.py @@ -45,7 +45,8 @@ def write_generated_reference_tables(table_name: Optional[str]): create_table() break else: - raise ValueError(f"Could not find generated table named '{table_name}'") + raise ValueError( + f"Could not find generated table named '{table_name}'") else: out_base = Path('out/reader_tests/generated') shutil.rmtree(out_base) diff --git a/out/reader_tests/generated/all_primitive_types/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/all_primitive_types/delta/_delta_log/00000000000000000000.json index 768b681..51be79e 100644 --- a/out/reader_tests/generated/all_primitive_types/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/all_primitive_types/delta/_delta_log/00000000000000000000.json @@ -1,4 +1,4 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"90ceea80-98b2-4a16-99b4-bd8590371cc5","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"utf8\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int64\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int32\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int16\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int8\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float32\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float64\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(5,3)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date32\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830585678}} -{"add":{"path":"part-00000-9a4d4f91-4cab-4294-9e26-97785c9556fc-c000.snappy.parquet","partitionValues":{},"size":3206,"modificationTime":1668830585964,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"utf8\":\"0\",\"int64\":0,\"int32\":0,\"int16\":0,\"int8\":0,\"float32\":0.0,\"float64\":0.0,\"decimal\":10.000,\"date32\":\"1970-01-01\",\"timestamp\":\"1970-01-01T00:00:00.000-08:00\"},\"maxValues\":{\"utf8\":\"4\",\"int64\":4,\"int32\":4,\"int16\":4,\"int8\":4,\"float32\":4.0,\"float64\":4.0,\"decimal\":14.000,\"date32\":\"1970-01-05\",\"timestamp\":\"1970-01-01T04:00:00.000-08:00\"},\"nullCount\":{\"utf8\":0,\"int64\":0,\"int32\":0,\"int16\":0,\"int8\":0,\"float32\":0,\"float64\":0,\"bool\":0,\"binary\":0,\"decimal\":0,\"date32\":0,\"timestamp\":0}}"}} -{"commitInfo":{"timestamp":1668830585986,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"3206"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"46722e06-4b7f-463f-8608-94b227d9c98c"}} +{"metaData":{"id":"0f4744d1-b3e6-48b4-b484-292858112c44","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"utf8\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int64\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int32\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int16\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int8\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float32\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"float64\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(5,3)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date32\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830722564}} +{"add":{"path":"part-00000-2aa4115c-ce78-44bc-9cb8-9b509ffe48cd-c000.snappy.parquet","partitionValues":{},"size":3206,"modificationTime":1668830722824,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"utf8\":\"0\",\"int64\":0,\"int32\":0,\"int16\":0,\"int8\":0,\"float32\":0.0,\"float64\":0.0,\"decimal\":10.000,\"date32\":\"1970-01-01\",\"timestamp\":\"1970-01-01T00:00:00.000-08:00\"},\"maxValues\":{\"utf8\":\"4\",\"int64\":4,\"int32\":4,\"int16\":4,\"int8\":4,\"float32\":4.0,\"float64\":4.0,\"decimal\":14.000,\"date32\":\"1970-01-05\",\"timestamp\":\"1970-01-01T04:00:00.000-08:00\"},\"nullCount\":{\"utf8\":0,\"int64\":0,\"int32\":0,\"int16\":0,\"int8\":0,\"float32\":0,\"float64\":0,\"bool\":0,\"binary\":0,\"decimal\":0,\"date32\":0,\"timestamp\":0}}"}} +{"commitInfo":{"timestamp":1668830722845,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"3206"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"d3b15c24-3d91-4597-9d83-6f514b45b92c"}} diff --git a/out/reader_tests/generated/all_primitive_types/delta/part-00000-9a4d4f91-4cab-4294-9e26-97785c9556fc-c000.snappy.parquet b/out/reader_tests/generated/all_primitive_types/delta/part-00000-2aa4115c-ce78-44bc-9cb8-9b509ffe48cd-c000.snappy.parquet similarity index 88% rename from out/reader_tests/generated/all_primitive_types/delta/part-00000-9a4d4f91-4cab-4294-9e26-97785c9556fc-c000.snappy.parquet rename to out/reader_tests/generated/all_primitive_types/delta/part-00000-2aa4115c-ce78-44bc-9cb8-9b509ffe48cd-c000.snappy.parquet index ebaa59e..9feba61 100644 Binary files a/out/reader_tests/generated/all_primitive_types/delta/part-00000-9a4d4f91-4cab-4294-9e26-97785c9556fc-c000.snappy.parquet and b/out/reader_tests/generated/all_primitive_types/delta/part-00000-2aa4115c-ce78-44bc-9cb8-9b509ffe48cd-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000000.json index 26ed51a..853a662 100644 --- a/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000000.json @@ -1,4 +1,4 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"d379f8f1-60b4-456c-a941-20ac267cbea3","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830560516}} -{"add":{"path":"part-00000-7ced08f4-9725-4547-a3dd-1e39a1eaf7b0-c000.snappy.parquet","partitionValues":{},"size":996,"modificationTime":1668830562714,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} -{"commitInfo":{"timestamp":1668830562838,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"996"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"92a218de-ba83-4830-b62a-caa0ee1e35ad"}} +{"metaData":{"id":"40c6ddf2-93cd-414e-92fa-3496d3368465","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830697027}} +{"add":{"path":"part-00000-e3224358-9a1b-47d1-a581-6f0b434b2fc6-c000.snappy.parquet","partitionValues":{},"size":996,"modificationTime":1668830699344,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1,\"a_float\":1.1},\"maxValues\":{\"letter\":\"c\",\"number\":3,\"a_float\":3.3},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} +{"commitInfo":{"timestamp":1668830699482,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"996"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"924fed05-58ad-4fab-92e4-e3b059d63c73"}} diff --git a/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000001.json index f4862c6..fcbbfe1 100644 --- a/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000001.json +++ b/out/reader_tests/generated/basic_append/delta/_delta_log/00000000000000000001.json @@ -1,2 +1,2 @@ -{"add":{"path":"part-00000-127f8ce7-badc-4b8e-b585-674644c96f90-c000.snappy.parquet","partitionValues":{},"size":984,"modificationTime":1668830568044,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} -{"commitInfo":{"timestamp":1668830568065,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"984"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"53ffff03-f331-461a-aa26-0e1aa244f2ca"}} +{"add":{"path":"part-00000-5f3baff2-c3a8-4b38-ba53-cd71291b3fbd-c000.snappy.parquet","partitionValues":{},"size":984,"modificationTime":1668830704664,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"letter\":\"d\",\"number\":4,\"a_float\":4.4},\"maxValues\":{\"letter\":\"e\",\"number\":5,\"a_float\":5.5},\"nullCount\":{\"letter\":0,\"number\":0,\"a_float\":0}}"}} +{"commitInfo":{"timestamp":1668830704686,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"2","numOutputBytes":"984"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"abca60bd-c6e3-4b1f-bce7-f4ddef5f7f2f"}} diff --git a/out/reader_tests/generated/basic_append/delta/part-00000-127f8ce7-badc-4b8e-b585-674644c96f90-c000.snappy.parquet b/out/reader_tests/generated/basic_append/delta/part-00000-5f3baff2-c3a8-4b38-ba53-cd71291b3fbd-c000.snappy.parquet similarity index 80% rename from out/reader_tests/generated/basic_append/delta/part-00000-127f8ce7-badc-4b8e-b585-674644c96f90-c000.snappy.parquet rename to out/reader_tests/generated/basic_append/delta/part-00000-5f3baff2-c3a8-4b38-ba53-cd71291b3fbd-c000.snappy.parquet index 7a02f6a..0f728b6 100644 Binary files a/out/reader_tests/generated/basic_append/delta/part-00000-127f8ce7-badc-4b8e-b585-674644c96f90-c000.snappy.parquet and b/out/reader_tests/generated/basic_append/delta/part-00000-5f3baff2-c3a8-4b38-ba53-cd71291b3fbd-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_append/delta/part-00000-7ced08f4-9725-4547-a3dd-1e39a1eaf7b0-c000.snappy.parquet b/out/reader_tests/generated/basic_append/delta/part-00000-e3224358-9a1b-47d1-a581-6f0b434b2fc6-c000.snappy.parquet similarity index 83% rename from out/reader_tests/generated/basic_append/delta/part-00000-7ced08f4-9725-4547-a3dd-1e39a1eaf7b0-c000.snappy.parquet rename to out/reader_tests/generated/basic_append/delta/part-00000-e3224358-9a1b-47d1-a581-6f0b434b2fc6-c000.snappy.parquet index 7127058..5727575 100644 Binary files a/out/reader_tests/generated/basic_append/delta/part-00000-7ced08f4-9725-4547-a3dd-1e39a1eaf7b0-c000.snappy.parquet and b/out/reader_tests/generated/basic_append/delta/part-00000-e3224358-9a1b-47d1-a581-6f0b434b2fc6-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000000.json index 2c6cfb7..5b4cce9 100644 --- a/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000000.json @@ -1,6 +1,6 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"755af727-900c-4f84-a24a-221b84e8f118","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["letter"],"configuration":{},"createdTime":1668830570542}} -{"add":{"path":"letter=a/part-00000-620b9d86-2586-444e-8c2b-758e38e0fea8.c000.snappy.parquet","partitionValues":{"letter":"a"},"size":751,"modificationTime":1668830570934,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":1,\"a_float\":1.1},\"maxValues\":{\"number\":1,\"a_float\":1.1},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} -{"add":{"path":"letter=b/part-00000-0be3b8ce-bed0-437d-a1ef-fbde2f7ebd42.c000.snappy.parquet","partitionValues":{"letter":"b"},"size":751,"modificationTime":1668830570944,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":2,\"a_float\":2.2},\"maxValues\":{\"number\":2,\"a_float\":2.2},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} -{"add":{"path":"letter=c/part-00000-8f5d6eea-fe56-4ae0-8ea2-73ed6a8f0fd2.c000.snappy.parquet","partitionValues":{"letter":"c"},"size":751,"modificationTime":1668830570954,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":3,\"a_float\":3.3},\"maxValues\":{\"number\":3,\"a_float\":3.3},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} -{"commitInfo":{"timestamp":1668830570977,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"letter\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"2253"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"a5f10439-c065-43cc-bd1a-576ae5c2d808"}} +{"metaData":{"id":"d938daae-9286-44c9-bdba-85ceb2c83114","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a_float\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["letter"],"configuration":{},"createdTime":1668830707231}} +{"add":{"path":"letter=a/part-00000-d616f7af-ff8f-47c9-9825-19b0b8ee4b81.c000.snappy.parquet","partitionValues":{"letter":"a"},"size":751,"modificationTime":1668830707594,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":1,\"a_float\":1.1},\"maxValues\":{\"number\":1,\"a_float\":1.1},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} +{"add":{"path":"letter=b/part-00000-24b5390a-7249-4367-8fbb-e5c625e6f4ba.c000.snappy.parquet","partitionValues":{"letter":"b"},"size":751,"modificationTime":1668830707604,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":2,\"a_float\":2.2},\"maxValues\":{\"number\":2,\"a_float\":2.2},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} +{"add":{"path":"letter=c/part-00000-e8a51544-5d46-4975-802b-d5087b0733ec.c000.snappy.parquet","partitionValues":{"letter":"c"},"size":751,"modificationTime":1668830707614,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":3,\"a_float\":3.3},\"maxValues\":{\"number\":3,\"a_float\":3.3},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} +{"commitInfo":{"timestamp":1668830707636,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"letter\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"2253"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"de5d3356-28c4-409b-a7ac-844d76e20247"}} diff --git a/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000001.json index f25bfb0..ea2e96b 100644 --- a/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000001.json +++ b/out/reader_tests/generated/basic_partitioned/delta/_delta_log/00000000000000000001.json @@ -1,4 +1,4 @@ -{"add":{"path":"letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f282cf1-dc7e-48df-8091-cd5a7f7be0cf.c000.snappy.parquet","partitionValues":{"letter":null},"size":751,"modificationTime":1668830573074,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":6,\"a_float\":6.6},\"maxValues\":{\"number\":6,\"a_float\":6.6},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} -{"add":{"path":"letter=a/part-00000-213acc83-9e68-4366-bdc3-9074170c3b1f.c000.snappy.parquet","partitionValues":{"letter":"a"},"size":751,"modificationTime":1668830573084,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":4,\"a_float\":4.4},\"maxValues\":{\"number\":4,\"a_float\":4.4},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} -{"add":{"path":"letter=e/part-00000-02057cfc-7ca6-4930-bcd5-9528fe59d7cb.c000.snappy.parquet","partitionValues":{"letter":"e"},"size":750,"modificationTime":1668830573094,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":5,\"a_float\":5.5},\"maxValues\":{\"number\":5,\"a_float\":5.5},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} -{"commitInfo":{"timestamp":1668830573108,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"letter\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"2252"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"d79e464f-a050-4914-81bf-9e92ca7dcf35"}} +{"add":{"path":"letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f6b60ea-222c-4a49-97ad-6e353b1465f2.c000.snappy.parquet","partitionValues":{"letter":null},"size":751,"modificationTime":1668830709774,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":6,\"a_float\":6.6},\"maxValues\":{\"number\":6,\"a_float\":6.6},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} +{"add":{"path":"letter=a/part-00000-65928a1f-2de0-4630-8c78-efa763e004c6.c000.snappy.parquet","partitionValues":{"letter":"a"},"size":751,"modificationTime":1668830709794,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":4,\"a_float\":4.4},\"maxValues\":{\"number\":4,\"a_float\":4.4},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} +{"add":{"path":"letter=e/part-00000-5a2ceccf-c9a9-4adc-b56d-d5f618abb2db.c000.snappy.parquet","partitionValues":{"letter":"e"},"size":750,"modificationTime":1668830709804,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":5,\"a_float\":5.5},\"maxValues\":{\"number\":5,\"a_float\":5.5},\"nullCount\":{\"number\":0,\"a_float\":0}}"}} +{"commitInfo":{"timestamp":1668830709818,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"letter\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"2252"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"0471bed1-f4c8-402f-8a30-724618cf8778"}} diff --git a/out/reader_tests/generated/basic_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f282cf1-dc7e-48df-8091-cd5a7f7be0cf.c000.snappy.parquet b/out/reader_tests/generated/basic_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f6b60ea-222c-4a49-97ad-6e353b1465f2.c000.snappy.parquet similarity index 82% rename from out/reader_tests/generated/basic_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f282cf1-dc7e-48df-8091-cd5a7f7be0cf.c000.snappy.parquet rename to out/reader_tests/generated/basic_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f6b60ea-222c-4a49-97ad-6e353b1465f2.c000.snappy.parquet index 1317ebb..d038761 100644 Binary files a/out/reader_tests/generated/basic_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f282cf1-dc7e-48df-8091-cd5a7f7be0cf.c000.snappy.parquet and b/out/reader_tests/generated/basic_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/part-00000-4f6b60ea-222c-4a49-97ad-6e353b1465f2.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-213acc83-9e68-4366-bdc3-9074170c3b1f.c000.snappy.parquet b/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-65928a1f-2de0-4630-8c78-efa763e004c6.c000.snappy.parquet similarity index 82% rename from out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-213acc83-9e68-4366-bdc3-9074170c3b1f.c000.snappy.parquet rename to out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-65928a1f-2de0-4630-8c78-efa763e004c6.c000.snappy.parquet index 2923c40..d8ca806 100644 Binary files a/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-213acc83-9e68-4366-bdc3-9074170c3b1f.c000.snappy.parquet and b/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-65928a1f-2de0-4630-8c78-efa763e004c6.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-620b9d86-2586-444e-8c2b-758e38e0fea8.c000.snappy.parquet b/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-d616f7af-ff8f-47c9-9825-19b0b8ee4b81.c000.snappy.parquet similarity index 82% rename from out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-620b9d86-2586-444e-8c2b-758e38e0fea8.c000.snappy.parquet rename to out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-d616f7af-ff8f-47c9-9825-19b0b8ee4b81.c000.snappy.parquet index 405a41c..d2fcbc5 100644 Binary files a/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-620b9d86-2586-444e-8c2b-758e38e0fea8.c000.snappy.parquet and b/out/reader_tests/generated/basic_partitioned/delta/letter=a/part-00000-d616f7af-ff8f-47c9-9825-19b0b8ee4b81.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/delta/letter=b/part-00000-0be3b8ce-bed0-437d-a1ef-fbde2f7ebd42.c000.snappy.parquet b/out/reader_tests/generated/basic_partitioned/delta/letter=b/part-00000-24b5390a-7249-4367-8fbb-e5c625e6f4ba.c000.snappy.parquet similarity index 82% rename from out/reader_tests/generated/basic_partitioned/delta/letter=b/part-00000-0be3b8ce-bed0-437d-a1ef-fbde2f7ebd42.c000.snappy.parquet rename to out/reader_tests/generated/basic_partitioned/delta/letter=b/part-00000-24b5390a-7249-4367-8fbb-e5c625e6f4ba.c000.snappy.parquet index c9a527c..b3a3818 100644 Binary files a/out/reader_tests/generated/basic_partitioned/delta/letter=b/part-00000-0be3b8ce-bed0-437d-a1ef-fbde2f7ebd42.c000.snappy.parquet and b/out/reader_tests/generated/basic_partitioned/delta/letter=b/part-00000-24b5390a-7249-4367-8fbb-e5c625e6f4ba.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/delta/letter=c/part-00000-8f5d6eea-fe56-4ae0-8ea2-73ed6a8f0fd2.c000.snappy.parquet b/out/reader_tests/generated/basic_partitioned/delta/letter=c/part-00000-e8a51544-5d46-4975-802b-d5087b0733ec.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/basic_partitioned/delta/letter=c/part-00000-8f5d6eea-fe56-4ae0-8ea2-73ed6a8f0fd2.c000.snappy.parquet rename to out/reader_tests/generated/basic_partitioned/delta/letter=c/part-00000-e8a51544-5d46-4975-802b-d5087b0733ec.c000.snappy.parquet index f455b21..7831b3e 100644 Binary files a/out/reader_tests/generated/basic_partitioned/delta/letter=c/part-00000-8f5d6eea-fe56-4ae0-8ea2-73ed6a8f0fd2.c000.snappy.parquet and b/out/reader_tests/generated/basic_partitioned/delta/letter=c/part-00000-e8a51544-5d46-4975-802b-d5087b0733ec.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/delta/letter=e/part-00000-02057cfc-7ca6-4930-bcd5-9528fe59d7cb.c000.snappy.parquet b/out/reader_tests/generated/basic_partitioned/delta/letter=e/part-00000-5a2ceccf-c9a9-4adc-b56d-d5f618abb2db.c000.snappy.parquet similarity index 82% rename from out/reader_tests/generated/basic_partitioned/delta/letter=e/part-00000-02057cfc-7ca6-4930-bcd5-9528fe59d7cb.c000.snappy.parquet rename to out/reader_tests/generated/basic_partitioned/delta/letter=e/part-00000-5a2ceccf-c9a9-4adc-b56d-d5f618abb2db.c000.snappy.parquet index 7a8c0aa..e6de68f 100644 Binary files a/out/reader_tests/generated/basic_partitioned/delta/letter=e/part-00000-02057cfc-7ca6-4930-bcd5-9528fe59d7cb.c000.snappy.parquet and b/out/reader_tests/generated/basic_partitioned/delta/letter=e/part-00000-5a2ceccf-c9a9-4adc-b56d-d5f618abb2db.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/basic_partitioned/expected/v0/table_content.parquet b/out/reader_tests/generated/basic_partitioned/expected/v0/table_content.parquet index 8d1fceb..db448d8 100644 Binary files a/out/reader_tests/generated/basic_partitioned/expected/v0/table_content.parquet and b/out/reader_tests/generated/basic_partitioned/expected/v0/table_content.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000000.json index e6673d0..e597e0d 100644 --- a/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000000.json @@ -1,6 +1,6 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"aa879a43-f32b-4de9-a156-2c5b87f0b708","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["letter","date","data"],"configuration":{},"createdTime":1668830574958}} -{"add":{"path":"letter=a/date=1970-01-01/data=hello/part-00000-fa7e241e-b43a-46de-b79c-4a2cde0edfa0.c000.snappy.parquet","partitionValues":{"letter":"a","date":"1970-01-01","data":"hello"},"size":490,"modificationTime":1668830575274,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":1},\"maxValues\":{\"number\":1},\"nullCount\":{\"number\":0}}"}} -{"add":{"path":"letter=b/date=1970-01-01/data=world/part-00000-ee4df237-1cf7-46e9-84d4-2d6640a11919.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-01","data":"world"},"size":490,"modificationTime":1668830575294,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":2},\"maxValues\":{\"number\":2},\"nullCount\":{\"number\":0}}"}} -{"add":{"path":"letter=b/date=1970-01-02/data=world/part-00000-73a80378-204c-4209-a66c-b1bbf52b620b.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-02","data":"world"},"size":490,"modificationTime":1668830575304,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":3},\"maxValues\":{\"number\":3},\"nullCount\":{\"number\":0}}"}} -{"commitInfo":{"timestamp":1668830575322,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"letter\",\"date\",\"data\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1470"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"1c2fba6e-fa95-40ad-913c-78e0df8ee62e"}} +{"metaData":{"id":"a3796428-190f-4cbe-8a0a-c0783f17bd11","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["letter","date","data"],"configuration":{},"createdTime":1668830711870}} +{"add":{"path":"letter=a/date=1970-01-01/data=hello/part-00000-984aae4a-4e99-4653-b3fc-32fd101c0535.c000.snappy.parquet","partitionValues":{"letter":"a","date":"1970-01-01","data":"hello"},"size":490,"modificationTime":1668830712224,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":1},\"maxValues\":{\"number\":1},\"nullCount\":{\"number\":0}}"}} +{"add":{"path":"letter=b/date=1970-01-01/data=world/part-00000-80f79a38-e3d9-44a1-9a51-42939f670e67.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-01","data":"world"},"size":490,"modificationTime":1668830712234,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":2},\"maxValues\":{\"number\":2},\"nullCount\":{\"number\":0}}"}} +{"add":{"path":"letter=b/date=1970-01-02/data=world/part-00000-db5d40e6-977e-46c1-bf60-c47b0d38242a.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-02","data":"world"},"size":490,"modificationTime":1668830712264,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":3},\"maxValues\":{\"number\":3},\"nullCount\":{\"number\":0}}"}} +{"commitInfo":{"timestamp":1668830712279,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"letter\",\"date\",\"data\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1470"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"89feff2b-5207-4495-b5b5-e1f474364568"}} diff --git a/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000001.json index c10f089..86dccb9 100644 --- a/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000001.json +++ b/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000001.json @@ -1,3 +1,3 @@ -{"add":{"path":"letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-37fed4d0-6f84-4993-bfd6-8161f1838711.c000.snappy.parquet","partitionValues":{"letter":null,"date":null,"data":null},"size":490,"modificationTime":1668830577224,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":5},\"maxValues\":{\"number\":5},\"nullCount\":{\"number\":0}}"}} -{"add":{"path":"letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-c5c82af9-c3f4-4387-ac57-0a55c187cdf1.c000.snappy.parquet","partitionValues":{"letter":"a","date":null,"data":"x"},"size":490,"modificationTime":1668830577234,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":4},\"maxValues\":{\"number\":4},\"nullCount\":{\"number\":0}}"}} -{"commitInfo":{"timestamp":1668830577263,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"2","numOutputBytes":"980"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"cacffbba-f6d9-4376-84bb-b3203a9712e0"}} +{"add":{"path":"letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-e25645d1-bb23-4b64-8e36-d76fa325ab90.c000.snappy.parquet","partitionValues":{"letter":null,"date":null,"data":null},"size":490,"modificationTime":1668830714274,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":5},\"maxValues\":{\"number\":5},\"nullCount\":{\"number\":0}}"}} +{"add":{"path":"letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-e6297cfb-1ce6-4b4f-b52e-d4640a5dca94.c000.snappy.parquet","partitionValues":{"letter":"a","date":null,"data":"x"},"size":490,"modificationTime":1668830714294,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":4},\"maxValues\":{\"number\":4},\"nullCount\":{\"number\":0}}"}} +{"commitInfo":{"timestamp":1668830714315,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"2","numOutputBytes":"980"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"d231fb18-99ff-44d6-808c-aba8b1fb50d7"}} diff --git a/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000002.json b/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000002.json index c87deb6..49ac005 100644 --- a/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000002.json +++ b/out/reader_tests/generated/multi_partitioned/delta/_delta_log/00000000000000000002.json @@ -1,8 +1,8 @@ -{"add":{"path":"letter=%252F%252520%2525f/date=1970-01-01/data=hello/part-00000-ed6ef6ac-a7f5-413d-99f3-4c97f2c2f2e5.c000.snappy.parquet","partitionValues":{"letter":"/%20%f","date":"1970-01-01","data":"hello"},"size":490,"modificationTime":1668830578854,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":6},\"maxValues\":{\"number\":6},\"nullCount\":{\"number\":0}}"}} -{"add":{"path":"letter=b/date=1970-01-01/data=😈/part-00000-ca5359df-5567-4812-bb6d-dbf5166e5519.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-01","data":"😈"},"size":490,"modificationTime":1668830578864,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":7},\"maxValues\":{\"number\":7},\"nullCount\":{\"number\":0}}"}} -{"remove":{"path":"letter=b/date=1970-01-02/data=world/part-00000-73a80378-204c-4209-a66c-b1bbf52b620b.c000.snappy.parquet","deletionTimestamp":1668830579089,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"b","date":"1970-01-02","data":"world"},"size":490}} -{"remove":{"path":"letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-37fed4d0-6f84-4993-bfd6-8161f1838711.c000.snappy.parquet","deletionTimestamp":1668830579089,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":490}} -{"remove":{"path":"letter=b/date=1970-01-01/data=world/part-00000-ee4df237-1cf7-46e9-84d4-2d6640a11919.c000.snappy.parquet","deletionTimestamp":1668830579089,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"b","date":"1970-01-01","data":"world"},"size":490}} -{"remove":{"path":"letter=a/date=1970-01-01/data=hello/part-00000-fa7e241e-b43a-46de-b79c-4a2cde0edfa0.c000.snappy.parquet","deletionTimestamp":1668830579089,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"a","date":"1970-01-01","data":"hello"},"size":490}} -{"remove":{"path":"letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-c5c82af9-c3f4-4387-ac57-0a55c187cdf1.c000.snappy.parquet","deletionTimestamp":1668830579089,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"a","data":"x"},"size":490}} -{"commitInfo":{"timestamp":1668830579089,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"2","numOutputBytes":"980"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"8336812e-1645-4d04-b0ea-5645d601a8b7"}} +{"add":{"path":"letter=%252F%252520%2525f/date=1970-01-01/data=hello/part-00000-eec7f1b3-710b-40e7-ac75-33f4f4e9295e.c000.snappy.parquet","partitionValues":{"letter":"/%20%f","date":"1970-01-01","data":"hello"},"size":490,"modificationTime":1668830715884,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":6},\"maxValues\":{\"number\":6},\"nullCount\":{\"number\":0}}"}} +{"add":{"path":"letter=b/date=1970-01-01/data=😈/part-00000-ae6a0bb2-147d-42f1-b645-340ba99fdd3e.c000.snappy.parquet","partitionValues":{"letter":"b","date":"1970-01-01","data":"😈"},"size":490,"modificationTime":1668830715894,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"number\":7},\"maxValues\":{\"number\":7},\"nullCount\":{\"number\":0}}"}} +{"remove":{"path":"letter=b/date=1970-01-02/data=world/part-00000-db5d40e6-977e-46c1-bf60-c47b0d38242a.c000.snappy.parquet","deletionTimestamp":1668830716121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"b","date":"1970-01-02","data":"world"},"size":490}} +{"remove":{"path":"letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-e6297cfb-1ce6-4b4f-b52e-d4640a5dca94.c000.snappy.parquet","deletionTimestamp":1668830716121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"a","data":"x"},"size":490}} +{"remove":{"path":"letter=b/date=1970-01-01/data=world/part-00000-80f79a38-e3d9-44a1-9a51-42939f670e67.c000.snappy.parquet","deletionTimestamp":1668830716121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"b","date":"1970-01-01","data":"world"},"size":490}} +{"remove":{"path":"letter=a/date=1970-01-01/data=hello/part-00000-984aae4a-4e99-4653-b3fc-32fd101c0535.c000.snappy.parquet","deletionTimestamp":1668830716121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"letter":"a","date":"1970-01-01","data":"hello"},"size":490}} +{"remove":{"path":"letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-e25645d1-bb23-4b64-8e36-d76fa325ab90.c000.snappy.parquet","deletionTimestamp":1668830716121,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":490}} +{"commitInfo":{"timestamp":1668830716122,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"2","numOutputBytes":"980"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"23c4da6c-b5fc-4670-891b-5a63cd5c3ebf"}} diff --git a/out/reader_tests/generated/multi_partitioned/delta/letter=%2F%2520%25f/date=1970-01-01/data=hello/part-00000-ed6ef6ac-a7f5-413d-99f3-4c97f2c2f2e5.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned/delta/letter=%2F%2520%25f/date=1970-01-01/data=hello/part-00000-eec7f1b3-710b-40e7-ac75-33f4f4e9295e.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned/delta/letter=%2F%2520%25f/date=1970-01-01/data=hello/part-00000-ed6ef6ac-a7f5-413d-99f3-4c97f2c2f2e5.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned/delta/letter=%2F%2520%25f/date=1970-01-01/data=hello/part-00000-eec7f1b3-710b-40e7-ac75-33f4f4e9295e.c000.snappy.parquet index 9d1992a..5296ba3 100644 Binary files a/out/reader_tests/generated/multi_partitioned/delta/letter=%2F%2520%25f/date=1970-01-01/data=hello/part-00000-ed6ef6ac-a7f5-413d-99f3-4c97f2c2f2e5.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned/delta/letter=%2F%2520%25f/date=1970-01-01/data=hello/part-00000-eec7f1b3-710b-40e7-ac75-33f4f4e9295e.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-37fed4d0-6f84-4993-bfd6-8161f1838711.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-e25645d1-bb23-4b64-8e36-d76fa325ab90.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-37fed4d0-6f84-4993-bfd6-8161f1838711.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-e25645d1-bb23-4b64-8e36-d76fa325ab90.c000.snappy.parquet index a8c15da..a104cb9 100644 Binary files a/out/reader_tests/generated/multi_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-37fed4d0-6f84-4993-bfd6-8161f1838711.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned/delta/letter=__HIVE_DEFAULT_PARTITION__/date=__HIVE_DEFAULT_PARTITION__/data=__HIVE_DEFAULT_PARTITION__/part-00000-e25645d1-bb23-4b64-8e36-d76fa325ab90.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=1970-01-01/data=hello/part-00000-fa7e241e-b43a-46de-b79c-4a2cde0edfa0.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=1970-01-01/data=hello/part-00000-984aae4a-4e99-4653-b3fc-32fd101c0535.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned/delta/letter=a/date=1970-01-01/data=hello/part-00000-fa7e241e-b43a-46de-b79c-4a2cde0edfa0.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned/delta/letter=a/date=1970-01-01/data=hello/part-00000-984aae4a-4e99-4653-b3fc-32fd101c0535.c000.snappy.parquet index c31ea92..76afa0e 100644 Binary files a/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=1970-01-01/data=hello/part-00000-fa7e241e-b43a-46de-b79c-4a2cde0edfa0.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=1970-01-01/data=hello/part-00000-984aae4a-4e99-4653-b3fc-32fd101c0535.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-c5c82af9-c3f4-4387-ac57-0a55c187cdf1.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-e6297cfb-1ce6-4b4f-b52e-d4640a5dca94.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned/delta/letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-c5c82af9-c3f4-4387-ac57-0a55c187cdf1.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned/delta/letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-e6297cfb-1ce6-4b4f-b52e-d4640a5dca94.c000.snappy.parquet index b9c54a8..ae61170 100644 Binary files a/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-c5c82af9-c3f4-4387-ac57-0a55c187cdf1.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned/delta/letter=a/date=__HIVE_DEFAULT_PARTITION__/data=x/part-00000-e6297cfb-1ce6-4b4f-b52e-d4640a5dca94.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=world/part-00000-ee4df237-1cf7-46e9-84d4-2d6640a11919.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=world/part-00000-80f79a38-e3d9-44a1-9a51-42939f670e67.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=world/part-00000-ee4df237-1cf7-46e9-84d4-2d6640a11919.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=world/part-00000-80f79a38-e3d9-44a1-9a51-42939f670e67.c000.snappy.parquet index b01e022..ea12589 100644 Binary files a/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=world/part-00000-ee4df237-1cf7-46e9-84d4-2d6640a11919.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=world/part-00000-80f79a38-e3d9-44a1-9a51-42939f670e67.c000.snappy.parquet differ diff --git "a/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=\360\237\230\210/part-00000-ca5359df-5567-4812-bb6d-dbf5166e5519.c000.snappy.parquet" "b/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=\360\237\230\210/part-00000-ae6a0bb2-147d-42f1-b645-340ba99fdd3e.c000.snappy.parquet" similarity index 86% rename from "out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=\360\237\230\210/part-00000-ca5359df-5567-4812-bb6d-dbf5166e5519.c000.snappy.parquet" rename to "out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=\360\237\230\210/part-00000-ae6a0bb2-147d-42f1-b645-340ba99fdd3e.c000.snappy.parquet" index 6956cc1..144e5fb 100644 Binary files "a/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=\360\237\230\210/part-00000-ca5359df-5567-4812-bb6d-dbf5166e5519.c000.snappy.parquet" and "b/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-01/data=\360\237\230\210/part-00000-ae6a0bb2-147d-42f1-b645-340ba99fdd3e.c000.snappy.parquet" differ diff --git a/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-02/data=world/part-00000-73a80378-204c-4209-a66c-b1bbf52b620b.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-02/data=world/part-00000-db5d40e6-977e-46c1-bf60-c47b0d38242a.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-02/data=world/part-00000-73a80378-204c-4209-a66c-b1bbf52b620b.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-02/data=world/part-00000-db5d40e6-977e-46c1-bf60-c47b0d38242a.c000.snappy.parquet index 5ac8f58..c7eafb2 100644 Binary files a/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-02/data=world/part-00000-73a80378-204c-4209-a66c-b1bbf52b620b.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned/delta/letter=b/date=1970-01-02/data=world/part-00000-db5d40e6-977e-46c1-bf60-c47b0d38242a.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned_2/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/multi_partitioned_2/delta/_delta_log/00000000000000000000.json index 7fcc4d3..2266d70 100644 --- a/out/reader_tests/generated/multi_partitioned_2/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/multi_partitioned_2/delta/_delta_log/00000000000000000000.json @@ -1,6 +1,6 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"2466465d-f524-4eba-bb29-f08318d55f7d","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"amount\",\"type\":\"decimal(38,18)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["bool","time","amount"],"configuration":{},"createdTime":1668830581012}} -{"add":{"path":"bool=false/time=1970-01-02%2008%253A45%253A00/amount=12.000000000000000000/part-00000-1f7682b8-a6e8-49e8-aff8-d9913a95ce4e.c000.snappy.parquet","partitionValues":{"bool":"false","time":"1970-01-02 08:45:00","amount":"12.000000000000000000"},"size":481,"modificationTime":1668830581214,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"int\":3},\"maxValues\":{\"int\":3},\"nullCount\":{\"int\":0}}"}} -{"add":{"path":"bool=true/time=1970-01-01%2000%253A00%253A00/amount=200.000000000000000000/part-00000-1917db48-e927-4880-b6b0-1377030d67d1.c000.snappy.parquet","partitionValues":{"bool":"true","time":"1970-01-01 00:00:00","amount":"200.000000000000000000"},"size":481,"modificationTime":1668830581224,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"int\":1},\"maxValues\":{\"int\":1},\"nullCount\":{\"int\":0}}"}} -{"add":{"path":"bool=true/time=1970-01-01%2012%253A30%253A00/amount=200.000000000000000000/part-00000-889fcbb8-e13f-40ce-942d-8ee22a83b66a.c000.snappy.parquet","partitionValues":{"bool":"true","time":"1970-01-01 12:30:00","amount":"200.000000000000000000"},"size":481,"modificationTime":1668830581234,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"int\":2},\"maxValues\":{\"int\":2},\"nullCount\":{\"int\":0}}"}} -{"commitInfo":{"timestamp":1668830581253,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"bool\",\"time\",\"amount\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1443"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"75a2740f-9b4e-481b-898b-a4db97ffd693"}} +{"metaData":{"id":"48bd3baa-ea4a-499b-b553-72d883deb29b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"time\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"amount\",\"type\":\"decimal(38,18)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["bool","time","amount"],"configuration":{},"createdTime":1668830717818}} +{"add":{"path":"bool=false/time=1970-01-02%2008%253A45%253A00/amount=12.000000000000000000/part-00000-fb43e8a9-0eda-4a01-8dd7-94a9455ea10f.c000.snappy.parquet","partitionValues":{"bool":"false","time":"1970-01-02 08:45:00","amount":"12.000000000000000000"},"size":481,"modificationTime":1668830718034,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"int\":3},\"maxValues\":{\"int\":3},\"nullCount\":{\"int\":0}}"}} +{"add":{"path":"bool=true/time=1970-01-01%2000%253A00%253A00/amount=200.000000000000000000/part-00000-91592a25-a5b1-48ce-b826-f12bf58f526d.c000.snappy.parquet","partitionValues":{"bool":"true","time":"1970-01-01 00:00:00","amount":"200.000000000000000000"},"size":481,"modificationTime":1668830718054,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"int\":1},\"maxValues\":{\"int\":1},\"nullCount\":{\"int\":0}}"}} +{"add":{"path":"bool=true/time=1970-01-01%2012%253A30%253A00/amount=200.000000000000000000/part-00000-055a2a73-0afe-469e-a336-2a2d6dc62c34.c000.snappy.parquet","partitionValues":{"bool":"true","time":"1970-01-01 12:30:00","amount":"200.000000000000000000"},"size":481,"modificationTime":1668830718064,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"int\":2},\"maxValues\":{\"int\":2},\"nullCount\":{\"int\":0}}"}} +{"commitInfo":{"timestamp":1668830718078,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"bool\",\"time\",\"amount\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1443"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"5ed2d91d-72d7-4646-8fb4-f6f240450795"}} diff --git a/out/reader_tests/generated/multi_partitioned_2/delta/bool=false/time=1970-01-02 08%3A45%3A00/amount=12.000000000000000000/part-00000-1f7682b8-a6e8-49e8-aff8-d9913a95ce4e.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned_2/delta/bool=false/time=1970-01-02 08%3A45%3A00/amount=12.000000000000000000/part-00000-fb43e8a9-0eda-4a01-8dd7-94a9455ea10f.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned_2/delta/bool=false/time=1970-01-02 08%3A45%3A00/amount=12.000000000000000000/part-00000-1f7682b8-a6e8-49e8-aff8-d9913a95ce4e.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned_2/delta/bool=false/time=1970-01-02 08%3A45%3A00/amount=12.000000000000000000/part-00000-fb43e8a9-0eda-4a01-8dd7-94a9455ea10f.c000.snappy.parquet index e20d87b..e302cbe 100644 Binary files a/out/reader_tests/generated/multi_partitioned_2/delta/bool=false/time=1970-01-02 08%3A45%3A00/amount=12.000000000000000000/part-00000-1f7682b8-a6e8-49e8-aff8-d9913a95ce4e.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned_2/delta/bool=false/time=1970-01-02 08%3A45%3A00/amount=12.000000000000000000/part-00000-fb43e8a9-0eda-4a01-8dd7-94a9455ea10f.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 00%3A00%3A00/amount=200.000000000000000000/part-00000-1917db48-e927-4880-b6b0-1377030d67d1.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 00%3A00%3A00/amount=200.000000000000000000/part-00000-91592a25-a5b1-48ce-b826-f12bf58f526d.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 00%3A00%3A00/amount=200.000000000000000000/part-00000-1917db48-e927-4880-b6b0-1377030d67d1.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 00%3A00%3A00/amount=200.000000000000000000/part-00000-91592a25-a5b1-48ce-b826-f12bf58f526d.c000.snappy.parquet index f0d1b70..e0c47a0 100644 Binary files a/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 00%3A00%3A00/amount=200.000000000000000000/part-00000-1917db48-e927-4880-b6b0-1377030d67d1.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 00%3A00%3A00/amount=200.000000000000000000/part-00000-91592a25-a5b1-48ce-b826-f12bf58f526d.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 12%3A30%3A00/amount=200.000000000000000000/part-00000-889fcbb8-e13f-40ce-942d-8ee22a83b66a.c000.snappy.parquet b/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 12%3A30%3A00/amount=200.000000000000000000/part-00000-055a2a73-0afe-469e-a336-2a2d6dc62c34.c000.snappy.parquet similarity index 86% rename from out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 12%3A30%3A00/amount=200.000000000000000000/part-00000-889fcbb8-e13f-40ce-942d-8ee22a83b66a.c000.snappy.parquet rename to out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 12%3A30%3A00/amount=200.000000000000000000/part-00000-055a2a73-0afe-469e-a336-2a2d6dc62c34.c000.snappy.parquet index 52ae8a6..647c614 100644 Binary files a/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 12%3A30%3A00/amount=200.000000000000000000/part-00000-889fcbb8-e13f-40ce-942d-8ee22a83b66a.c000.snappy.parquet and b/out/reader_tests/generated/multi_partitioned_2/delta/bool=true/time=1970-01-01 12%3A30%3A00/amount=200.000000000000000000/part-00000-055a2a73-0afe-469e-a336-2a2d6dc62c34.c000.snappy.parquet differ diff --git a/out/reader_tests/generated/nested_types/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/nested_types/delta/_delta_log/00000000000000000000.json index 0abd968..26e6f02 100644 --- a/out/reader_tests/generated/nested_types/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/nested_types/delta/_delta_log/00000000000000000000.json @@ -1,4 +1,4 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"a77f08ef-8a6c-4aed-8a3b-14a097f12563","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"float64\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"array\",\"type\":{\"type\":\"array\",\"elementType\":\"short\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830587297}} -{"add":{"path":"part-00000-180fee3b-7919-4e3f-8b90-4fcba3ba3414-c000.snappy.parquet","partitionValues":{},"size":1842,"modificationTime":1668830587544,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"struct\":{\"float64\":0.0}},\"maxValues\":{\"struct\":{\"float64\":4.0}},\"nullCount\":{\"struct\":{\"float64\":0,\"bool\":0},\"array\":0,\"map\":0}}"}} -{"commitInfo":{"timestamp":1668830587567,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1842"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"6c02ab82-8bcf-4d37-9f58-6f1493386062"}} +{"metaData":{"id":"89c145b7-7355-4040-92dd-ab7a424116b7","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"float64\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"bool\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"array\",\"type\":{\"type\":\"array\",\"elementType\":\"short\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"integer\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830724010}} +{"add":{"path":"part-00000-16c511ec-8da2-46dc-81ba-a58cb990d77b-c000.snappy.parquet","partitionValues":{},"size":1842,"modificationTime":1668830724194,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"struct\":{\"float64\":0.0}},\"maxValues\":{\"struct\":{\"float64\":4.0}},\"nullCount\":{\"struct\":{\"float64\":0,\"bool\":0},\"array\":0,\"map\":0}}"}} +{"commitInfo":{"timestamp":1668830724216,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1842"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"4387a438-2743-4549-9dbb-5a0031bfe73d"}} diff --git a/out/reader_tests/generated/nested_types/delta/part-00000-180fee3b-7919-4e3f-8b90-4fcba3ba3414-c000.snappy.parquet b/out/reader_tests/generated/nested_types/delta/part-00000-16c511ec-8da2-46dc-81ba-a58cb990d77b-c000.snappy.parquet similarity index 96% rename from out/reader_tests/generated/nested_types/delta/part-00000-180fee3b-7919-4e3f-8b90-4fcba3ba3414-c000.snappy.parquet rename to out/reader_tests/generated/nested_types/delta/part-00000-16c511ec-8da2-46dc-81ba-a58cb990d77b-c000.snappy.parquet index cd0bce8..8507e55 100644 Binary files a/out/reader_tests/generated/nested_types/delta/part-00000-180fee3b-7919-4e3f-8b90-4fcba3ba3414-c000.snappy.parquet and b/out/reader_tests/generated/nested_types/delta/part-00000-16c511ec-8da2-46dc-81ba-a58cb990d77b-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/no_replay/delta/_delta_log/._last_checkpoint.crc b/out/reader_tests/generated/no_replay/delta/_delta_log/._last_checkpoint.crc new file mode 100644 index 0000000..0ed415f Binary files /dev/null and b/out/reader_tests/generated/no_replay/delta/_delta_log/._last_checkpoint.crc differ diff --git a/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000002.checkpoint.parquet b/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000002.checkpoint.parquet new file mode 100644 index 0000000..5145e53 Binary files /dev/null and b/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000002.checkpoint.parquet differ diff --git a/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000002.json b/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000002.json new file mode 100644 index 0000000..51e7c92 --- /dev/null +++ b/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-97768067-cdda-4b62-8619-45919d1e0336-c000.snappy.parquet","partitionValues":{},"size":976,"modificationTime":1668830732474,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":120,\"date\":\"1971-07-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":667,\"date\":\"2018-02-01\"},\"nullCount\":{\"letter\":2,\"int\":0,\"date\":0}}"}} +{"remove":{"path":"part-00000-290f59b5-ec6c-4dde-9e69-aac929934233-c000.snappy.parquet","deletionTimestamp":1668830732657,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":965}} +{"commitInfo":{"timestamp":1668830732657,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"976"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"9662a209-105d-4690-8ab0-d549df6560d4"}} diff --git a/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000003.json b/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000003.json new file mode 100644 index 0000000..18125a0 --- /dev/null +++ b/out/reader_tests/generated/no_replay/delta/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-153a2d45-c3b5-4561-9d15-253edae7bb62-c000.snappy.parquet","partitionValues":{},"size":1010,"modificationTime":1668830733794,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":93,\"date\":\"1975-06-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":753,\"date\":\"2013-03-01\"},\"nullCount\":{\"letter\":1,\"int\":0,\"date\":0}}"}} +{"remove":{"path":"part-00000-97768067-cdda-4b62-8619-45919d1e0336-c000.snappy.parquet","deletionTimestamp":1668830733981,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":976}} +{"commitInfo":{"timestamp":1668830733982,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1010"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"299c18a5-ef66-46a9-ad02-a281c05ab0cf"}} diff --git a/out/reader_tests/generated/no_replay/delta/_delta_log/_last_checkpoint b/out/reader_tests/generated/no_replay/delta/_delta_log/_last_checkpoint new file mode 100644 index 0000000..311182d --- /dev/null +++ b/out/reader_tests/generated/no_replay/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":2,"size":4,"sizeInBytes":12712,"numOfAddFiles":1,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"3c1a4a7ad9b68673d4095534b71f050a"} diff --git a/out/reader_tests/generated/no_replay/delta/part-00000-153a2d45-c3b5-4561-9d15-253edae7bb62-c000.snappy.parquet b/out/reader_tests/generated/no_replay/delta/part-00000-153a2d45-c3b5-4561-9d15-253edae7bb62-c000.snappy.parquet new file mode 100644 index 0000000..0556bc1 Binary files /dev/null and b/out/reader_tests/generated/no_replay/delta/part-00000-153a2d45-c3b5-4561-9d15-253edae7bb62-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/no_replay/expected/latest/table_content.parquet b/out/reader_tests/generated/no_replay/expected/latest/table_content.parquet new file mode 100644 index 0000000..e79b239 Binary files /dev/null and b/out/reader_tests/generated/no_replay/expected/latest/table_content.parquet differ diff --git a/out/reader_tests/generated/no_replay/expected/latest/table_version_metadata.json b/out/reader_tests/generated/no_replay/expected/latest/table_version_metadata.json new file mode 100644 index 0000000..0dd8463 --- /dev/null +++ b/out/reader_tests/generated/no_replay/expected/latest/table_version_metadata.json @@ -0,0 +1,8 @@ +{ + "version": 3, + "properties": { + "delta.checkpointInterval": "2" + }, + "min_reader_version": 1, + "min_writer_version": 2 +} \ No newline at end of file diff --git a/out/reader_tests/generated/no_replay/test_case_info.json b/out/reader_tests/generated/no_replay/test_case_info.json new file mode 100644 index 0000000..4d7565d --- /dev/null +++ b/out/reader_tests/generated/no_replay/test_case_info.json @@ -0,0 +1,4 @@ +{ + "name": "no_replay", + "description": "Table with a checkpoint and prior commits cleaned up" +} \ No newline at end of file diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/._last_checkpoint.crc b/out/reader_tests/generated/no_stats/delta/_delta_log/._last_checkpoint.crc new file mode 100644 index 0000000..9c45b14 Binary files /dev/null and b/out/reader_tests/generated/no_stats/delta/_delta_log/._last_checkpoint.crc differ diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000000.json new file mode 100644 index 0000000..e3f1f7d --- /dev/null +++ b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"5f6e6c6d-21f8-449a-8bee-cf171d3dcbb7","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsStruct":"false","delta.dataSkippingNumIndexedCols":"0","delta.checkpoint.writeStatsAsJson":"false","delta.checkpointInterval":"2"},"createdTime":1668830755617}} +{"commitInfo":{"timestamp":1668830755622,"operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{\"delta.checkpoint.writeStatsAsStruct\":\"false\",\"delta.dataSkippingNumIndexedCols\":\"0\",\"delta.checkpoint.writeStatsAsJson\":\"false\",\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"aaedb685-0371-45cb-ac3c-052d6840c2d2"}} diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000001.json new file mode 100644 index 0000000..6e38099 --- /dev/null +++ b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"add":{"path":"part-00000-30bb1ceb-4ec4-4edf-a46a-98b83b61f227-c000.snappy.parquet","partitionValues":{},"size":965,"modificationTime":1668830756434,"dataChange":true,"stats":"{\"numRecords\":5}"}} +{"commitInfo":{"timestamp":1668830756602,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"965"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"eec5f97a-b146-4722-bdb7-ca9e10746f1b"}} diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000002.checkpoint.parquet b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000002.checkpoint.parquet new file mode 100644 index 0000000..dc0708d Binary files /dev/null and b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000002.checkpoint.parquet differ diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000002.json b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000002.json new file mode 100644 index 0000000..3b5a3b9 --- /dev/null +++ b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-74b4ed44-31b2-49b6-8a93-a9892a61cf66-c000.snappy.parquet","partitionValues":{},"size":976,"modificationTime":1668830757374,"dataChange":true,"stats":"{\"numRecords\":5}"}} +{"remove":{"path":"part-00000-30bb1ceb-4ec4-4edf-a46a-98b83b61f227-c000.snappy.parquet","deletionTimestamp":1668830757553,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":965}} +{"commitInfo":{"timestamp":1668830757553,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"976"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"a0dababd-feb2-40f3-9041-95786a476262"}} diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000003.json b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000003.json new file mode 100644 index 0000000..6163e4c --- /dev/null +++ b/out/reader_tests/generated/no_stats/delta/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-7caaa98a-7f34-4104-8b05-5191f68aaad3-c000.snappy.parquet","partitionValues":{},"size":1010,"modificationTime":1668830758504,"dataChange":true,"stats":"{\"numRecords\":5}"}} +{"remove":{"path":"part-00000-74b4ed44-31b2-49b6-8a93-a9892a61cf66-c000.snappy.parquet","deletionTimestamp":1668830758653,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":976}} +{"commitInfo":{"timestamp":1668830758653,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1010"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"4682aefb-a85e-4931-9df9-05e966bed60e"}} diff --git a/out/reader_tests/generated/no_stats/delta/_delta_log/_last_checkpoint b/out/reader_tests/generated/no_stats/delta/_delta_log/_last_checkpoint new file mode 100644 index 0000000..21ad055 --- /dev/null +++ b/out/reader_tests/generated/no_stats/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":2,"size":4,"sizeInBytes":11636,"numOfAddFiles":1,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"2810b06500b0e86962168064dbae9968"} diff --git a/out/reader_tests/generated/no_stats/delta/part-00000-30bb1ceb-4ec4-4edf-a46a-98b83b61f227-c000.snappy.parquet b/out/reader_tests/generated/no_stats/delta/part-00000-30bb1ceb-4ec4-4edf-a46a-98b83b61f227-c000.snappy.parquet new file mode 100644 index 0000000..6a69c83 Binary files /dev/null and b/out/reader_tests/generated/no_stats/delta/part-00000-30bb1ceb-4ec4-4edf-a46a-98b83b61f227-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/no_stats/delta/part-00000-74b4ed44-31b2-49b6-8a93-a9892a61cf66-c000.snappy.parquet b/out/reader_tests/generated/no_stats/delta/part-00000-74b4ed44-31b2-49b6-8a93-a9892a61cf66-c000.snappy.parquet new file mode 100644 index 0000000..c8a6c89 Binary files /dev/null and b/out/reader_tests/generated/no_stats/delta/part-00000-74b4ed44-31b2-49b6-8a93-a9892a61cf66-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/no_stats/delta/part-00000-7caaa98a-7f34-4104-8b05-5191f68aaad3-c000.snappy.parquet b/out/reader_tests/generated/no_stats/delta/part-00000-7caaa98a-7f34-4104-8b05-5191f68aaad3-c000.snappy.parquet new file mode 100644 index 0000000..0556bc1 Binary files /dev/null and b/out/reader_tests/generated/no_stats/delta/part-00000-7caaa98a-7f34-4104-8b05-5191f68aaad3-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/no_stats/expected/latest/table_content.parquet b/out/reader_tests/generated/no_stats/expected/latest/table_content.parquet new file mode 100644 index 0000000..e79b239 Binary files /dev/null and b/out/reader_tests/generated/no_stats/expected/latest/table_content.parquet differ diff --git a/out/reader_tests/generated/no_stats/expected/latest/table_version_metadata.json b/out/reader_tests/generated/no_stats/expected/latest/table_version_metadata.json new file mode 100644 index 0000000..7de2521 --- /dev/null +++ b/out/reader_tests/generated/no_stats/expected/latest/table_version_metadata.json @@ -0,0 +1,11 @@ +{ + "version": 3, + "properties": { + "delta.checkpoint.writeStatsAsJson": "false", + "delta.checkpoint.writeStatsAsStruct": "false", + "delta.dataSkippingNumIndexedCols": "0", + "delta.checkpointInterval": "2" + }, + "min_reader_version": 1, + "min_writer_version": 2 +} \ No newline at end of file diff --git a/out/reader_tests/generated/no_stats/test_case_info.json b/out/reader_tests/generated/no_stats/test_case_info.json new file mode 100644 index 0000000..16103b0 --- /dev/null +++ b/out/reader_tests/generated/no_stats/test_case_info.json @@ -0,0 +1,4 @@ +{ + "name": "no_stats", + "description": "Table with no stats" +} \ No newline at end of file diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/._last_checkpoint.crc b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/._last_checkpoint.crc new file mode 100644 index 0000000..e145158 Binary files /dev/null and b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/._last_checkpoint.crc differ diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000000.json new file mode 100644 index 0000000..a47c9f3 --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"61b97425-df1b-438b-90df-25c6385aa17a","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpoint.writeStatsAsStruct":"true","delta.checkpoint.writeStatsAsJson":"false","delta.checkpointInterval":"2"},"createdTime":1668830751638}} +{"commitInfo":{"timestamp":1668830751643,"operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{\"delta.checkpoint.writeStatsAsStruct\":\"true\",\"delta.checkpoint.writeStatsAsJson\":\"false\",\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"76560977-da91-4aee-b0d5-198450d89e18"}} diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000001.json new file mode 100644 index 0000000..6515bbc --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"add":{"path":"part-00000-f5d3c3c9-4859-4415-877a-ef94def22f1d-c000.snappy.parquet","partitionValues":{},"size":965,"modificationTime":1668830752514,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"b\",\"int\":288,\"date\":\"1978-02-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":988,\"date\":\"2020-05-01\"},\"nullCount\":{\"letter\":3,\"int\":0,\"date\":0}}"}} +{"commitInfo":{"timestamp":1668830752684,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"965"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"e86731a9-231e-479a-b8c6-49d7ee189cff"}} diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000002.checkpoint.parquet b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000002.checkpoint.parquet new file mode 100644 index 0000000..6680d02 Binary files /dev/null and b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000002.checkpoint.parquet differ diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000002.json b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000002.json new file mode 100644 index 0000000..2a12425 --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-ee36045d-3936-44e4-b6e3-f2f7ce8fef2f-c000.snappy.parquet","partitionValues":{},"size":976,"modificationTime":1668830753444,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":120,\"date\":\"1971-07-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":667,\"date\":\"2018-02-01\"},\"nullCount\":{\"letter\":2,\"int\":0,\"date\":0}}"}} +{"remove":{"path":"part-00000-f5d3c3c9-4859-4415-877a-ef94def22f1d-c000.snappy.parquet","deletionTimestamp":1668830753595,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":965}} +{"commitInfo":{"timestamp":1668830753595,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"976"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"877a7dc9-8078-4d9b-8210-ce0657baee47"}} diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000003.json b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000003.json new file mode 100644 index 0000000..ac6e00d --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"add":{"path":"part-00000-d47d1f3a-a5c0-4f52-b420-8acaa5b92c15-c000.snappy.parquet","partitionValues":{},"size":1010,"modificationTime":1668830754604,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":93,\"date\":\"1975-06-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":753,\"date\":\"2013-03-01\"},\"nullCount\":{\"letter\":1,\"int\":0,\"date\":0}}"}} +{"remove":{"path":"part-00000-ee36045d-3936-44e4-b6e3-f2f7ce8fef2f-c000.snappy.parquet","deletionTimestamp":1668830754749,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":976}} +{"commitInfo":{"timestamp":1668830754749,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1010"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"39f4142f-e5c3-4c1e-ae1c-b7c4ba73ba4a"}} diff --git a/out/reader_tests/generated/stats_as_struct/delta/_delta_log/_last_checkpoint b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/_last_checkpoint new file mode 100644 index 0000000..5748a38 --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":2,"size":4,"sizeInBytes":11551,"numOfAddFiles":1,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"021ca7a06fdca9eec0916077be873aa7"} diff --git a/out/reader_tests/generated/stats_as_struct/delta/part-00000-d47d1f3a-a5c0-4f52-b420-8acaa5b92c15-c000.snappy.parquet b/out/reader_tests/generated/stats_as_struct/delta/part-00000-d47d1f3a-a5c0-4f52-b420-8acaa5b92c15-c000.snappy.parquet new file mode 100644 index 0000000..0556bc1 Binary files /dev/null and b/out/reader_tests/generated/stats_as_struct/delta/part-00000-d47d1f3a-a5c0-4f52-b420-8acaa5b92c15-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/stats_as_struct/delta/part-00000-ee36045d-3936-44e4-b6e3-f2f7ce8fef2f-c000.snappy.parquet b/out/reader_tests/generated/stats_as_struct/delta/part-00000-ee36045d-3936-44e4-b6e3-f2f7ce8fef2f-c000.snappy.parquet new file mode 100644 index 0000000..c8a6c89 Binary files /dev/null and b/out/reader_tests/generated/stats_as_struct/delta/part-00000-ee36045d-3936-44e4-b6e3-f2f7ce8fef2f-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/stats_as_struct/delta/part-00000-f5d3c3c9-4859-4415-877a-ef94def22f1d-c000.snappy.parquet b/out/reader_tests/generated/stats_as_struct/delta/part-00000-f5d3c3c9-4859-4415-877a-ef94def22f1d-c000.snappy.parquet new file mode 100644 index 0000000..6a69c83 Binary files /dev/null and b/out/reader_tests/generated/stats_as_struct/delta/part-00000-f5d3c3c9-4859-4415-877a-ef94def22f1d-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/stats_as_struct/expected/latest/table_content.parquet b/out/reader_tests/generated/stats_as_struct/expected/latest/table_content.parquet new file mode 100644 index 0000000..e79b239 Binary files /dev/null and b/out/reader_tests/generated/stats_as_struct/expected/latest/table_content.parquet differ diff --git a/out/reader_tests/generated/stats_as_struct/expected/latest/table_version_metadata.json b/out/reader_tests/generated/stats_as_struct/expected/latest/table_version_metadata.json new file mode 100644 index 0000000..af20c60 --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/expected/latest/table_version_metadata.json @@ -0,0 +1,10 @@ +{ + "version": 3, + "properties": { + "delta.checkpoint.writeStatsAsStruct": "true", + "delta.checkpointInterval": "2", + "delta.checkpoint.writeStatsAsJson": "false" + }, + "min_reader_version": 1, + "min_writer_version": 2 +} \ No newline at end of file diff --git a/out/reader_tests/generated/stats_as_struct/test_case_info.json b/out/reader_tests/generated/stats_as_struct/test_case_info.json new file mode 100644 index 0000000..f7eaa12 --- /dev/null +++ b/out/reader_tests/generated/stats_as_struct/test_case_info.json @@ -0,0 +1,4 @@ +{ + "name": "stats_as_struct", + "description": "Table with stats only written as struct (not JSON) with Checkpoint" +} \ No newline at end of file diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/._last_checkpoint.crc b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/._last_checkpoint.crc index a4d2608..0ed415f 100644 Binary files a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/._last_checkpoint.crc and b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/._last_checkpoint.crc differ diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000000.json index 12023b2..ed17767 100644 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000000.json @@ -1,3 +1,3 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"cefd4403-bb16-497d-92d8-ea62c51adb47","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.logRetentionDuration":"0 days","delta.checkpointInterval":"2"},"createdTime":1668830588704}} -{"commitInfo":{"timestamp":1668830588712,"operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{\"delta.logRetentionDuration\":\"0 days\",\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"afdf3047-71d4-4b6a-a670-152203f29023"}} +{"metaData":{"id":"8ff7eebe-c1b8-430f-b553-c39b39864fca","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"int\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1668830725517}} +{"commitInfo":{"timestamp":1668830725525,"operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"9308c36d-7f36-434c-8f2d-c338bd387eff"}} diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000001.json index 8e80292..45871ac 100644 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000001.json +++ b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000001.json @@ -1,2 +1,2 @@ -{"add":{"path":"part-00000-2e555f10-73cc-4e32-8f32-fcedfb1398dc-c000.snappy.parquet","partitionValues":{},"size":965,"modificationTime":1668830589724,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"b\",\"int\":288,\"date\":\"1978-02-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":988,\"date\":\"2020-05-01\"},\"nullCount\":{\"letter\":3,\"int\":0,\"date\":0}}"}} -{"commitInfo":{"timestamp":1668830589904,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"965"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"77b8a598-0d27-4fb6-9aa9-682b4b36db4a"}} +{"add":{"path":"part-00000-489b4323-6f66-4fac-9228-01848530b142-c000.snappy.parquet","partitionValues":{},"size":965,"modificationTime":1668830726594,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"b\",\"int\":288,\"date\":\"1978-02-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":988,\"date\":\"2020-05-01\"},\"nullCount\":{\"letter\":3,\"int\":0,\"date\":0}}"}} +{"commitInfo":{"timestamp":1668830726748,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"965"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"60701ae4-48fc-40ec-9dc2-a4ac96f0e3f0"}} diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.checkpoint.parquet b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.checkpoint.parquet index 9a3ed74..94450a5 100644 Binary files a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.checkpoint.parquet and b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.checkpoint.parquet differ diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.json b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.json index 9aeabbd..2435924 100644 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.json +++ b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000002.json @@ -1,3 +1,3 @@ -{"add":{"path":"part-00000-c4d01b94-46fa-4b26-8002-c3870554b939-c000.snappy.parquet","partitionValues":{},"size":976,"modificationTime":1668830590864,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":120,\"date\":\"1971-07-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":667,\"date\":\"2018-02-01\"},\"nullCount\":{\"letter\":2,\"int\":0,\"date\":0}}"}} -{"remove":{"path":"part-00000-2e555f10-73cc-4e32-8f32-fcedfb1398dc-c000.snappy.parquet","deletionTimestamp":1668830591068,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":965}} -{"commitInfo":{"timestamp":1668830591068,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"976"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"8ca9808c-16a5-4e15-b5e7-72a80c07ea69"}} +{"add":{"path":"part-00000-9cbbca06-b904-4b8b-9228-e348b2ec853b-c000.snappy.parquet","partitionValues":{},"size":976,"modificationTime":1668830727644,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":120,\"date\":\"1971-07-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":667,\"date\":\"2018-02-01\"},\"nullCount\":{\"letter\":2,\"int\":0,\"date\":0}}"}} +{"remove":{"path":"part-00000-489b4323-6f66-4fac-9228-01848530b142-c000.snappy.parquet","deletionTimestamp":1668830727800,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":965}} +{"commitInfo":{"timestamp":1668830727800,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"976"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"7d72344d-a5ea-42c4-84c4-a883e544574b"}} diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000003.json b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000003.json index 0ac1572..de6d46d 100644 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000003.json +++ b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000003.json @@ -1,3 +1,3 @@ -{"add":{"path":"part-00000-0a4a95b4-9c45-41d9-a7e8-44ae2fc7d955-c000.snappy.parquet","partitionValues":{},"size":1010,"modificationTime":1668830592384,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":93,\"date\":\"1975-06-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":753,\"date\":\"2013-03-01\"},\"nullCount\":{\"letter\":1,\"int\":0,\"date\":0}}"}} -{"remove":{"path":"part-00000-c4d01b94-46fa-4b26-8002-c3870554b939-c000.snappy.parquet","deletionTimestamp":1668830592557,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":976}} -{"commitInfo":{"timestamp":1668830592557,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1010"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"72477a82-0b05-4a35-ba0c-ec5b5e2848d0"}} +{"add":{"path":"part-00000-d84f0399-6fd6-4c4f-9d82-cbe88de4cb19-c000.snappy.parquet","partitionValues":{},"size":1010,"modificationTime":1668830729094,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":93,\"date\":\"1975-06-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":753,\"date\":\"2013-03-01\"},\"nullCount\":{\"letter\":1,\"int\":0,\"date\":0}}"}} +{"remove":{"path":"part-00000-9cbbca06-b904-4b8b-9228-e348b2ec853b-c000.snappy.parquet","deletionTimestamp":1668830729287,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":976}} +{"commitInfo":{"timestamp":1668830729287,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1010"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"39546b92-5f16-4304-9eb4-7549b5d70845"}} diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000004.checkpoint.parquet b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000004.checkpoint.parquet deleted file mode 100644 index 2a1870a..0000000 Binary files a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000004.checkpoint.parquet and /dev/null differ diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000004.json b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000004.json deleted file mode 100644 index a370442..0000000 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000004.json +++ /dev/null @@ -1,3 +0,0 @@ -{"add":{"path":"part-00000-b7bc4ff0-11a0-4649-86d4-d8bbce3f48be-c000.snappy.parquet","partitionValues":{},"size":1015,"modificationTime":1668830593474,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":553,\"date\":\"1970-08-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":937,\"date\":\"2008-08-01\"},\"nullCount\":{\"letter\":1,\"int\":0,\"date\":0}}"}} -{"remove":{"path":"part-00000-0a4a95b4-9c45-41d9-a7e8-44ae2fc7d955-c000.snappy.parquet","deletionTimestamp":1668830593639,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1010}} -{"commitInfo":{"timestamp":1668830593639,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1015"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"c90b6bff-529c-4ad7-bcf4-8020cb398fe1"}} diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000005.json b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000005.json deleted file mode 100644 index 545330e..0000000 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/00000000000000000005.json +++ /dev/null @@ -1,3 +0,0 @@ -{"add":{"path":"part-00000-72e9f46b-5a92-4a57-8388-ebfaeb2de70f-c000.snappy.parquet","partitionValues":{},"size":1010,"modificationTime":1668830594744,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"letter\":\"a\",\"int\":20,\"date\":\"1976-12-01\"},\"maxValues\":{\"letter\":\"c\",\"int\":819,\"date\":\"2018-01-01\"},\"nullCount\":{\"letter\":1,\"int\":0,\"date\":0}}"}} -{"remove":{"path":"part-00000-b7bc4ff0-11a0-4649-86d4-d8bbce3f48be-c000.snappy.parquet","deletionTimestamp":1668830594960,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1015}} -{"commitInfo":{"timestamp":1668830594960,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1010"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"746b722c-447b-471d-abc2-ddd9b9f230d7"}} diff --git a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/_last_checkpoint b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/_last_checkpoint index ddb0e87..311182d 100644 --- a/out/reader_tests/generated/with_checkpoint/delta/_delta_log/_last_checkpoint +++ b/out/reader_tests/generated/with_checkpoint/delta/_delta_log/_last_checkpoint @@ -1 +1 @@ -{"version":4,"size":6,"sizeInBytes":12700,"numOfAddFiles":1,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"91a8b47d5a56d8e4a3f0c0ba1b9d38de"} +{"version":2,"size":4,"sizeInBytes":12712,"numOfAddFiles":1,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"3c1a4a7ad9b68673d4095534b71f050a"} diff --git a/out/reader_tests/generated/with_checkpoint/delta/part-00000-489b4323-6f66-4fac-9228-01848530b142-c000.snappy.parquet b/out/reader_tests/generated/with_checkpoint/delta/part-00000-489b4323-6f66-4fac-9228-01848530b142-c000.snappy.parquet new file mode 100644 index 0000000..6a69c83 Binary files /dev/null and b/out/reader_tests/generated/with_checkpoint/delta/part-00000-489b4323-6f66-4fac-9228-01848530b142-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/with_checkpoint/delta/part-00000-72e9f46b-5a92-4a57-8388-ebfaeb2de70f-c000.snappy.parquet b/out/reader_tests/generated/with_checkpoint/delta/part-00000-72e9f46b-5a92-4a57-8388-ebfaeb2de70f-c000.snappy.parquet deleted file mode 100644 index 790b2d3..0000000 Binary files a/out/reader_tests/generated/with_checkpoint/delta/part-00000-72e9f46b-5a92-4a57-8388-ebfaeb2de70f-c000.snappy.parquet and /dev/null differ diff --git a/out/reader_tests/generated/with_checkpoint/delta/part-00000-9cbbca06-b904-4b8b-9228-e348b2ec853b-c000.snappy.parquet b/out/reader_tests/generated/with_checkpoint/delta/part-00000-9cbbca06-b904-4b8b-9228-e348b2ec853b-c000.snappy.parquet new file mode 100644 index 0000000..c8a6c89 Binary files /dev/null and b/out/reader_tests/generated/with_checkpoint/delta/part-00000-9cbbca06-b904-4b8b-9228-e348b2ec853b-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/with_checkpoint/delta/part-00000-d84f0399-6fd6-4c4f-9d82-cbe88de4cb19-c000.snappy.parquet b/out/reader_tests/generated/with_checkpoint/delta/part-00000-d84f0399-6fd6-4c4f-9d82-cbe88de4cb19-c000.snappy.parquet new file mode 100644 index 0000000..0556bc1 Binary files /dev/null and b/out/reader_tests/generated/with_checkpoint/delta/part-00000-d84f0399-6fd6-4c4f-9d82-cbe88de4cb19-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/with_checkpoint/expected/latest/table_content.parquet b/out/reader_tests/generated/with_checkpoint/expected/latest/table_content.parquet index 08eb598..e79b239 100644 Binary files a/out/reader_tests/generated/with_checkpoint/expected/latest/table_content.parquet and b/out/reader_tests/generated/with_checkpoint/expected/latest/table_content.parquet differ diff --git a/out/reader_tests/generated/with_checkpoint/expected/latest/table_version_metadata.json b/out/reader_tests/generated/with_checkpoint/expected/latest/table_version_metadata.json index 6a82f97..0dd8463 100644 --- a/out/reader_tests/generated/with_checkpoint/expected/latest/table_version_metadata.json +++ b/out/reader_tests/generated/with_checkpoint/expected/latest/table_version_metadata.json @@ -1,8 +1,7 @@ { - "version": 5, + "version": 3, "properties": { - "delta.checkpointInterval": "2", - "delta.logRetentionDuration": "0 days" + "delta.checkpointInterval": "2" }, "min_reader_version": 1, "min_writer_version": 2 diff --git a/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000000.json b/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000000.json index d017505..ad3a5a3 100644 --- a/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000000.json +++ b/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000000.json @@ -1,4 +1,4 @@ {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"metaData":{"id":"9bf03b55-3ce6-4326-9550-6991f9e4286f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830582682}} -{"add":{"path":"part-00000-ff4479a7-de1f-4c66-bd44-f8bb23e4cee7-c000.snappy.parquet","partitionValues":{},"size":724,"modificationTime":1668830582884,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1},\"maxValues\":{\"letter\":\"c\",\"number\":3},\"nullCount\":{\"letter\":0,\"number\":0}}"}} -{"commitInfo":{"timestamp":1668830582899,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"724"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"550e3716-bd6d-42d3-8ecf-8703a5d4e428"}} +{"metaData":{"id":"2e70051f-6e5f-4e12-b7be-e7ff5d4f1201","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"letter\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"number\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830719662}} +{"add":{"path":"part-00000-2cc614f8-a092-4a20-8a10-b36e62e80026-c000.snappy.parquet","partitionValues":{},"size":724,"modificationTime":1668830719884,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"letter\":\"a\",\"number\":1},\"maxValues\":{\"letter\":\"c\",\"number\":3},\"nullCount\":{\"letter\":0,\"number\":0}}"}} +{"commitInfo":{"timestamp":1668830719904,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"724"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"03c98623-7e1e-421d-aba2-110e400f8655"}} diff --git a/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000001.json b/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000001.json index 1193027..a9f29e5 100644 --- a/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000001.json +++ b/out/reader_tests/generated/with_schema_change/delta/_delta_log/00000000000000000001.json @@ -1,4 +1,4 @@ -{"metaData":{"id":"9bf03b55-3ce6-4326-9550-6991f9e4286f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"num1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"num2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830582682}} -{"add":{"path":"part-00000-211e1af8-e705-4a24-b728-bb600cdd6638-c000.snappy.parquet","partitionValues":{},"size":760,"modificationTime":1668830584034,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"num1\":22,\"num2\":33},\"maxValues\":{\"num1\":66,\"num2\":77},\"nullCount\":{\"num1\":0,\"num2\":0}}"}} -{"remove":{"path":"part-00000-ff4479a7-de1f-4c66-bd44-f8bb23e4cee7-c000.snappy.parquet","deletionTimestamp":1668830584234,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":724}} -{"commitInfo":{"timestamp":1668830584234,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"760"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"e563c05b-f917-4160-9f06-0e2ab49c8a26"}} +{"metaData":{"id":"2e70051f-6e5f-4e12-b7be-e7ff5d4f1201","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"num1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"num2\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1668830719662}} +{"add":{"path":"part-00000-9515ac5f-9e8c-4e7e-8239-2ff3e13700ba-c000.snappy.parquet","partitionValues":{},"size":760,"modificationTime":1668830720934,"dataChange":true,"stats":"{\"numRecords\":3,\"minValues\":{\"num1\":22,\"num2\":33},\"maxValues\":{\"num1\":66,\"num2\":77},\"nullCount\":{\"num1\":0,\"num2\":0}}"}} +{"remove":{"path":"part-00000-2cc614f8-a092-4a20-8a10-b36e62e80026-c000.snappy.parquet","deletionTimestamp":1668830721119,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":724}} +{"commitInfo":{"timestamp":1668830721120,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"3","numOutputBytes":"760"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.1.1","txnId":"bb0ecdf1-841f-40df-ad97-d0c7ec40eba5"}} diff --git a/out/reader_tests/generated/with_schema_change/delta/part-00000-ff4479a7-de1f-4c66-bd44-f8bb23e4cee7-c000.snappy.parquet b/out/reader_tests/generated/with_schema_change/delta/part-00000-2cc614f8-a092-4a20-8a10-b36e62e80026-c000.snappy.parquet similarity index 82% rename from out/reader_tests/generated/with_schema_change/delta/part-00000-ff4479a7-de1f-4c66-bd44-f8bb23e4cee7-c000.snappy.parquet rename to out/reader_tests/generated/with_schema_change/delta/part-00000-2cc614f8-a092-4a20-8a10-b36e62e80026-c000.snappy.parquet index 022452b..1122f3f 100644 Binary files a/out/reader_tests/generated/with_schema_change/delta/part-00000-ff4479a7-de1f-4c66-bd44-f8bb23e4cee7-c000.snappy.parquet and b/out/reader_tests/generated/with_schema_change/delta/part-00000-2cc614f8-a092-4a20-8a10-b36e62e80026-c000.snappy.parquet differ diff --git a/out/reader_tests/generated/with_schema_change/delta/part-00000-211e1af8-e705-4a24-b728-bb600cdd6638-c000.snappy.parquet b/out/reader_tests/generated/with_schema_change/delta/part-00000-9515ac5f-9e8c-4e7e-8239-2ff3e13700ba-c000.snappy.parquet similarity index 83% rename from out/reader_tests/generated/with_schema_change/delta/part-00000-211e1af8-e705-4a24-b728-bb600cdd6638-c000.snappy.parquet rename to out/reader_tests/generated/with_schema_change/delta/part-00000-9515ac5f-9e8c-4e7e-8239-2ff3e13700ba-c000.snappy.parquet index cd0578e..1a7cfe4 100644 Binary files a/out/reader_tests/generated/with_schema_change/delta/part-00000-211e1af8-e705-4a24-b728-bb600cdd6638-c000.snappy.parquet and b/out/reader_tests/generated/with_schema_change/delta/part-00000-9515ac5f-9e8c-4e7e-8239-2ff3e13700ba-c000.snappy.parquet differ diff --git a/setup.cfg b/setup.cfg index f6b2ac6..9175508 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,4 +9,5 @@ per-file-ignores = # WPS202 Found too many module members tests/*: S101 WPS114 WPS226 WPS202 dat/external_tables.py: WPS226 WPS114 - dat/generated_tables.py: WPS226 WPS114 \ No newline at end of file + dat/generated_tables.py: WPS226 WPS114 +max-line-length = 90 \ No newline at end of file