Skip to content

Commit daab3d2

Browse files
Patryk Jatczakpiotrczarnas
Patryk Jatczak
authored and
piotrczarnas
committed
Merged PR 2440: simple import for file formats
- retrieve table metadata ready for csv, json, parquet - duckdb use only one query to get table metadata - frontend adjustments to keep only filepaths and file type input on table view Related work items: #10970
2 parents 15b9af3 + 2d5027c commit daab3d2

File tree

33 files changed

+694
-230
lines changed

33 files changed

+694
-230
lines changed

‎dqops/sampledata/files/json/continuous_days_one_row_per_day.json

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{"id:INTEGER":1,"date:LOCAL_DATE":"2022-02-01","value:STRING":"abc"}
2+
{"id:INTEGER":2,"date:LOCAL_DATE":"2022-02-02","value:STRING":"abc"}
3+
{"id:INTEGER":3,"date:LOCAL_DATE":"2022-02-03","value:STRING":"abc"}
4+
{"id:INTEGER":4,"date:LOCAL_DATE":"2022-02-04","value:STRING":"abc"}
5+
{"id:INTEGER":5,"date:LOCAL_DATE":"2022-02-05","value:STRING":"abc"}
6+
{"id:INTEGER":6,"date:LOCAL_DATE":"2022-02-06","value:STRING":"abc"}
7+
{"id:INTEGER":7,"date:LOCAL_DATE":"2022-02-07","value:STRING":"abc"}
8+
{"id:INTEGER":8,"date:LOCAL_DATE":"2022-02-08","value:STRING":"abc"}
9+
{"id:INTEGER":9,"date:LOCAL_DATE":"2022-02-09","value:STRING":"abc"}
10+
{"id:INTEGER":10,"date:LOCAL_DATE":"2022-02-10","value:STRING":"abc"}
11+
{"id:INTEGER":11,"date:LOCAL_DATE":"2022-02-11","value:STRING":"abc"}
12+
{"id:INTEGER":12,"date:LOCAL_DATE":"2022-02-12","value:STRING":"abc"}
13+
{"id:INTEGER":13,"date:LOCAL_DATE":"2022-02-13","value:STRING":"abc"}
14+
{"id:INTEGER":14,"date:LOCAL_DATE":"2022-02-14","value:STRING":"abc"}
15+
{"id:INTEGER":15,"date:LOCAL_DATE":"2022-02-15","value:STRING":"abc"}
16+
{"id:INTEGER":16,"date:LOCAL_DATE":"2022-02-16","value:STRING":"abc"}
17+
{"id:INTEGER":17,"date:LOCAL_DATE":"2022-02-17","value:STRING":"abc"}
18+
{"id:INTEGER":18,"date:LOCAL_DATE":"2022-02-18","value:STRING":"abc"}
19+
{"id:INTEGER":19,"date:LOCAL_DATE":"2022-02-19","value:STRING":"abc"}
20+
{"id:INTEGER":20,"date:LOCAL_DATE":"2022-02-20","value:STRING":"abc"}
21+
{"id:INTEGER":21,"date:LOCAL_DATE":"2022-02-21","value:STRING":"abc"}
22+
{"id:INTEGER":22,"date:LOCAL_DATE":"2022-02-22","value:STRING":"abc"}
23+
{"id:INTEGER":23,"date:LOCAL_DATE":"2022-02-23","value:STRING":"abc"}
24+
{"id:INTEGER":24,"date:LOCAL_DATE":"2022-02-24","value:STRING":"abc"}

‎dqops/sampledata/files/parquet/continuous_days_one_row_per_day.parquet

705 Bytes
Binary file not shown.

‎dqops/src/integration-test/java/com/dqops/connectors/duckdb/DuckdbSourceConnectionTest.java

+71-21
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,16 @@
11
package com.dqops.connectors.duckdb;
22

33
import com.dqops.BaseTest;
4-
import com.dqops.core.filesystem.virtual.FolderTreeNode;
54
import com.dqops.core.secrets.SecretValueLookupContext;
65
import com.dqops.metadata.sources.*;
76
import com.dqops.metadata.sources.fileformat.FileFormatSpec;
87
import com.dqops.metadata.sources.fileformat.FileFormatSpecObjectMother;
9-
import com.dqops.metadata.storage.localfiles.sources.FileConnectionListImpl;
10-
import com.dqops.metadata.storage.localfiles.sources.FileConnectionWrapperImpl;
118
import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext;
129
import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother;
1310
import com.dqops.metadata.userhome.UserHome;
1411
import com.dqops.sampledata.SampleCsvFileNames;
15-
import com.dqops.utils.serialization.YamlSerializer;
16-
import com.dqops.utils.serialization.YamlSerializerObjectMother;
12+
import com.dqops.sampledata.SampleJsonFileNames;
13+
import com.dqops.sampledata.SampleParquetFileNames;
1714
import org.junit.jupiter.api.Assertions;
1815
import org.junit.jupiter.api.BeforeEach;
1916
import org.junit.jupiter.api.Test;
@@ -26,45 +23,68 @@
2623
class DuckdbSourceConnectionTest extends BaseTest {
2724

2825
private DuckdbSourceConnection sut;
29-
private FileConnectionWrapperImpl fileConnectionWrapper;
26+
private ConnectionWrapperImpl connectionWrapper;
3027
private SecretValueLookupContext secretValueLookupContext;
28+
private String connectionName = "test-connection";
29+
private String tableSchemaName = "example_schema";
30+
private String tableName = "example_table";
3131

3232
@BeforeEach
3333
void setUp() {
34-
String connectionName = "test-connection";
3534
this.sut = DuckdbSourceConnectionObjectMother.getDuckdbSourceConnection();
3635
UserHomeContext userHomeContext = UserHomeContextObjectMother.createTemporaryFileHomeContext(true);
3736
UserHome userHome = userHomeContext.getUserHome();
3837
userHome.getConnections().createAndAddNew(connectionName);
3938
secretValueLookupContext = new SecretValueLookupContext(userHome);
40-
FileConnectionListImpl connections = (FileConnectionListImpl) userHome.getConnections();
41-
42-
FolderTreeNode connectionFolder = connections.getSourcesFolder().getOrAddDirectFolder(connectionName);
43-
YamlSerializer yamlSerializer = YamlSerializerObjectMother.createNew();
44-
this.fileConnectionWrapper = new FileConnectionWrapperImpl(connectionFolder, yamlSerializer);
39+
this.connectionWrapper = new ConnectionWrapperImpl();
4540
}
4641

4742
@Test
4843
void retrieveTableMetadata_fromTableSpecWithCsvFilePath_readColumnTypes() {
49-
ConnectionSpec spec = DuckdbConnectionSpecObjectMother.createForCsv();
44+
ConnectionSpec spec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
5045
this.sut.setConnectionSpec(spec);
51-
this.fileConnectionWrapper.setSpec(spec);
52-
53-
String tableSchemaName = "example_schema";
54-
String tableName = "example_table";
55-
TableWrapper tableWrapper = fileConnectionWrapper.getTables().createAndAddNew(new PhysicalTableName(tableSchemaName, tableName));
56-
46+
this.connectionWrapper.setSpec(spec);
47+
TableWrapper tableWrapper = connectionWrapper.getTables().createAndAddNew(new PhysicalTableName(tableSchemaName, tableName));
5748
FileFormatSpec fileFormatSpec = FileFormatSpecObjectMother.createForCsvFile(SampleCsvFileNames.continuous_days_one_row_per_day);
5849
tableWrapper.getSpec().setFileFormat(fileFormatSpec);
50+
List<String> tableNames = connectionWrapper
51+
.getTables().toList().stream()
52+
.map(tw -> tw.getPhysicalTableName().toString())
53+
.collect(Collectors.toList());
54+
55+
56+
this.sut.open(secretValueLookupContext);
57+
List<TableSpec> tableSpecs = sut.retrieveTableMetadata(tableSchemaName, tableNames, connectionWrapper);
58+
59+
60+
ColumnSpecMap firstTableColumns = tableSpecs.get(0).getColumns();
61+
ColumnSpec idColumn = firstTableColumns.get("id:INTEGER");
62+
Assertions.assertEquals("BIGINT", idColumn.getTypeSnapshot().getColumnType());
63+
Assertions.assertTrue(idColumn.getTypeSnapshot().getNullable());
64+
65+
ColumnSpec dateColumn = firstTableColumns.get("date:LOCAL_DATE");
66+
Assertions.assertEquals("DATE", dateColumn.getTypeSnapshot().getColumnType());
5967

60-
List<String> tableNames = fileConnectionWrapper
68+
ColumnSpec valueColumn = firstTableColumns.get("value:STRING");
69+
Assertions.assertEquals("VARCHAR", valueColumn.getTypeSnapshot().getColumnType());
70+
}
71+
72+
@Test
73+
void retrieveTableMetadata_fromTableSpecWithJsonFilePath_readColumnTypes() {
74+
ConnectionSpec spec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.json);
75+
this.sut.setConnectionSpec(spec);
76+
this.connectionWrapper.setSpec(spec);
77+
TableWrapper tableWrapper = connectionWrapper.getTables().createAndAddNew(new PhysicalTableName(tableSchemaName, tableName));
78+
FileFormatSpec fileFormatSpec = FileFormatSpecObjectMother.createForJsonFile(SampleJsonFileNames.continuous_days_one_row_per_day);
79+
tableWrapper.getSpec().setFileFormat(fileFormatSpec);
80+
List<String> tableNames = connectionWrapper
6181
.getTables().toList().stream()
6282
.map(tw -> tw.getPhysicalTableName().toString())
6383
.collect(Collectors.toList());
6484

6585

6686
this.sut.open(secretValueLookupContext);
67-
List<TableSpec> tableSpecs = sut.retrieveTableMetadata(tableSchemaName, tableNames, fileConnectionWrapper);
87+
List<TableSpec> tableSpecs = sut.retrieveTableMetadata(tableSchemaName, tableNames, connectionWrapper);
6888

6989

7090
ColumnSpecMap firstTableColumns = tableSpecs.get(0).getColumns();
@@ -77,6 +97,36 @@ void retrieveTableMetadata_fromTableSpecWithCsvFilePath_readColumnTypes() {
7797

7898
ColumnSpec valueColumn = firstTableColumns.get("value:STRING");
7999
Assertions.assertEquals("VARCHAR", valueColumn.getTypeSnapshot().getColumnType());
100+
}
101+
102+
@Test
103+
void retrieveTableMetadata_fromTableSpecWithParquetFilePath_readColumnTypes() {
104+
ConnectionSpec spec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.parquet);
105+
this.sut.setConnectionSpec(spec);
106+
this.connectionWrapper.setSpec(spec);
107+
TableWrapper tableWrapper = connectionWrapper.getTables().createAndAddNew(new PhysicalTableName(tableSchemaName, tableName));
108+
FileFormatSpec fileFormatSpec = FileFormatSpecObjectMother.createForParquetFile(SampleParquetFileNames.continuous_days_one_row_per_day);
109+
tableWrapper.getSpec().setFileFormat(fileFormatSpec);
110+
List<String> tableNames = connectionWrapper
111+
.getTables().toList().stream()
112+
.map(tw -> tw.getPhysicalTableName().toString())
113+
.collect(Collectors.toList());
114+
115+
116+
this.sut.open(secretValueLookupContext);
117+
List<TableSpec> tableSpecs = sut.retrieveTableMetadata(tableSchemaName, tableNames, connectionWrapper);
80118

119+
120+
ColumnSpecMap firstTableColumns = tableSpecs.get(0).getColumns();
121+
ColumnSpec idColumn = firstTableColumns.get("id:INTEGER");
122+
Assertions.assertEquals("BIGINT", idColumn.getTypeSnapshot().getColumnType());
123+
Assertions.assertTrue(idColumn.getTypeSnapshot().getNullable());
124+
125+
ColumnSpec dateColumn = firstTableColumns.get("date:LOCAL_DATE");
126+
Assertions.assertEquals("DATE", dateColumn.getTypeSnapshot().getColumnType());
127+
128+
ColumnSpec valueColumn = firstTableColumns.get("value:STRING");
129+
Assertions.assertEquals("VARCHAR", valueColumn.getTypeSnapshot().getColumnType());
81130
}
131+
82132
}

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/FileDuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import com.dqops.checks.column.checkspecs.acceptedvalues.ColumnTextFoundInSetPercentCheckSpec;
1919
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
20+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
2021
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
2122
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
2223
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -47,7 +48,7 @@ public class FileDuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParameters
4748

4849
@BeforeEach
4950
void setUp() {
50-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
51+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
5152
String csvFileName = SampleCsvFileNames.test_data_values_in_set;
5253
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
5354
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/accuracy/FileDuckdbColumnAccuracyTotalAverageMatchPercentSensorParametersSpecIntegrationTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import com.dqops.checks.column.checkspecs.accuracy.ColumnAccuracyTotalAverageMatchPercentCheckSpec;
1919
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
20+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
2021
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
2122
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
2223
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -45,7 +46,7 @@ public class FileDuckdbColumnAccuracyTotalAverageMatchPercentSensorParametersSpe
4546

4647
@BeforeEach
4748
void setUp() {
48-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
49+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
4950
String csvFileName = SampleCsvFileNames.ip4_test;
5051
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
5152
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datatype/FileDuckdbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.dqops.checks.CheckTimeScale;
1919
import com.dqops.checks.column.checkspecs.datatype.ColumnDatatypeStringDatatypeChangedCheckSpec;
2020
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
21+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
2122
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
2223
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
2324
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -46,7 +47,7 @@ public class FileDuckdbColumnDatatypeStringDatatypeDetectSensorParametersSpecInt
4647

4748
@BeforeEach
4849
void setUp() {
49-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
50+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
5051
String csvFileName = SampleCsvFileNames.detect_datatype_test;
5152
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
5253
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/FileDuckdbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import com.dqops.checks.column.checkspecs.blanks.ColumnBlanksNullPlaceholderTextFoundCheckSpec;
1919
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
20+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
2021
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
2122
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
2223
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -44,7 +45,7 @@ public class FileDuckdbColumnWhitespaceBlankNullPlaceholderTextCountSensorParame
4445

4546
@BeforeEach
4647
void setUp() {
47-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
48+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
4849
String csvFileName = SampleCsvFileNames.string_test_data;
4950
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
5051
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);
@@ -56,7 +57,7 @@ void setUp() {
5657
@Test
5758
void runSensor_withUseOfLocalCsvFile_thenReturnsValues() {
5859
SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck(
59-
sampleTableMetadata, "null_placeholder", this.checkSpec);
60+
sampleTableMetadata, "null_placeholder:STRING", this.checkSpec);
6061

6162
SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters);
6263

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/availability/FileDuckdbTableAvailabilitySensorParametersSpecIntegrationTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import com.dqops.checks.table.checkspecs.availability.TableAvailabilityCheckSpec;
1919
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
20+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
2021
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
2122
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
2223
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -44,7 +45,7 @@ public class FileDuckdbTableAvailabilitySensorParametersSpecIntegrationTest exte
4445

4546
@BeforeEach
4647
void setUp() {
47-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
48+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
4849
String csvFileName = SampleCsvFileNames.continuous_days_one_row_per_day;
4950
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
5051
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/timeliness/FileDuckdbTableTimelinessDataFreshnessSensorParametersSpecIntegrationTest.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import com.dqops.checks.table.checkspecs.timeliness.TableDataFreshnessCheckSpec;
1919
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
20+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
2021
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
2122
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
2223
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -47,7 +48,7 @@ public class FileDuckdbTableTimelinessDataFreshnessSensorParametersSpecIntegrati
4748

4849
@BeforeEach
4950
void setUp() {
50-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
51+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
5152
String csvFileName = SampleCsvFileNames.test_average_delay;
5253
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
5354
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);
@@ -58,18 +59,15 @@ void setUp() {
5859

5960
@Test
6061
void runSensor_withUseOfLocalCsvFile_thenReturnsValues() {
61-
// todo: in case of LOCAL_DATE_TIME that is supported in tablesaw, duckdb cant load it as a DATE or TIME.
62-
// It uses its internal sniffer to recognize type from the data.
63-
// Also, changing the type in csv header will not work due to that it is not recognized by tablesaw.
64-
this.sampleTableMetadata.getTableSpec().getTimestampColumns().setEventTimestampColumn("date1");
62+
this.sampleTableMetadata.getTableSpec().getTimestampColumns().setEventTimestampColumn("date1:LOCAL_DATE_TIME");
6563

6664
SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck(
6765
sampleTableMetadata, this.checkSpec);
6866

6967
SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters);
7068

7169
LocalDateTime ldt = LocalDateTime.now();
72-
Duration timeDiff = Duration.between(this.sampleTableMetadata.getTableData().getTable().dateTimeColumn("date1").max(),ldt);
70+
Duration timeDiff = Duration.between(this.sampleTableMetadata.getTableData().getTable().dateTimeColumn("date1:LOCAL_DATE_TIME").max(),ldt);
7371
double min = timeDiff.toMillis() / 24.0 / 3600.0 / 1000.0 - 1;
7472
double max = timeDiff.toMillis() / 24.0 / 3600.0 / 1000.0 + 1;
7573

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/volume/FileDuckdbTableVolumeRowCountSensorParametersSpecIntegrationTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.dqops.checks.table.checkspecs.volume.TableRowCountCheckSpec;
44
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
5+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
56
import com.dqops.core.filesystem.localfiles.HomeLocationFindService;
67
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
78
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
@@ -32,7 +33,7 @@ public class FileDuckdbTableVolumeRowCountSensorParametersSpecIntegrationTest ex
3233

3334
@BeforeEach
3435
void setUp() {
35-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
36+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
3637
String csvFileName = SampleCsvFileNames.continuous_days_one_row_per_day;
3738
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile(csvFileName, connectionSpec);
3839
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata);

‎dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/volume/MultipleFilesDuckdbTableVolumeRowCountSensorParametersSpecIntegrationTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.dqops.checks.table.checkspecs.volume.TableRowCountCheckSpec;
44
import com.dqops.checks.table.profiling.TableVolumeProfilingChecksSpec;
55
import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother;
6+
import com.dqops.connectors.duckdb.DuckdbSourceFilesType;
67
import com.dqops.duckdb.BaseDuckdbIntegrationTest;
78
import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother;
89
import com.dqops.execution.sensors.SensorExecutionResult;
@@ -31,7 +32,7 @@ public class MultipleFilesDuckdbTableVolumeRowCountSensorParametersSpecIntegrati
3132

3233
@BeforeEach
3334
void setUp() {
34-
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForCsv();
35+
ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbSourceFilesType.csv);
3536
String csvFilesFolder = SampleCsvFilesFolderNames.continuous_days_one_row_per_day_divided;
3637
this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitMultipleCsvFiles(csvFilesFolder, connectionSpec);
3738
this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContext();

0 commit comments

Comments
 (0)