Analyze nested columns in JSON files.

Author: piotrczarnas

dqops/src/main/frontend/src/pages/TableColumnsView/TableColumns.tsx

@@ -181,7 +181,7 @@ const TableColumns = ({
         columnHash: Number(hashData?.[i]),
         isColumnSelected: false,
         dimentions: rewriteDimensions(status)[columnNameData?.[i] ?? ''],
-        labels: getLabelsOverview(columns[i].labels ?? [])
+        labels: getLabelsOverview(columns[i]?.labels ?? [])
       };
 
       dataArray.push(newData);

dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbSourceConnection.java

@@ -19,6 +19,8 @@
 import com.dqops.connectors.SourceTableModel;
 import com.dqops.connectors.duckdb.fileslisting.AwsTablesLister;
 import com.dqops.connectors.duckdb.fileslisting.LocalSystemTablesLister;
+import com.dqops.connectors.duckdb.schema.DuckDBDataTypeParser;
+import com.dqops.connectors.duckdb.schema.DuckDBField;
 import com.dqops.connectors.jdbc.AbstractJdbcSourceConnection;
 import com.dqops.connectors.jdbc.JdbcConnectionPool;
 import com.dqops.connectors.jdbc.JdbcQueryFailedException;
@@ -66,6 +68,7 @@ public class DuckdbSourceConnection extends AbstractJdbcSourceConnection {
     private static final Object registerExtensionsLock = new Object();
     private static boolean extensionsRegistered = false;
     private final DqoDuckdbConfiguration dqoDuckdbConfiguration;
+    private final DuckDBDataTypeParser dataTypeParser;
     private static final Object settingsExecutionLock = new Object();
     private static final boolean settingsConfigured = false;
     private static final String temporaryDirectoryPrefix = "dqops_duckdb_temp_";
@@ -77,16 +80,19 @@ public class DuckdbSourceConnection extends AbstractJdbcSourceConnection {
      * @param secretValueProvider     Secret value provider for the environment variable expansion.
      * @param homeLocationFindService Home location find service.
      * @param dqoDuckdbConfiguration  Configuration settings for duckdb.
+     * @param dataTypeParser          Data type parser that parses the schema of structures.
      */
     @Autowired
     public DuckdbSourceConnection(JdbcConnectionPool jdbcConnectionPool,
                                   SecretValueProvider secretValueProvider,
                                   DuckdbConnectionProvider duckdbConnectionProvider,
                                   HomeLocationFindService homeLocationFindService,
-                                  DqoDuckdbConfiguration dqoDuckdbConfiguration) {
+                                  DqoDuckdbConfiguration dqoDuckdbConfiguration,
+                                  DuckDBDataTypeParser dataTypeParser) {
         super(jdbcConnectionPool, secretValueProvider, duckdbConnectionProvider);
         this.homeLocationFindService = homeLocationFindService;
         this.dqoDuckdbConfiguration = dqoDuckdbConfiguration;
+        this.dataTypeParser = dataTypeParser;
     }
 
     /**
@@ -412,7 +418,16 @@ public List<TableSpec> retrieveTableMetadata(String schemaName,
                     String dataType = colRow.getString("column_type");
                     boolean isNullable = Objects.equals(colRow.getString("null"), "YES");
                     ColumnSpec columnSpec = prepareNewColumnSpec(dataType, isNullable);
+                    if (dataType != null && (dataType.startsWith("STRUCT") || dataType.startsWith("UNION") || dataType.startsWith("MAP"))) {
+                        columnSpec.setTypeSnapshot(new ColumnTypeSnapshotSpec(dataType, isNullable));
+                    }
                     tableSpec.getColumns().put(columnName, columnSpec);
+
+                    DuckDBField parsedField = this.dataTypeParser.parseFieldType(dataType, columnName);
+                    if (parsedField.isStruct()) {
+                        String parentColumnPrefix = parsedField.isArray() ? columnName + "[0]" : columnName;
+                        addNestedFieldsFromStructs(parentColumnPrefix, parsedField, tableSpec.getColumns());
+                    }
                 }
             } catch (Exception e){
                 if (!e.getMessage().contains("SQL query failed: java.sql.SQLException: IO Error: No files found that match the pattern")){
@@ -424,6 +439,28 @@ public List<TableSpec> retrieveTableMetadata(String schemaName,
         return tableSpecs;
     }
 
+    /**
+     * Traverses a structure of nested fields inside STRUCT data types and adds all nested fields.
+     * @param parentColumnName Parent column name used as a prefix to access nested fields.
+     * @param structField DuckDB field schema that as parsed - must be a STRUCT field.
+     * @param targetColumnsMap Target column map to add generated columns.
+     */
+    private void addNestedFieldsFromStructs(String parentColumnName, DuckDBField structField, ColumnSpecMap targetColumnsMap) {
+        if (structField.isStruct() && structField.getNestedFields() != null) {
+            for (DuckDBField childField : structField.getNestedFields()) {
+                ColumnSpec columnSpec = prepareNewColumnSpec(childField.getTypeName(), childField.isNullable());
+                columnSpec.getTypeSnapshot().setNested(true);
+                String nestedFieldName = parentColumnName + "." + childField.getFieldName();
+                targetColumnsMap.put(nestedFieldName, columnSpec);
+
+                if (childField.isStruct()) {
+                    String childColumnPrefix = childField.isArray() ? nestedFieldName + "[0]" : nestedFieldName;
+                    addNestedFieldsFromStructs(childColumnPrefix, childField, targetColumnsMap);
+                }
+            }
+        }
+    }
+
     /**
      * Creates a new column spec.
      * @param dataType A data type of the column.

dqops/src/main/java/com/dqops/connectors/duckdb/schema/DuckDBDataTypeParser.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright © 2021 DQOps (support@dqops.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.dqops.connectors.duckdb.schema;
+
+/**
+ * Parser that is able to parse data types from DuckDB.
+ */
+public interface DuckDBDataTypeParser {
+    /**
+     * Parses a text in the <code>dataType</code> into an object that describes the field. Parses also structures and arrays.
+     * @param dataType Data type to parse.
+     * @param fieldName Field name to store in the result object (because the root data type has no field name, only nested fields contain a name).
+     * @return Data type, also for nested structures.
+     */
+    DuckDBField parseFieldType(String dataType, String fieldName);
+}