Merged PR 2604: duckdb directories has a special string for directory mapping

duckdb directories has a special string for directory mapping

Related work items: #11648

Author: Patryk Jatczak

dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbConnectionProvider.java

@@ -32,9 +32,7 @@
 import tech.tablesaw.api.ColumnType;
 import tech.tablesaw.columns.Column;
 
-import java.util.Arrays;
-import java.util.List;
-import java.util.NoSuchElementException;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -139,30 +137,37 @@ public void promptForConnectionParameters(ConnectionSpec connectionSpec, boolean
             duckdbSpec.setFilesFormatType(terminalReader.promptEnum("Type of source files for DuckDB", DuckdbFilesFormatType.class, null, false));
         }
 
-        if (duckdbSpec.getDirectories().isEmpty()) {
+        if (duckdbSpec.getDirectoriesString().isEmpty()) {
             if (isHeadless) {
                 throw new CliRequiredParameterMissingException("--duckdb-directories");
             }
+            duckdbSpec.setDirectoriesString(terminalReader.prompt("Virtual schema name and path in a pattern: schema=path. For multiple use pattern: schema=path1,schema=path2", null, false));
+        }
 
-            String directoriesRaw = terminalReader.prompt("Virtual schema names and paths (in a pattern schema=path)", null, false);
-            if(!directoriesRaw.contains("/") && !directoriesRaw.contains("\\")){
-                throw new RuntimeException("The provided path is invalid. The path should be an absolute path to the directory with folders or files. On Windows use double backslash (\\\\).");
-            }
-            List<String> directories = Arrays.stream(directoriesRaw.split(",")).collect(Collectors.toList());
-
-            for (String directory : directories) {
-                List<String> schemaDirectory = Arrays.stream(directory.split("="))
-                        .map(String::trim)
-                        .collect(Collectors.toList());
-                if(schemaDirectory.size() != 2){
-                    throw new RuntimeException("Unbalanced values for " + directory + "." +
-                            "Ensure you provide directories in a schema=path pattern.");
-                }
-                duckdbSpec.getDirectories().put(schemaDirectory.get(0), schemaDirectory.get(1));
-            }
+        Map<String, String> directories = parseDirectoriesString(duckdbSpec.getDirectoriesString());
+        duckdbSpec.setDirectories(directories);
+    }
 
+    Map<String, String> parseDirectoriesString(String directoriesRaw){
+        Map<String, String> directories = new HashMap<>();
+
+        if(!directoriesRaw.contains("/") && !directoriesRaw.contains("\\")){
+            throw new RuntimeException("The provided path is invalid. The path should be an absolute path to the directory with folders or files. On Windows use double backslash (\\\\).");
+        }
+        List<String> singleMapping = Arrays.stream(directoriesRaw.split(",")).collect(Collectors.toList());
+
+        for (String mapping : singleMapping) {
+            List<String> schemaDirectory = Arrays.stream(mapping.split("="))
+                    .map(String::trim)
+                    .collect(Collectors.toList());
+            if(schemaDirectory.size() != 2){
+                throw new RuntimeException("Unbalanced values for " + mapping + "." +
+                        "Ensure you provide directories in a schema=path pattern.");
+            }
+            directories.put(schemaDirectory.get(0), schemaDirectory.get(1));
         }
 
+        return directories;
     }
 
     /**

dqops/src/main/java/com/dqops/connectors/duckdb/DuckdbParametersSpec.java

@@ -90,11 +90,14 @@ public class DuckdbParametersSpec extends BaseProviderParametersSpec
     @JsonSerialize(using = IgnoreEmptyYamlSerializer.class)
     private ParquetFileFormatSpec parquet;
 
-    @CommandLine.Option(names = "--duckdb-directories", split = ",")
     @JsonPropertyDescription("Virtual schema name to directory mappings. The path must be an absolute path.")
     @JsonInclude(JsonInclude.Include.NON_EMPTY)
     private Map<String, String> directories = new HashMap<>();
 
+    @CommandLine.Option(names = {"--duckdb-directories"}, description = "Virtual schema name to directory mappings. The path must be an absolute path.")
+    @JsonIgnore
+    private String directoriesString;
+
     @CommandLine.Option(names = {"--duckdb-storage-type"}, description = "The storage type.")
     @JsonPropertyDescription("The storage type.")
     private DuckdbStorageType storageType;
@@ -254,6 +257,23 @@ public void setDirectories(Map<String, String> directories) {
         this.directories = directories != null ? Collections.unmodifiableMap(directories) : null;
     }
 
+    /**
+     * Returns a raw directories string containing mapping between schemas and directories
+     * @return Raw directories string containing mapping between schemas and directories
+     */
+    public String getDirectoriesString() {
+        return directoriesString;
+    }
+
+    /**
+     * Sets a raw directories string containing mapping between schemas and directories
+     * @param directoriesString Raw directories string containing mapping between schemas and directories
+     */
+    public void setDirectoriesString(String directoriesString) {
+        setDirtyIf(!Objects.equals(this.directoriesString, directoriesString));
+        this.directoriesString = directoriesString;
+    }
+
     /**
      * Returns the storage type.
      * @return the storage type.

dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderObjectMother.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright © 2021 DQOps (support@dqops.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dqops.connectors.duckdb;
+
+import com.dqops.utils.BeanFactoryObjectMother;
+import org.springframework.beans.factory.BeanFactory;
+
+/**
+ * DuckdbConnectionProvider object mother.
+ */
+public class DuckdbConnectionProviderObjectMother {
+    /**
+     * Returns the DuckdbConnectionProvider.
+     * @return DuckdbConnectionProvider.
+     */
+    public static DuckdbConnectionProvider getProvider() {
+        BeanFactory beanFactory = BeanFactoryObjectMother.getBeanFactory();
+        return beanFactory.getBean(DuckdbConnectionProvider.class);
+    }
+
+}

dqops/src/test/java/com/dqops/connectors/duckdb/DuckdbConnectionProviderTest.java

@@ -0,0 +1,48 @@
+package com.dqops.connectors.duckdb;
+
+import com.dqops.BaseTest;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import java.util.Map;
+
+@SpringBootTest
+class DuckdbConnectionProviderTest extends BaseTest {
+
+    private DuckdbConnectionProvider sut;
+
+
+    @BeforeEach
+    void setUp() {
+        sut = DuckdbConnectionProviderObjectMother.getProvider();
+    }
+
+    @Test
+    void parseDirectoriesString_whenOneValidMapping_parsesIt() {
+        String inputString = "schema=/directory";
+
+        Map<String, String> mapping = sut.parseDirectoriesString(inputString);
+
+        Assertions.assertEquals(1, mapping.size());
+        Assertions.assertTrue(mapping.containsKey("schema"));
+        Assertions.assertEquals("/directory", mapping.get("schema"));
+    }
+
+    @Test
+    void parseDirectoriesString_whenMultiMapping_parsesIt() {
+        String inputString = "schema1=/directory1,schema2=/directory2";
+
+        Map<String, String> mapping = sut.parseDirectoriesString(inputString);
+
+        Assertions.assertEquals(2, mapping.size());
+
+        Assertions.assertTrue(mapping.containsKey("schema1"));
+        Assertions.assertEquals("/directory1", mapping.get("schema1"));
+
+        Assertions.assertTrue(mapping.containsKey("schema2"));
+        Assertions.assertEquals("/directory2", mapping.get("schema2"));
+    }
+
+}