Merge branch 'develop' of https://dev.azure.com/imagetemplates/documati.io/_git/dqoado into feature/10970_DuckDb_connector_suppors_simple_import_of_metadata_by_using_scan_csv_calls_in_the_importTables_operation_in_the_the_duckdb_connector

Author: Patryk Jatczak

distribution/python/dqops/client/client.py

@@ -38,7 +38,7 @@ class Client:
     _base_url: str
     _cookies: Dict[str, str] = field(factory=dict, kw_only=True)
     _headers: Dict[str, str] = field(factory=dict, kw_only=True)
-    _timeout: Optional[httpx.Timeout] = field(default=httpx.Timeout(120.0), kw_only=True)
+    _timeout: Optional[httpx.Timeout] = field(default=None, kw_only=True)
     _verify_ssl: Union[str, bool, ssl.SSLContext] = field(default=True, kw_only=True)
     _follow_redirects: bool = field(default=False, kw_only=True)
     _httpx_args: Dict[str, Any] = field(factory=dict, kw_only=True)
@@ -168,7 +168,7 @@ class AuthenticatedClient:
     _base_url: str
     _cookies: Dict[str, str] = field(factory=dict, kw_only=True)
     _headers: Dict[str, str] = field(factory=dict, kw_only=True)
-    _timeout: Optional[httpx.Timeout] = field(default=httpx.Timeout(120.0), kw_only=True)
+    _timeout: Optional[httpx.Timeout] = field(default=None, kw_only=True)
     _verify_ssl: Union[str, bool, ssl.SSLContext] = field(default=True, kw_only=True)
     _follow_redirects: bool = field(default=False, kw_only=True)
     _httpx_args: Dict[str, Any] = field(factory=dict, kw_only=True)

distribution/python/dqops/client/models/__init__.py

@@ -1350,6 +1350,8 @@
 from .dqo_user_role import DqoUserRole
 from .duckdb_parameters_spec import DuckdbParametersSpec
 from .duckdb_parameters_spec_properties import DuckdbParametersSpecProperties
+from .duckdb_read_mode import DuckdbReadMode
+from .duckdb_source_files_type import DuckdbSourceFilesType
 from .duration import Duration
 from .effective_schedule_level_model import EffectiveScheduleLevelModel
 from .effective_schedule_model import EffectiveScheduleModel
@@ -1382,6 +1384,7 @@
 from .incident_webhook_notifications_spec import IncidentWebhookNotificationsSpec
 from .incidents_per_connection_model import IncidentsPerConnectionModel
 from .json_file_format_spec import JsonFileFormatSpec
+from .json_file_format_spec_columns import JsonFileFormatSpecColumns
 from .max_count_rule_0_error_parameters_spec import MaxCountRule0ErrorParametersSpec
 from .max_count_rule_0_warning_parameters_spec import MaxCountRule0WarningParametersSpec
 from .max_count_rule_100_parameters_spec import MaxCountRule100ParametersSpec
@@ -2437,6 +2440,8 @@
     "DqoUserRole",
     "DuckdbParametersSpec",
     "DuckdbParametersSpecProperties",
+    "DuckdbReadMode",
+    "DuckdbSourceFilesType",
     "Duration",
     "EffectiveScheduleLevelModel",
     "EffectiveScheduleModel",
@@ -2469,6 +2474,7 @@
     "IncidentStatus",
     "IncidentWebhookNotificationsSpec",
     "JsonFileFormatSpec",
+    "JsonFileFormatSpecColumns",
     "MaxCountRule0ErrorParametersSpec",
     "MaxCountRule0WarningParametersSpec",
     "MaxCountRule100ParametersSpec",

distribution/python/dqops/client/models/connection_model.py

@@ -13,6 +13,7 @@
     from ..models.delete_stored_data_queue_job_parameters import (
         DeleteStoredDataQueueJobParameters,
     )
+    from ..models.duckdb_parameters_spec import DuckdbParametersSpec
     from ..models.mysql_parameters_spec import MysqlParametersSpec
     from ..models.oracle_parameters_spec import OracleParametersSpec
     from ..models.postgresql_parameters_spec import PostgresqlParametersSpec
@@ -43,6 +44,7 @@ class ConnectionModel:
         bigquery (Union[Unset, BigQueryParametersSpec]):
         snowflake (Union[Unset, SnowflakeParametersSpec]):
         postgresql (Union[Unset, PostgresqlParametersSpec]):
+        duckdb (Union[Unset, DuckdbParametersSpec]):
         redshift (Union[Unset, RedshiftParametersSpec]):
         sqlserver (Union[Unset, SqlServerParametersSpec]):
         presto (Union[Unset, PrestoParametersSpec]):
@@ -79,6 +81,7 @@ class ConnectionModel:
     bigquery: Union[Unset, "BigQueryParametersSpec"] = UNSET
     snowflake: Union[Unset, "SnowflakeParametersSpec"] = UNSET
     postgresql: Union[Unset, "PostgresqlParametersSpec"] = UNSET
+    duckdb: Union[Unset, "DuckdbParametersSpec"] = UNSET
     redshift: Union[Unset, "RedshiftParametersSpec"] = UNSET
     sqlserver: Union[Unset, "SqlServerParametersSpec"] = UNSET
     presto: Union[Unset, "PrestoParametersSpec"] = UNSET
@@ -122,6 +125,10 @@ def to_dict(self) -> Dict[str, Any]:
         if not isinstance(self.postgresql, Unset):
             postgresql = self.postgresql.to_dict()
 
+        duckdb: Union[Unset, Dict[str, Any]] = UNSET
+        if not isinstance(self.duckdb, Unset):
+            duckdb = self.duckdb.to_dict()
+
         redshift: Union[Unset, Dict[str, Any]] = UNSET
         if not isinstance(self.redshift, Unset):
             redshift = self.redshift.to_dict()
@@ -209,6 +216,8 @@ def to_dict(self) -> Dict[str, Any]:
             field_dict["snowflake"] = snowflake
         if postgresql is not UNSET:
             field_dict["postgresql"] = postgresql
+        if duckdb is not UNSET:
+            field_dict["duckdb"] = duckdb
         if redshift is not UNSET:
             field_dict["redshift"] = redshift
         if sqlserver is not UNSET:
@@ -266,6 +275,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         from ..models.delete_stored_data_queue_job_parameters import (
             DeleteStoredDataQueueJobParameters,
         )
+        from ..models.duckdb_parameters_spec import DuckdbParametersSpec
         from ..models.mysql_parameters_spec import MysqlParametersSpec
         from ..models.oracle_parameters_spec import OracleParametersSpec
         from ..models.postgresql_parameters_spec import PostgresqlParametersSpec
@@ -314,6 +324,13 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         else:
             postgresql = PostgresqlParametersSpec.from_dict(_postgresql)
 
+        _duckdb = d.pop("duckdb", UNSET)
+        duckdb: Union[Unset, DuckdbParametersSpec]
+        if isinstance(_duckdb, Unset):
+            duckdb = UNSET
+        else:
+            duckdb = DuckdbParametersSpec.from_dict(_duckdb)
+
         _redshift = d.pop("redshift", UNSET)
         redshift: Union[Unset, RedshiftParametersSpec]
         if isinstance(_redshift, Unset):
@@ -452,6 +469,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
             bigquery=bigquery,
             snowflake=snowflake,
             postgresql=postgresql,
+            duckdb=duckdb,
             redshift=redshift,
             sqlserver=sqlserver,
             presto=presto,

distribution/python/dqops/client/models/csv_file_format_spec.py

@@ -14,25 +14,33 @@
 
 @_attrs_define
 class CsvFileFormatSpec:
-    """
+    r"""
     Attributes:
-        all_varchar (Union[Unset, bool]):
-        allow_quoted_nulls (Union[Unset, bool]):
-        auto_detect (Union[Unset, bool]):
-        columns (Union[Unset, CsvFileFormatSpecColumns]):
-        compression (Union[Unset, str]):
-        dateformat (Union[Unset, str]):
-        decimal_separator (Union[Unset, str]):
-        delim (Union[Unset, str]):
-        escape (Union[Unset, str]):
-        filename (Union[Unset, bool]):
-        header (Union[Unset, bool]):
-        hive_partitioning (Union[Unset, bool]):
-        ignore_errors (Union[Unset, bool]):
-        new_line (Union[Unset, str]):
-        quote (Union[Unset, str]):
-        skip (Union[Unset, int]):
-        timestampformat (Union[Unset, str]):
+        all_varchar (Union[Unset, bool]): Option to skip type detection for CSV parsing and assume all columns to be of
+            type VARCHAR.
+        allow_quoted_nulls (Union[Unset, bool]): Option to allow the conversion of quoted values to NULL values.
+        auto_detect (Union[Unset, bool]): Enables auto detection of CSV parameters.
+        columns (Union[Unset, CsvFileFormatSpecColumns]): A struct that specifies the column names and column types
+            contained within the CSV file (e.g., {'col1': 'INTEGER', 'col2': 'VARCHAR'}). Using this option implies that
+            auto detection is not used.
+        compression (Union[Unset, str]): The compression type for the file. By default this will be detected
+            automatically from the file extension (e.g., t.csv.gz will use gzip, t.csv will use none). Options are none,
+            gzip, zstd.
+        dateformat (Union[Unset, str]): Specifies the date format to use when parsing dates.
+        decimal_separator (Union[Unset, str]): The decimal separator of numbers.
+        delim (Union[Unset, str]): Specifies the string that separates columns within each row (line) of the file.
+        escape (Union[Unset, str]): Specifies the string that should appear before a data character sequence that
+            matches the quote value.
+        filename (Union[Unset, bool]): Whether or not an extra filename column should be included in the result.
+        header (Union[Unset, bool]): Specifies that the file contains a header line with the names of each column in the
+            file.
+        hive_partitioning (Union[Unset, bool]): Whether or not to interpret the path as a hive partitioned path.
+        ignore_errors (Union[Unset, bool]): Option to ignore any parsing errors encountered - and instead ignore rows
+            with errors.
+        new_line (Union[Unset, str]): Set the new line character(s) in the file. Options are '\r','\n', or '\r\n'.
+        quote (Union[Unset, str]): Specifies the quoting string to be used when a data value is quoted.
+        skip (Union[Unset, int]): The number of lines at the top of the file to skip.
+        timestampformat (Union[Unset, str]): Specifies the date format to use when parsing timestamps.
     """
 
     all_varchar: Union[Unset, bool] = UNSET

distribution/python/dqops/client/models/csv_file_format_spec_columns.py

@@ -8,7 +8,10 @@
 
 @_attrs_define
 class CsvFileFormatSpecColumns:
-    """ """
+    """A struct that specifies the column names and column types contained within the CSV file (e.g., {'col1': 'INTEGER',
+    'col2': 'VARCHAR'}). Using this option implies that auto detection is not used.
+
+    """
 
     additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict)
 

distribution/python/dqops/client/models/duckdb_parameters_spec.py

@@ -3,6 +3,8 @@
 from attrs import define as _attrs_define
 from attrs import field as _attrs_field
 
+from ..models.duckdb_read_mode import DuckdbReadMode
+from ..models.duckdb_source_files_type import DuckdbSourceFilesType
 from ..types import UNSET, Unset
 
 if TYPE_CHECKING:
@@ -18,9 +20,8 @@
 class DuckdbParametersSpec:
     """
     Attributes:
-        in_memory (Union[Unset, bool]): To use the special value :memory: to create an in-memory database where no data
-            is persisted to disk (i.e., all data is lost when you exit the process). The value can be in the
-            ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
+        read_mode (Union[Unset, DuckdbReadMode]):
+        source_files_type (Union[Unset, DuckdbSourceFilesType]):
         database (Union[Unset, str]): DuckDB database name. The value can be in the ${ENVIRONMENT_VARIABLE_NAME} format
             to use dynamic substitution.
         options (Union[Unset, str]): DuckDB connection 'options' initialization parameter. For example setting this to
@@ -30,14 +31,22 @@ class DuckdbParametersSpec:
             to the JDBC connection string, a key/value dictionary.
     """
 
-    in_memory: Union[Unset, bool] = UNSET
+    read_mode: Union[Unset, DuckdbReadMode] = UNSET
+    source_files_type: Union[Unset, DuckdbSourceFilesType] = UNSET
     database: Union[Unset, str] = UNSET
     options: Union[Unset, str] = UNSET
     properties: Union[Unset, "DuckdbParametersSpecProperties"] = UNSET
     additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
-        in_memory = self.in_memory
+        read_mode: Union[Unset, str] = UNSET
+        if not isinstance(self.read_mode, Unset):
+            read_mode = self.read_mode.value
+
+        source_files_type: Union[Unset, str] = UNSET
+        if not isinstance(self.source_files_type, Unset):
+            source_files_type = self.source_files_type.value
+
         database = self.database
         options = self.options
         properties: Union[Unset, Dict[str, Any]] = UNSET
@@ -47,8 +56,10 @@ def to_dict(self) -> Dict[str, Any]:
         field_dict: Dict[str, Any] = {}
         field_dict.update(self.additional_properties)
         field_dict.update({})
-        if in_memory is not UNSET:
-            field_dict["in_memory"] = in_memory
+        if read_mode is not UNSET:
+            field_dict["read_mode"] = read_mode
+        if source_files_type is not UNSET:
+            field_dict["source_files_type"] = source_files_type
         if database is not UNSET:
             field_dict["database"] = database
         if options is not UNSET:
@@ -65,7 +76,19 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         )
 
         d = src_dict.copy()
-        in_memory = d.pop("in_memory", UNSET)
+        _read_mode = d.pop("read_mode", UNSET)
+        read_mode: Union[Unset, DuckdbReadMode]
+        if isinstance(_read_mode, Unset):
+            read_mode = UNSET
+        else:
+            read_mode = DuckdbReadMode(_read_mode)
+
+        _source_files_type = d.pop("source_files_type", UNSET)
+        source_files_type: Union[Unset, DuckdbSourceFilesType]
+        if isinstance(_source_files_type, Unset):
+            source_files_type = UNSET
+        else:
+            source_files_type = DuckdbSourceFilesType(_source_files_type)
 
         database = d.pop("database", UNSET)
 
@@ -79,7 +102,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
             properties = DuckdbParametersSpecProperties.from_dict(_properties)
 
         duckdb_parameters_spec = cls(
-            in_memory=in_memory,
+            read_mode=read_mode,
+            source_files_type=source_files_type,
             database=database,
             options=options,
             properties=properties,

distribution/python/dqops/client/models/duckdb_read_mode.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class DuckdbReadMode(str, Enum):
+    FILES = "files"
+    IN_MEMORY = "in_memory"
+
+    def __str__(self) -> str:
+        return str(self.value)

distribution/python/dqops/client/models/duckdb_source_files_type.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class DuckdbSourceFilesType(str, Enum):
+    CSV = "csv"
+    JSON = "json"
+    PARQUET = "parquet"
+
+    def __str__(self) -> str:
+        return str(self.value)

distribution/python/dqops/client/models/file_format_spec.py

@@ -21,14 +21,12 @@ class FileFormatSpec:
         csv_file_format (Union[Unset, CsvFileFormatSpec]):
         json_file_format (Union[Unset, JsonFileFormatSpec]):
         parquet_file_format (Union[Unset, ParquetFileFormatSpec]):
-        file_path_list (Union[Unset, List[str]]): The list of paths to files with data that are used as a source.
-        file_paths (Union[Unset, List[str]]):
+        file_paths (Union[Unset, List[str]]): The list of paths to files with data that are used as a source.
     """
 
     csv_file_format: Union[Unset, "CsvFileFormatSpec"] = UNSET
     json_file_format: Union[Unset, "JsonFileFormatSpec"] = UNSET
     parquet_file_format: Union[Unset, "ParquetFileFormatSpec"] = UNSET
-    file_path_list: Union[Unset, List[str]] = UNSET
     file_paths: Union[Unset, List[str]] = UNSET
     additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)
 
@@ -45,10 +43,6 @@ def to_dict(self) -> Dict[str, Any]:
         if not isinstance(self.parquet_file_format, Unset):
             parquet_file_format = self.parquet_file_format.to_dict()
 
-        file_path_list: Union[Unset, List[str]] = UNSET
-        if not isinstance(self.file_path_list, Unset):
-            file_path_list = self.file_path_list
-
         file_paths: Union[Unset, List[str]] = UNSET
         if not isinstance(self.file_paths, Unset):
             file_paths = self.file_paths
@@ -62,8 +56,6 @@ def to_dict(self) -> Dict[str, Any]:
             field_dict["json_file_format"] = json_file_format
         if parquet_file_format is not UNSET:
             field_dict["parquet_file_format"] = parquet_file_format
-        if file_path_list is not UNSET:
-            field_dict["file_path_list"] = file_path_list
         if file_paths is not UNSET:
             field_dict["file_paths"] = file_paths
 
@@ -97,15 +89,12 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         else:
             parquet_file_format = ParquetFileFormatSpec.from_dict(_parquet_file_format)
 
-        file_path_list = cast(List[str], d.pop("file_path_list", UNSET))
-
         file_paths = cast(List[str], d.pop("file_paths", UNSET))
 
         file_format_spec = cls(
             csv_file_format=csv_file_format,
             json_file_format=json_file_format,
             parquet_file_format=parquet_file_format,
-            file_path_list=file_path_list,
             file_paths=file_paths,
         )