Merge pull request #22 from dqops/1.3.0

1.3.0

Author: dqops

.run/dqo run.run.xml

@@ -5,7 +5,7 @@
       <option name="region" />
       <option name="useCurrentConnection" value="false" />
     </extension>
-    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.2.0.jar" />
+    <option name="JAR_PATH" value="$PROJECT_DIR$/dqops/target/dqo-dqops-1.3.0.jar" />
     <option name="VM_PARAMETERS" value="-XX:MaxRAMPercentage=60.0 --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED" />
     <option name="PROGRAM_PARAMETERS" value="--server.port=8888" />
     <option name="WORKING_DIRECTORY" value="C:\dev\dqoado" />
@@ -14,6 +14,7 @@
       <env name="DQO_HOME" value="$PROJECT_DIR$/home" />
       <env name="DQO_USER_HOME" value="$PROJECT_DIR$/userhome" />
       <env name="HADOOP_HOME" value="$PROJECT_DIR$/home" />
+      <env name="AZURE_ENABLE_HTTP_CLIENT_SHARING" value="true" />
     </envs>
     <method v="2" />
   </configuration>

CHANGELOG.md

@@ -1,8 +1,4 @@
-# 1.2.0
-* Small bug fixes
-* Small layout changes (fonts)
-* Updates in the documentation
-* MySQL and Snowflake drivers upgraded
-* Column list shows the status of data quality dimensions
-* Corrections of queries for mysql, oracle
-* The platform tracks the data quality status of tables and columns
+# 1.3.0
+* Screens that show a list of tables and the data quality status divided by data quality dimensions
+* Analyze nested values in JSON fields
+* Analyze files on Azure storage buckets, using the az:// prefix.

Dockerfile

@@ -61,6 +61,7 @@ ENV DQO_USER_HOME=/dqo/userhome
 ENV DQO_USER_INITIALIZE_USER_HOME=true
 ENV DQO_USER_INITIALIZE_DEFAULT_CLOUD_CREDENTIALS=true
 ENV DQO_LOGGING_CONSOLE_IMMEDIATE_FLUSH=true
+ENV AZURE_ENABLE_HTTP_CLIENT_SHARING=true
 RUN mkdir $DQO_USER_HOME
 RUN touch $DQO_USER_HOME/.DQO_USER_HOME_NOT_MOUNTED
 COPY --from=dqo-home /dqo/home home

Dockerfile-fast

@@ -41,6 +41,7 @@ ENV DQO_USER_HOME=/dqo/userhome
 ENV DQO_USER_INITIALIZE_USER_HOME=true
 ENV DQO_USER_INITIALIZE_DEFAULT_CLOUD_CREDENTIALS=true
 ENV DQO_LOGGING_CONSOLE_IMMEDIATE_FLUSH=true
+ENV AZURE_ENABLE_HTTP_CLIENT_SHARING=true
 RUN mkdir $DQO_USER_HOME
 RUN touch $DQO_USER_HOME/.DQO_USER_HOME_NOT_MOUNTED
 COPY --from=dqo-home /dqo/home home

VERSION

@@ -1 +1 @@
-1.2.0
+1.3.0

distribution/pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>com.dqops</groupId>
 	<artifactId>dqo-distribution</artifactId>
-    <version>1.2.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
+    <version>1.3.0</version>  <!-- DQOps Version, do not touch (changed automatically) -->
 	<name>dqo-distribution</name>
 	<description>DQOps Data Quality Operations Center final assembly</description>
 	<packaging>pom</packaging>

distribution/python/dqops/client/models/__init__.py

@@ -24,6 +24,7 @@
 )
 from .authenticated_dashboard_model import AuthenticatedDashboardModel
 from .aws_authentication_mode import AwsAuthenticationMode
+from .azure_authentication_mode import AzureAuthenticationMode
 from .between_floats_rule_parameters_spec import BetweenFloatsRuleParametersSpec
 from .between_ints_rule_parameters_spec import BetweenIntsRuleParametersSpec
 from .between_percent_rule_parameters_spec import BetweenPercentRuleParametersSpec
@@ -1937,6 +1938,7 @@
     "AnomalyStationaryPercentileMovingAverageRuleWarning1PctParametersSpec",
     "AuthenticatedDashboardModel",
     "AwsAuthenticationMode",
+    "AzureAuthenticationMode",
     "BetweenFloatsRuleParametersSpec",
     "BetweenIntsRuleParametersSpec",
     "BetweenPercentRuleParametersSpec",

distribution/python/dqops/client/models/azure_authentication_mode.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class AzureAuthenticationMode(str, Enum):
+    CONNECTION_STRING = "connection_string"
+    CREDENTIAL_CHAIN = "credential_chain"
+    SERVICE_PRINCIPAL = "service_principal"
+
+    def __str__(self) -> str:
+        return str(self.value)

distribution/python/dqops/client/models/column_type_snapshot_spec.py

@@ -17,13 +17,16 @@ class ColumnTypeSnapshotSpec:
         length (Union[Unset, int]): Maximum length of text and binary columns.
         precision (Union[Unset, int]): Precision of a numeric (decimal) data type.
         scale (Union[Unset, int]): Scale of a numeric (decimal) data type.
+        nested (Union[Unset, bool]): This field is a nested field inside another STRUCT. It is used to identify nested
+            fields in JSON files.
     """
 
     column_type: Union[Unset, str] = UNSET
     nullable: Union[Unset, bool] = UNSET
     length: Union[Unset, int] = UNSET
     precision: Union[Unset, int] = UNSET
     scale: Union[Unset, int] = UNSET
+    nested: Union[Unset, bool] = UNSET
     additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
@@ -32,6 +35,7 @@ def to_dict(self) -> Dict[str, Any]:
         length = self.length
         precision = self.precision
         scale = self.scale
+        nested = self.nested
 
         field_dict: Dict[str, Any] = {}
         field_dict.update(self.additional_properties)
@@ -46,6 +50,8 @@ def to_dict(self) -> Dict[str, Any]:
             field_dict["precision"] = precision
         if scale is not UNSET:
             field_dict["scale"] = scale
+        if nested is not UNSET:
+            field_dict["nested"] = nested
 
         return field_dict
 
@@ -62,12 +68,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
 
         scale = d.pop("scale", UNSET)
 
+        nested = d.pop("nested", UNSET)
+
         column_type_snapshot_spec = cls(
             column_type=column_type,
             nullable=nullable,
             length=length,
             precision=precision,
             scale=scale,
+            nested=nested,
         )
 
         column_type_snapshot_spec.additional_properties = d

distribution/python/dqops/client/models/duckdb_parameters_spec.py

@@ -4,6 +4,7 @@
 from attrs import field as _attrs_field
 
 from ..models.aws_authentication_mode import AwsAuthenticationMode
+from ..models.azure_authentication_mode import AzureAuthenticationMode
 from ..models.duckdb_files_format_type import DuckdbFilesFormatType
 from ..models.duckdb_read_mode import DuckdbReadMode
 from ..models.duckdb_storage_type import DuckdbStorageType
@@ -41,12 +42,21 @@ class DuckdbParametersSpec:
             must be an absolute path.
         storage_type (Union[Unset, DuckdbStorageType]):
         aws_authentication_mode (Union[Unset, AwsAuthenticationMode]):
+        azure_authentication_mode (Union[Unset, AzureAuthenticationMode]):
         user (Union[Unset, str]): DuckDB user name for a remote storage type. The value can be in the
             ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
         password (Union[Unset, str]): DuckDB password for a remote storage type. The value can be in the
             ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
         region (Union[Unset, str]): The region for the storage credentials. The value can be in the
             ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
+        tenant_id (Union[Unset, str]): Azure Tenant ID used by DuckDB Secret Manager. The value can be in the
+            ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
+        client_id (Union[Unset, str]): Azure Client ID used by DuckDB Secret Manager. The value can be in the
+            ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
+        client_secret (Union[Unset, str]): Azure Client Secret used by DuckDB Secret Manager. The value can be in the
+            ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
+        account_name (Union[Unset, str]): Azure Storage Account Name used by DuckDB Secret Manager. The value can be in
+            the ${ENVIRONMENT_VARIABLE_NAME} format to use dynamic substitution.
     """
 
     read_mode: Union[Unset, DuckdbReadMode] = UNSET
@@ -59,9 +69,14 @@ class DuckdbParametersSpec:
     directories: Union[Unset, "DuckdbParametersSpecDirectories"] = UNSET
     storage_type: Union[Unset, DuckdbStorageType] = UNSET
     aws_authentication_mode: Union[Unset, AwsAuthenticationMode] = UNSET
+    azure_authentication_mode: Union[Unset, AzureAuthenticationMode] = UNSET
     user: Union[Unset, str] = UNSET
     password: Union[Unset, str] = UNSET
     region: Union[Unset, str] = UNSET
+    tenant_id: Union[Unset, str] = UNSET
+    client_id: Union[Unset, str] = UNSET
+    client_secret: Union[Unset, str] = UNSET
+    account_name: Union[Unset, str] = UNSET
     additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
@@ -102,9 +117,17 @@ def to_dict(self) -> Dict[str, Any]:
         if not isinstance(self.aws_authentication_mode, Unset):
             aws_authentication_mode = self.aws_authentication_mode.value
 
+        azure_authentication_mode: Union[Unset, str] = UNSET
+        if not isinstance(self.azure_authentication_mode, Unset):
+            azure_authentication_mode = self.azure_authentication_mode.value
+
         user = self.user
         password = self.password
         region = self.region
+        tenant_id = self.tenant_id
+        client_id = self.client_id
+        client_secret = self.client_secret
+        account_name = self.account_name
 
         field_dict: Dict[str, Any] = {}
         field_dict.update(self.additional_properties)
@@ -129,12 +152,22 @@ def to_dict(self) -> Dict[str, Any]:
             field_dict["storage_type"] = storage_type
         if aws_authentication_mode is not UNSET:
             field_dict["aws_authentication_mode"] = aws_authentication_mode
+        if azure_authentication_mode is not UNSET:
+            field_dict["azure_authentication_mode"] = azure_authentication_mode
         if user is not UNSET:
             field_dict["user"] = user
         if password is not UNSET:
             field_dict["password"] = password
         if region is not UNSET:
             field_dict["region"] = region
+        if tenant_id is not UNSET:
+            field_dict["tenant_id"] = tenant_id
+        if client_id is not UNSET:
+            field_dict["client_id"] = client_id
+        if client_secret is not UNSET:
+            field_dict["client_secret"] = client_secret
+        if account_name is not UNSET:
+            field_dict["account_name"] = account_name
 
         return field_dict
 
@@ -216,12 +249,29 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         else:
             aws_authentication_mode = AwsAuthenticationMode(_aws_authentication_mode)
 
+        _azure_authentication_mode = d.pop("azure_authentication_mode", UNSET)
+        azure_authentication_mode: Union[Unset, AzureAuthenticationMode]
+        if isinstance(_azure_authentication_mode, Unset):
+            azure_authentication_mode = UNSET
+        else:
+            azure_authentication_mode = AzureAuthenticationMode(
+                _azure_authentication_mode
+            )
+
         user = d.pop("user", UNSET)
 
         password = d.pop("password", UNSET)
 
         region = d.pop("region", UNSET)
 
+        tenant_id = d.pop("tenant_id", UNSET)
+
+        client_id = d.pop("client_id", UNSET)
+
+        client_secret = d.pop("client_secret", UNSET)
+
+        account_name = d.pop("account_name", UNSET)
+
         duckdb_parameters_spec = cls(
             read_mode=read_mode,
             files_format_type=files_format_type,
@@ -233,9 +283,14 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
             directories=directories,
             storage_type=storage_type,
             aws_authentication_mode=aws_authentication_mode,
+            azure_authentication_mode=azure_authentication_mode,
             user=user,
             password=password,
             region=region,
+            tenant_id=tenant_id,
+            client_id=client_id,
+            client_secret=client_secret,
+            account_name=account_name,
         )
 
         duckdb_parameters_spec.additional_properties = d

distribution/python/dqops/client/models/duckdb_storage_type.py

@@ -2,6 +2,7 @@
 
 
 class DuckdbStorageType(str, Enum):
+    AZURE = "azure"
     LOCAL = "local"
     S3 = "s3"
 

distribution/python/dqops/version.py

@@ -15,8 +15,8 @@
 # limit
 
 # WARNING: the next two lines with the version numbers (VERSION =, PIP_VERSION =) should not be modified manually. They are changed by a maven profile at compile time.
-VERSION = "1.2.0"
-PIP_VERSION = "1.2.0"
+VERSION = "1.3.0"
+PIP_VERSION = "1.3.0"
 GITHUB_RELEASE = "v" + VERSION + ""
 JAVA_VERSION = "17"