diff --git a/dqops/src/main/frontend/src/utils/index.ts b/dqops/src/main/frontend/src/utils/index.ts
index c20f4aa3b8..cd58e028d0 100644
--- a/dqops/src/main/frontend/src/utils/index.ts
+++ b/dqops/src/main/frontend/src/utils/index.ts
@@ -957,3 +957,37 @@ export const getProviderTypeTitle = (
return '';
}
};
+
+export const getRouteValidLabel = (route: string) => {
+ if (route.includes('/columns/all')) {
+ return 'Columns';
+ }
+ if (route.endsWith('checks/advanced-profiling')) {
+ return 'Profiling checks';
+ }
+ if (route.endsWith('monitoring/daily')) {
+ return 'Daily monitoring';
+ }
+ if (route.endsWith('monitoring/monthly')) {
+ return 'Monthly monitoring';
+ }
+ if (route.endsWith('partitioned/daily')) {
+ return 'Daily period checks';
+ }
+ if (route.endsWith('partitioned/monthly')) {
+ return 'Monthly period checks';
+ }
+ return route.split('/')[route.split('/').length - 1];
+};
+
+export const isValidRouteWithoutTab = (route: string) => {
+ return (
+ route.endsWith('/columns/all') ||
+ route.endsWith('monitoring/daily') ||
+ route.endsWith('monitoring/monthly') ||
+ route.endsWith('partitioned/daily') ||
+ route.endsWith('partitioned/monthly') ||
+ route.endsWith('checks/advanced-profiling') ||
+ route.endsWith('/:category/:checkName')
+ );
+};
diff --git a/dqops/src/main/java/com/dqops/execution/statistics/TableMetadataConfigServiceImpl.java b/dqops/src/main/java/com/dqops/execution/statistics/TableMetadataConfigServiceImpl.java
index d8e6c60f9b..3e83283500 100644
--- a/dqops/src/main/java/com/dqops/execution/statistics/TableMetadataConfigServiceImpl.java
+++ b/dqops/src/main/java/com/dqops/execution/statistics/TableMetadataConfigServiceImpl.java
@@ -117,6 +117,9 @@ public void configureTableMetadata(UserDomainIdentity userDomainIdentity,
for (StatisticsResultsForColumnModel columnStatistics : statisticsResultsForTableModel.getColumns().values()) {
StatisticsMetricModel maxValueMetric = columnStatistics.getMetricByCollectorName(ColumnRangeMaxValueStatisticsCollectorSpec.COLLECTOR_NAME);
+ if (maxValueMetric == null) {
+ continue;
+ }
Instant maxDate = DateTypesConverter.toInstant(maxValueMetric.getResult(), defaultTimeZoneId);
if (maxDate == null) {
diff --git a/dqops/src/main/resources/banner.txt b/dqops/src/main/resources/banner.txt
index dbabe5a331..24707e65d0 100644
--- a/dqops/src/main/resources/banner.txt
+++ b/dqops/src/main/resources/banner.txt
@@ -3,4 +3,4 @@
| |) | | (_) | | (_) | | '_ \ (_-<
|___/ \__\_\ \___/ | .__/ /__/
|_|
- :: DQOps Data Quality Operations Center :: (v1.10.0)
+ :: DQOps Data Quality Operations Center :: (v1.10.1)
diff --git a/examples/data-consistency/detect-data-type-changes/sources/HousePricePredictionTreatedDataset/files.below_above_value_test.csv.dqotable.yaml b/examples/data-consistency/detect-data-type-changes/sources/HousePricePredictionTreatedDataset/files.below_above_value_test.csv.dqotable.yaml
deleted file mode 100644
index 86cdcc285b..0000000000
--- a/examples/data-consistency/detect-data-type-changes/sources/HousePricePredictionTreatedDataset/files.below_above_value_test.csv.dqotable.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json
-apiVersion: dqo/v1
-kind: table
-spec:
- incremental_time_window:
- daily_partitioning_recent_days: 7
- monthly_partitioning_recent_months: 1
- columns:
- value:INTEGER:
- type_snapshot:
- column_type: BIGINT
- nullable: true
- result:INTEGER:
- type_snapshot:
- column_type: BIGINT
- nullable: true
- date_text:STRING:
- type_snapshot:
- column_type: DATE
- nullable: true
- date:LOCAL_DATE:
- type_snapshot:
- column_type: DATE
- nullable: true
- file_format:
- file_paths:
- - ..\..\..\dqops\sampledata\below_above_value_test.csv
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/.DQO_USER_HOME b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/.DQO_USER_HOME
new file mode 100644
index 0000000000..1fd4028b56
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/.DQO_USER_HOME
@@ -0,0 +1 @@
+This is a marker file to identify a DQO_USER_HOME folder. Please check this file to Git.
\ No newline at end of file
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/.gitignore b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/.gitignore
new file mode 100644
index 0000000000..76b5d986b2
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/.gitignore
@@ -0,0 +1,8 @@
+.credentials/
+.data/
+.index/
+.logs/
+bin/
+jars/
+.venv/
+.localsettings.dqosettings.yaml
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the count of distinct values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the count of distinct values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..be2c60e214
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the count of distinct values.dqocolumnpattern.yaml
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the count of distinct values in a column and raises an issue
+ when an anomaly is detected.
+ monitoring_checks:
+ daily:
+ uniqueness:
+ daily_distinct_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ partitioned_checks:
+ daily:
+ uniqueness:
+ daily_partition_distinct_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the percentage of null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the percentage of null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..55899426c7
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the percentage of null values.dqocolumnpattern.yaml
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the scale of null values in columns and raises an issue when
+ the day-to-day change is significant.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_percent_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_percent_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the sum and average of numeric values at a partition level.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the sum and average of numeric values at a partition level.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..0df2b47127
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the sum and average of numeric values at a partition level.dqocolumnpattern.yaml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the sum and average (mean) aggregated values of numeric values
+ and raises a data quality issue when the value changes too much between daily
+ partitions.
+ partitioned_checks:
+ daily:
+ anomaly:
+ daily_partition_sum_anomaly:
+ warning:
+ anomaly_percent: 0.05
+ daily_partition_mean_anomaly:
+ warning:
+ anomaly_percent: 0.05
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the sum and average of numeric values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the sum and average of numeric values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..97e890bb0c
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect anomalies in the sum and average of numeric values.dqocolumnpattern.yaml
@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the sum and average (mean) aggregated values of numeric values
+ and raises a data quality issue when the value changes too much day-to-day.
+ monitoring_checks:
+ daily:
+ anomaly:
+ daily_sum_anomaly:
+ warning:
+ anomaly_percent: 0.05
+ daily_mean_anomaly:
+ warning:
+ anomaly_percent: 0.05
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect big day-to-day data volume changes.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect big day-to-day data volume changes.dqotablepattern.yaml
new file mode 100644
index 0000000000..bf324dc1fe
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect big day-to-day data volume changes.dqotablepattern.yaml
@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors data volume of the whole table daily and raises an issue when
+ the volume has increased of decreased significantly.
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count_change:
+ warning:
+ max_percent: 10.0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect change of the data type of values in text columns.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect change of the data type of values in text columns.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..0934d92d64
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect change of the data type of values in text columns.dqocolumnpattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Detects when the values stored in a text column change their type.
+ This policy should be activated on raw tables in the landing zones for table that
+ store all values (also numeric an dates) in text columns.
+ monitoring_checks:
+ daily:
+ datatype:
+ daily_detected_datatype_in_text_changed:
+ warning: {}
+ partitioned_checks:
+ daily:
+ datatype:
+ daily_partition_detected_datatype_in_text_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect column schema changes.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect column schema changes.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..2e1780fdf0
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect column schema changes.dqocolumnpattern.yaml
@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: "Monitors the schema of columns registered in DQOps. Raises a data\
+ \ quality issue when the column is missing, or its data has changed."
+ monitoring_checks:
+ daily:
+ schema:
+ daily_column_exists:
+ warning: {}
+ daily_column_type_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect columns containing any null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect columns containing any null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..292745aa4d
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect columns containing any null values.dqocolumnpattern.yaml
@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Detects columns containing any null values using both monitoring checks
+ and daily partitioned checks.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ warning:
+ max_count: 0
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_count:
+ warning:
+ max_count: 0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect data freshness anomalies daily.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect data freshness anomalies daily.dqotablepattern.yaml
new file mode 100644
index 0000000000..40c8e6b742
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect data freshness anomalies daily.dqotablepattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors data freshness anomalies daily.
+ monitoring_checks:
+ daily:
+ timeliness:
+ daily_data_freshness_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect data volume anomalies.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect data volume anomalies.dqotablepattern.yaml
new file mode 100644
index 0000000000..753b42908d
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect data volume anomalies.dqotablepattern.yaml
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: "Monitors data volume of the whole table (using daily monitoring checks)\
+ \ and for each daily partition, using daily partition checks."
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ partitioned_checks:
+ daily:
+ volume:
+ daily_partition_row_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect empty columns.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect empty columns.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..0d5befdc90
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect empty columns.dqocolumnpattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Detects empty columns using both monitoring checks an daily partitioned
+ checks.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_empty_column_found:
+ warning: {}
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_empty_column_found:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect empty tables daily.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect empty tables daily.dqotablepattern.yaml
new file mode 100644
index 0000000000..0ccb67129c
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect empty tables daily.dqotablepattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Detects empty tables using daily monitoring checks.
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect outliers in numeric values across daily partitions.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect outliers in numeric values across daily partitions.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..98b7ab78b2
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect outliers in numeric values across daily partitions.dqocolumnpattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Monitors numeric columns to detect new smallest (min) or biggest (max)
+ value for each daily partition. Raises a data quality issue when the partition
+ contains a big or small value that exceeds regular ranges.
+ partitioned_checks:
+ daily:
+ anomaly:
+ daily_partition_min_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ daily_partition_max_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect outliers in numeric values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect outliers in numeric values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..c779860fa7
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect outliers in numeric values.dqocolumnpattern.yaml
@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: "Monitors numeric columns to detect new smallest (min) or biggest (max)\
+ \ value, which must be an anomaly."
+ monitoring_checks:
+ daily:
+ anomaly:
+ daily_min_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ daily_max_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect significant changes in the percentage of null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect significant changes in the percentage of null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..e9c96256fc
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect significant changes in the percentage of null values.dqocolumnpattern.yaml
@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Monitors the percentage of null values in columns and raises an issue
+ when the day-to-day change is above a threshold.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_percent_change:
+ warning:
+ max_percent: 10.0
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_percent_change:
+ warning:
+ max_percent: 10.0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect table availability issues daily.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect table availability issues daily.dqotablepattern.yaml
new file mode 100644
index 0000000000..5f1209bf31
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect table availability issues daily.dqotablepattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors table availability issues daily.
+ monitoring_checks:
+ daily:
+ availability:
+ daily_table_availability:
+ warning:
+ max_failures: 0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect table schema changes.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect table schema changes.dqotablepattern.yaml
new file mode 100644
index 0000000000..9e87b21ed8
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Detect table schema changes.dqotablepattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors the table schema and raises issues when the schema of the
+ table was changed.
+ monitoring_checks:
+ daily:
+ schema:
+ daily_column_count_changed:
+ warning: {}
+ daily_column_list_changed:
+ warning: {}
+ daily_column_list_or_order_changed:
+ warning: {}
+ daily_column_types_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Profile text columns to detect PII values (sensitive data).dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Profile text columns to detect PII values (sensitive data).dqocolumnpattern.yaml
new file mode 100644
index 0000000000..c6467d938c
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Profile text columns to detect PII values (sensitive data).dqocolumnpattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: "Activates data profiling checks on all text columns to detect if they\
+ \ contain sensitive data (emails, phone numbers). Enabling this policy allows\
+ \ the data quality rule miner to set up PII checks when sensitive values are identified."
+ profiling_checks:
+ pii:
+ profile_contains_usa_phone_percent: {}
+ profile_contains_email_percent: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Track the count and percentage of null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Track the count and percentage of null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..46a8084088
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Track the count and percentage of null values.dqocolumnpattern.yaml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 2000
+ description: Monitors the count and the percentage of null values without raising
+ data quality issues.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count: {}
+ daily_nulls_percent: {}
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_count: {}
+ daily_partition_nulls_percent: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Track volume of daily partitions.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Track volume of daily partitions.dqotablepattern.yaml
new file mode 100644
index 0000000000..2978eafcf3
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/patterns/Track volume of daily partitions.dqotablepattern.yaml
@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors volume (row count) of daily partitions.
+ partitioned_checks:
+ daily:
+ volume:
+ daily_partition_row_count: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/rules/requirements.txt b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/rules/requirements.txt
new file mode 100644
index 0000000000..6440576ccc
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/rules/requirements.txt
@@ -0,0 +1 @@
+# packages in this file are installed when DQOps starts
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/run_dqo.cmd b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/run_dqo.cmd
new file mode 100644
index 0000000000..92fd4f335a
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/run_dqo.cmd
@@ -0,0 +1,2 @@
+@echo off
+..\..\..\..\dqo.cmd
\ No newline at end of file
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/run_dqo.sh b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/run_dqo.sh
new file mode 100644
index 0000000000..7b9e2c19d3
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/run_dqo.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+../../../../dqo
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/dashboardslist.dqodashboards.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/dashboardslist.dqodashboards.yaml
new file mode 100644
index 0000000000..20264e44c9
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/dashboardslist.dqodashboards.yaml
@@ -0,0 +1,9 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DashboardYaml-schema.json
+apiVersion: dqo/v1
+kind: dashboards
+spec:
+- folder_name: Profiling
+- folder_name: Monitoring
+- folder_name: Partitions
+- folder_name: DQOps usage
+- folder_name: Aggregated results for all check types
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/default.dqodefaultchecks.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/default.dqodefaultchecks.yaml
new file mode 100644
index 0000000000..a453fc4fe5
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/default.dqodefaultchecks.yaml
@@ -0,0 +1,81 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultObservabilityChecksYaml-schema.json
+apiVersion: dqo/v1
+kind: default_checks
+spec:
+ profiling:
+ table:
+ volume:
+ profile_row_count:
+ warning:
+ min_count: 1
+ schema:
+ profile_column_count: {}
+ column:
+ nulls:
+ profile_nulls_count:
+ warning:
+ max_count: 0
+ profile_nulls_percent: {}
+ profile_not_nulls_count:
+ warning:
+ min_count: 1
+ monitoring_daily:
+ table:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 1
+ daily_row_count_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ daily_row_count_change:
+ warning:
+ max_percent: 10.0
+ availability:
+ daily_table_availability:
+ warning:
+ max_failures: 0
+ schema:
+ daily_column_count: {}
+ daily_column_count_changed:
+ warning: {}
+ daily_column_list_changed:
+ warning: {}
+ daily_column_list_or_order_changed:
+ warning: {}
+ daily_column_types_changed:
+ warning: {}
+ column:
+ nulls:
+ daily_nulls_count: {}
+ daily_nulls_percent: {}
+ daily_nulls_percent_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ daily_not_nulls_count:
+ warning:
+ min_count: 1
+ daily_not_nulls_percent: {}
+ daily_nulls_percent_change_1_day:
+ warning:
+ max_percent: 10.0
+ exact_day: true
+ uniqueness:
+ daily_distinct_count_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ anomaly:
+ daily_sum_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ daily_mean_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ datatype:
+ daily_detected_datatype_in_text_changed:
+ warning: {}
+ schema:
+ daily_column_exists:
+ warning: {}
+ daily_column_type_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultchecks.dqochecks.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultchecks.dqochecks.yaml
new file mode 100644
index 0000000000..896c152ca7
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultchecks.dqochecks.yaml
@@ -0,0 +1,69 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultObservabilityChecksYaml-schema.json
+apiVersion: dqo/v1
+kind: default_checks
+spec:
+ profiling:
+ table:
+ volume:
+ profile_row_count:
+ warning:
+ min_count: 1
+ schema:
+ profile_column_count: {}
+ column:
+ nulls:
+ profile_nulls_count: {}
+ profile_nulls_percent: {}
+ monitoring_daily:
+ table:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 1
+ daily_row_count_change:
+ warning:
+ max_percent: 10.0
+ daily_row_count_anomaly_differencing:
+ warning:
+ anomaly_percent: 1.0
+ availability:
+ daily_table_availability:
+ warning:
+ max_failures: 0
+ schema:
+ daily_column_count_changed:
+ warning: {}
+ daily_column_list_changed:
+ warning: {}
+ daily_column_list_or_order_changed:
+ warning: {}
+ daily_column_types_changed:
+ warning: {}
+ column:
+ nulls:
+ daily_nulls_count: {}
+ daily_nulls_percent: {}
+ daily_nulls_percent_anomaly_stationary:
+ warning:
+ anomaly_percent: 1.0
+ daily_nulls_percent_change_yesterday:
+ warning:
+ max_percent: 10.0
+ exact_day: false
+ daily_not_nulls_percent: {}
+ datatype:
+ daily_string_datatype_changed:
+ warning: {}
+ anomaly:
+ daily_mean_anomaly_stationary:
+ warning:
+ anomaly_percent: 1.0
+ daily_sum_anomaly_differencing:
+ warning:
+ anomaly_percent: 1.0
+ schema:
+ daily_column_exists:
+ warning:
+ expected_value: 1
+ daily_column_type_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultnotifications.dqonotifications.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultnotifications.dqonotifications.yaml
new file mode 100644
index 0000000000..ffa2ac0ffa
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultnotifications.dqonotifications.yaml
@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultNotificationsYaml-schema.json
+apiVersion: dqo/v1
+kind: default_notifications
+spec: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultschedules.dqoschedules.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultschedules.dqoschedules.yaml
new file mode 100644
index 0000000000..d30ad1762e
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultschedules.dqoschedules.yaml
@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultSchedulesYaml-schema.json
+apiVersion: dqo/v1
+kind: default_schedules
+spec:
+ profiling:
+ cron_expression: 0 1 1 * *
+ monitoring_daily:
+ cron_expression: 0 12 * * *
+ monitoring_monthly:
+ cron_expression: 0 12 * * *
+ partitioned_daily:
+ cron_expression: 0 12 * * *
+ partitioned_monthly:
+ cron_expression: 0 12 * * *
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultwebhooks.dqowebhooks.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultwebhooks.dqowebhooks.yaml
new file mode 100644
index 0000000000..48b53febb1
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/settings/defaultwebhooks.dqowebhooks.yaml
@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultIncidentWebhookNotificationsYaml-schema.json
+apiVersion: dqo/v1
+kind: notification_webhooks
+spec: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/sources/thelook-ecomerce/connection.dqoconnection.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/sources/thelook-ecomerce/connection.dqoconnection.yaml
new file mode 100644
index 0000000000..0c2e14a03f
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/sources/thelook-ecomerce/connection.dqoconnection.yaml
@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ConnectionYaml-schema.json
+apiVersion: dqo/v1
+kind: source
+spec:
+ provider_type: postgresql
+ postgresql:
+ host: localhost
+ port: 5432
+ database: postgres_dev
+ user: postgres
+ password: Gola1984
+ sslmode: disable
+ postgresql_engine_type: postgresql
+ schedules:
+ profiling:
+ cron_expression: 0 1 1 * *
+ monitoring_daily:
+ cron_expression: 0 12 * * *
+ monitoring_monthly:
+ cron_expression: 0 12 * * *
+ partitioned_daily:
+ cron_expression: 0 12 * * *
+ partitioned_monthly:
+ cron_expression: 0 12 * * *
+ incident_grouping:
+ grouping_level: table_dimension_category
+ minimum_severity: warning
+ max_incident_length_days: 60
+ mute_for_days: 60
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/sources/thelook-ecomerce/public.thelook-ecomerce-orders.dqotable.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/sources/thelook-ecomerce/public.thelook-ecomerce-orders.dqotable.yaml
new file mode 100644
index 0000000000..21c8257422
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/development-user-home/sources/thelook-ecomerce/public.thelook-ecomerce-orders.dqotable.yaml
@@ -0,0 +1,70 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json
+apiVersion: dqo/v1
+kind: table
+spec:
+ incremental_time_window:
+ daily_partitioning_recent_days: 7
+ monthly_partitioning_recent_months: 1
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 100
+ columns:
+ order_id:
+ type_snapshot:
+ column_type: INTEGER
+ nullable: true
+ precision: 0
+ user_id:
+ type_snapshot:
+ column_type: INTEGER
+ nullable: true
+ precision: 0
+ status:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ gender:
+ type_snapshot:
+ column_type: '"CHAR"'
+ nullable: true
+ created_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ error:
+ max_count: 0
+ returned_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ error:
+ max_count: 800
+ shipped_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ delivered_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ num_of_item:
+ type_snapshot:
+ column_type: INTEGER
+ nullable: true
+ precision: 0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/.DQO_USER_HOME b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/.DQO_USER_HOME
new file mode 100644
index 0000000000..1fd4028b56
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/.DQO_USER_HOME
@@ -0,0 +1 @@
+This is a marker file to identify a DQO_USER_HOME folder. Please check this file to Git.
\ No newline at end of file
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/.gitignore b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/.gitignore
new file mode 100644
index 0000000000..76b5d986b2
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/.gitignore
@@ -0,0 +1,8 @@
+.credentials/
+.data/
+.index/
+.logs/
+bin/
+jars/
+.venv/
+.localsettings.dqosettings.yaml
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the count of distinct values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the count of distinct values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..be2c60e214
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the count of distinct values.dqocolumnpattern.yaml
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the count of distinct values in a column and raises an issue
+ when an anomaly is detected.
+ monitoring_checks:
+ daily:
+ uniqueness:
+ daily_distinct_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ partitioned_checks:
+ daily:
+ uniqueness:
+ daily_partition_distinct_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the percentage of null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the percentage of null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..55899426c7
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the percentage of null values.dqocolumnpattern.yaml
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the scale of null values in columns and raises an issue when
+ the day-to-day change is significant.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_percent_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_percent_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the sum and average of numeric values at a partition level.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the sum and average of numeric values at a partition level.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..0df2b47127
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the sum and average of numeric values at a partition level.dqocolumnpattern.yaml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the sum and average (mean) aggregated values of numeric values
+ and raises a data quality issue when the value changes too much between daily
+ partitions.
+ partitioned_checks:
+ daily:
+ anomaly:
+ daily_partition_sum_anomaly:
+ warning:
+ anomaly_percent: 0.05
+ daily_partition_mean_anomaly:
+ warning:
+ anomaly_percent: 0.05
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the sum and average of numeric values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the sum and average of numeric values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..97e890bb0c
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect anomalies in the sum and average of numeric values.dqocolumnpattern.yaml
@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Monitors the sum and average (mean) aggregated values of numeric values
+ and raises a data quality issue when the value changes too much day-to-day.
+ monitoring_checks:
+ daily:
+ anomaly:
+ daily_sum_anomaly:
+ warning:
+ anomaly_percent: 0.05
+ daily_mean_anomaly:
+ warning:
+ anomaly_percent: 0.05
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect big day-to-day data volume changes.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect big day-to-day data volume changes.dqotablepattern.yaml
new file mode 100644
index 0000000000..bf324dc1fe
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect big day-to-day data volume changes.dqotablepattern.yaml
@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors data volume of the whole table daily and raises an issue when
+ the volume has increased of decreased significantly.
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count_change:
+ warning:
+ max_percent: 10.0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect change of the data type of values in text columns.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect change of the data type of values in text columns.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..0934d92d64
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect change of the data type of values in text columns.dqocolumnpattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: Detects when the values stored in a text column change their type.
+ This policy should be activated on raw tables in the landing zones for table that
+ store all values (also numeric an dates) in text columns.
+ monitoring_checks:
+ daily:
+ datatype:
+ daily_detected_datatype_in_text_changed:
+ warning: {}
+ partitioned_checks:
+ daily:
+ datatype:
+ daily_partition_detected_datatype_in_text_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect column schema changes.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect column schema changes.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..2e1780fdf0
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect column schema changes.dqocolumnpattern.yaml
@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: "Monitors the schema of columns registered in DQOps. Raises a data\
+ \ quality issue when the column is missing, or its data has changed."
+ monitoring_checks:
+ daily:
+ schema:
+ daily_column_exists:
+ warning: {}
+ daily_column_type_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect columns containing any null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect columns containing any null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..292745aa4d
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect columns containing any null values.dqocolumnpattern.yaml
@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Detects columns containing any null values using both monitoring checks
+ and daily partitioned checks.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ warning:
+ max_count: 0
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_count:
+ warning:
+ max_count: 0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect data freshness anomalies daily.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect data freshness anomalies daily.dqotablepattern.yaml
new file mode 100644
index 0000000000..40c8e6b742
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect data freshness anomalies daily.dqotablepattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors data freshness anomalies daily.
+ monitoring_checks:
+ daily:
+ timeliness:
+ daily_data_freshness_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect data volume anomalies.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect data volume anomalies.dqotablepattern.yaml
new file mode 100644
index 0000000000..753b42908d
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect data volume anomalies.dqotablepattern.yaml
@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: "Monitors data volume of the whole table (using daily monitoring checks)\
+ \ and for each daily partition, using daily partition checks."
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ partitioned_checks:
+ daily:
+ volume:
+ daily_partition_row_count_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect empty columns.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect empty columns.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..0d5befdc90
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect empty columns.dqocolumnpattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Detects empty columns using both monitoring checks an daily partitioned
+ checks.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_empty_column_found:
+ warning: {}
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_empty_column_found:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect empty tables daily.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect empty tables daily.dqotablepattern.yaml
new file mode 100644
index 0000000000..0ccb67129c
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect empty tables daily.dqotablepattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Detects empty tables using daily monitoring checks.
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect outliers in numeric values across daily partitions.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect outliers in numeric values across daily partitions.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..98b7ab78b2
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect outliers in numeric values across daily partitions.dqocolumnpattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Monitors numeric columns to detect new smallest (min) or biggest (max)
+ value for each daily partition. Raises a data quality issue when the partition
+ contains a big or small value that exceeds regular ranges.
+ partitioned_checks:
+ daily:
+ anomaly:
+ daily_partition_min_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ daily_partition_max_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect outliers in numeric values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect outliers in numeric values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..c779860fa7
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect outliers in numeric values.dqocolumnpattern.yaml
@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: "Monitors numeric columns to detect new smallest (min) or biggest (max)\
+ \ value, which must be an anomaly."
+ monitoring_checks:
+ daily:
+ anomaly:
+ daily_min_anomaly:
+ warning:
+ anomaly_percent: 0.1
+ daily_max_anomaly:
+ warning:
+ anomaly_percent: 0.1
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect significant changes in the percentage of null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect significant changes in the percentage of null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..e9c96256fc
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect significant changes in the percentage of null values.dqocolumnpattern.yaml
@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ disabled: true
+ description: Monitors the percentage of null values in columns and raises an issue
+ when the day-to-day change is above a threshold.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_percent_change:
+ warning:
+ max_percent: 10.0
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_percent_change:
+ warning:
+ max_percent: 10.0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect table availability issues daily.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect table availability issues daily.dqotablepattern.yaml
new file mode 100644
index 0000000000..5f1209bf31
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect table availability issues daily.dqotablepattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors table availability issues daily.
+ monitoring_checks:
+ daily:
+ availability:
+ daily_table_availability:
+ warning:
+ max_failures: 0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect table schema changes.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect table schema changes.dqotablepattern.yaml
new file mode 100644
index 0000000000..9e87b21ed8
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Detect table schema changes.dqotablepattern.yaml
@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors the table schema and raises issues when the schema of the
+ table was changed.
+ monitoring_checks:
+ daily:
+ schema:
+ daily_column_count_changed:
+ warning: {}
+ daily_column_list_changed:
+ warning: {}
+ daily_column_list_or_order_changed:
+ warning: {}
+ daily_column_types_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Profile text columns to detect PII values (sensitive data).dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Profile text columns to detect PII values (sensitive data).dqocolumnpattern.yaml
new file mode 100644
index 0000000000..c6467d938c
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Profile text columns to detect PII values (sensitive data).dqocolumnpattern.yaml
@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 1000
+ description: "Activates data profiling checks on all text columns to detect if they\
+ \ contain sensitive data (emails, phone numbers). Enabling this policy allows\
+ \ the data quality rule miner to set up PII checks when sensitive values are identified."
+ profiling_checks:
+ pii:
+ profile_contains_usa_phone_percent: {}
+ profile_contains_email_percent: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Track the count and percentage of null values.dqocolumnpattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Track the count and percentage of null values.dqocolumnpattern.yaml
new file mode 100644
index 0000000000..46a8084088
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Track the count and percentage of null values.dqocolumnpattern.yaml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ColumnLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_column_checks
+spec:
+ priority: 2000
+ description: Monitors the count and the percentage of null values without raising
+ data quality issues.
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count: {}
+ daily_nulls_percent: {}
+ partitioned_checks:
+ daily:
+ nulls:
+ daily_partition_nulls_count: {}
+ daily_partition_nulls_percent: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Track volume of daily partitions.dqotablepattern.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Track volume of daily partitions.dqotablepattern.yaml
new file mode 100644
index 0000000000..2978eafcf3
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/patterns/Track volume of daily partitions.dqotablepattern.yaml
@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableLevelDataQualityPolicyYaml-schema.json
+apiVersion: dqo/v1
+kind: default_table_checks
+spec:
+ priority: 1000
+ description: Monitors volume (row count) of daily partitions.
+ partitioned_checks:
+ daily:
+ volume:
+ daily_partition_row_count: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/rules/requirements.txt b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/rules/requirements.txt
new file mode 100644
index 0000000000..6440576ccc
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/rules/requirements.txt
@@ -0,0 +1 @@
+# packages in this file are installed when DQOps starts
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/run_dqo.cmd b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/run_dqo.cmd
new file mode 100644
index 0000000000..92fd4f335a
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/run_dqo.cmd
@@ -0,0 +1,2 @@
+@echo off
+..\..\..\..\dqo.cmd
\ No newline at end of file
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/run_dqo.sh b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/run_dqo.sh
new file mode 100644
index 0000000000..7b9e2c19d3
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/run_dqo.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+../../../../dqo
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/dashboardslist.dqodashboards.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/dashboardslist.dqodashboards.yaml
new file mode 100644
index 0000000000..20264e44c9
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/dashboardslist.dqodashboards.yaml
@@ -0,0 +1,9 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DashboardYaml-schema.json
+apiVersion: dqo/v1
+kind: dashboards
+spec:
+- folder_name: Profiling
+- folder_name: Monitoring
+- folder_name: Partitions
+- folder_name: DQOps usage
+- folder_name: Aggregated results for all check types
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/default.dqodefaultchecks.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/default.dqodefaultchecks.yaml
new file mode 100644
index 0000000000..a453fc4fe5
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/default.dqodefaultchecks.yaml
@@ -0,0 +1,81 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultObservabilityChecksYaml-schema.json
+apiVersion: dqo/v1
+kind: default_checks
+spec:
+ profiling:
+ table:
+ volume:
+ profile_row_count:
+ warning:
+ min_count: 1
+ schema:
+ profile_column_count: {}
+ column:
+ nulls:
+ profile_nulls_count:
+ warning:
+ max_count: 0
+ profile_nulls_percent: {}
+ profile_not_nulls_count:
+ warning:
+ min_count: 1
+ monitoring_daily:
+ table:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 1
+ daily_row_count_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ daily_row_count_change:
+ warning:
+ max_percent: 10.0
+ availability:
+ daily_table_availability:
+ warning:
+ max_failures: 0
+ schema:
+ daily_column_count: {}
+ daily_column_count_changed:
+ warning: {}
+ daily_column_list_changed:
+ warning: {}
+ daily_column_list_or_order_changed:
+ warning: {}
+ daily_column_types_changed:
+ warning: {}
+ column:
+ nulls:
+ daily_nulls_count: {}
+ daily_nulls_percent: {}
+ daily_nulls_percent_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ daily_not_nulls_count:
+ warning:
+ min_count: 1
+ daily_not_nulls_percent: {}
+ daily_nulls_percent_change_1_day:
+ warning:
+ max_percent: 10.0
+ exact_day: true
+ uniqueness:
+ daily_distinct_count_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ anomaly:
+ daily_sum_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ daily_mean_anomaly:
+ warning:
+ anomaly_percent: 1.0
+ datatype:
+ daily_detected_datatype_in_text_changed:
+ warning: {}
+ schema:
+ daily_column_exists:
+ warning: {}
+ daily_column_type_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultchecks.dqochecks.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultchecks.dqochecks.yaml
new file mode 100644
index 0000000000..896c152ca7
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultchecks.dqochecks.yaml
@@ -0,0 +1,69 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultObservabilityChecksYaml-schema.json
+apiVersion: dqo/v1
+kind: default_checks
+spec:
+ profiling:
+ table:
+ volume:
+ profile_row_count:
+ warning:
+ min_count: 1
+ schema:
+ profile_column_count: {}
+ column:
+ nulls:
+ profile_nulls_count: {}
+ profile_nulls_percent: {}
+ monitoring_daily:
+ table:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 1
+ daily_row_count_change:
+ warning:
+ max_percent: 10.0
+ daily_row_count_anomaly_differencing:
+ warning:
+ anomaly_percent: 1.0
+ availability:
+ daily_table_availability:
+ warning:
+ max_failures: 0
+ schema:
+ daily_column_count_changed:
+ warning: {}
+ daily_column_list_changed:
+ warning: {}
+ daily_column_list_or_order_changed:
+ warning: {}
+ daily_column_types_changed:
+ warning: {}
+ column:
+ nulls:
+ daily_nulls_count: {}
+ daily_nulls_percent: {}
+ daily_nulls_percent_anomaly_stationary:
+ warning:
+ anomaly_percent: 1.0
+ daily_nulls_percent_change_yesterday:
+ warning:
+ max_percent: 10.0
+ exact_day: false
+ daily_not_nulls_percent: {}
+ datatype:
+ daily_string_datatype_changed:
+ warning: {}
+ anomaly:
+ daily_mean_anomaly_stationary:
+ warning:
+ anomaly_percent: 1.0
+ daily_sum_anomaly_differencing:
+ warning:
+ anomaly_percent: 1.0
+ schema:
+ daily_column_exists:
+ warning:
+ expected_value: 1
+ daily_column_type_changed:
+ warning: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultnotifications.dqonotifications.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultnotifications.dqonotifications.yaml
new file mode 100644
index 0000000000..ffa2ac0ffa
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultnotifications.dqonotifications.yaml
@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultNotificationsYaml-schema.json
+apiVersion: dqo/v1
+kind: default_notifications
+spec: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultschedules.dqoschedules.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultschedules.dqoschedules.yaml
new file mode 100644
index 0000000000..d30ad1762e
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultschedules.dqoschedules.yaml
@@ -0,0 +1,14 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultSchedulesYaml-schema.json
+apiVersion: dqo/v1
+kind: default_schedules
+spec:
+ profiling:
+ cron_expression: 0 1 1 * *
+ monitoring_daily:
+ cron_expression: 0 12 * * *
+ monitoring_monthly:
+ cron_expression: 0 12 * * *
+ partitioned_daily:
+ cron_expression: 0 12 * * *
+ partitioned_monthly:
+ cron_expression: 0 12 * * *
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultwebhooks.dqowebhooks.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultwebhooks.dqowebhooks.yaml
new file mode 100644
index 0000000000..48b53febb1
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/settings/defaultwebhooks.dqowebhooks.yaml
@@ -0,0 +1,4 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/DefaultIncidentWebhookNotificationsYaml-schema.json
+apiVersion: dqo/v1
+kind: notification_webhooks
+spec: {}
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/house_price_prediction/connection.dqoconnection.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/house_price_prediction/connection.dqoconnection.yaml
new file mode 100644
index 0000000000..691a6f9104
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/house_price_prediction/connection.dqoconnection.yaml
@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ConnectionYaml-schema.json
+apiVersion: dqo/v1
+kind: source
+spec:
+ provider_type: postgresql
+ postgresql:
+ host: localhost
+ port: 5432
+ database: postgres_prod
+ user: postgres
+ password: Gola1984
+ sslmode: disable
+ postgresql_engine_type: postgresql
+ schedules:
+ profiling:
+ cron_expression: 0 1 1 * *
+ monitoring_daily:
+ cron_expression: 0 12 * * *
+ monitoring_monthly:
+ cron_expression: 0 12 * * *
+ partitioned_daily:
+ cron_expression: 0 12 * * *
+ partitioned_monthly:
+ cron_expression: 0 12 * * *
+ incident_grouping:
+ grouping_level: table_dimension_category
+ minimum_severity: warning
+ max_incident_length_days: 60
+ mute_for_days: 60
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/house_price_prediction/public.house_price_prediction_treated_dataset.dqotable.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/house_price_prediction/public.house_price_prediction_treated_dataset.dqotable.yaml
new file mode 100644
index 0000000000..f88fe1d4f8
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/house_price_prediction/public.house_price_prediction_treated_dataset.dqotable.yaml
@@ -0,0 +1,78 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json
+apiVersion: dqo/v1
+kind: table
+spec:
+ incremental_time_window:
+ daily_partitioning_recent_days: 7
+ monthly_partitioning_recent_months: 1
+ columns:
+ date:
+ type_snapshot:
+ column_type: DATE
+ nullable: true
+ precision: 0
+ price:
+ type_snapshot:
+ column_type: DOUBLE PRECISION
+ nullable: true
+ precision: 53
+ bedrooms:
+ type_snapshot:
+ column_type: BIGINT
+ nullable: true
+ precision: 0
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ error:
+ max_count: 0
+ grade:
+ type_snapshot:
+ column_type: BIGINT
+ nullable: true
+ precision: 0
+ has_basement:
+ type_snapshot:
+ column_type: BOOLEAN
+ nullable: true
+ living_in_m2:
+ type_snapshot:
+ column_type: DOUBLE PRECISION
+ nullable: true
+ precision: 53
+ renovated:
+ type_snapshot:
+ column_type: BOOLEAN
+ nullable: true
+ nice_view:
+ type_snapshot:
+ column_type: BOOLEAN
+ nullable: true
+ perfect_condition:
+ type_snapshot:
+ column_type: BOOLEAN
+ nullable: true
+ real_bathrooms:
+ type_snapshot:
+ column_type: BIGINT
+ nullable: true
+ precision: 0
+ has_lavatory:
+ type_snapshot:
+ column_type: BOOLEAN
+ nullable: true
+ single_floor:
+ type_snapshot:
+ column_type: BOOLEAN
+ nullable: true
+ month:
+ type_snapshot:
+ column_type: BIGINT
+ nullable: true
+ precision: 0
+ quartile_zone:
+ type_snapshot:
+ column_type: BIGINT
+ nullable: true
+ precision: 0
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/thelook-ecomerce/connection.dqoconnection.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/thelook-ecomerce/connection.dqoconnection.yaml
new file mode 100644
index 0000000000..0c2e14a03f
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/thelook-ecomerce/connection.dqoconnection.yaml
@@ -0,0 +1,29 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/ConnectionYaml-schema.json
+apiVersion: dqo/v1
+kind: source
+spec:
+ provider_type: postgresql
+ postgresql:
+ host: localhost
+ port: 5432
+ database: postgres_dev
+ user: postgres
+ password: Gola1984
+ sslmode: disable
+ postgresql_engine_type: postgresql
+ schedules:
+ profiling:
+ cron_expression: 0 1 1 * *
+ monitoring_daily:
+ cron_expression: 0 12 * * *
+ monitoring_monthly:
+ cron_expression: 0 12 * * *
+ partitioned_daily:
+ cron_expression: 0 12 * * *
+ partitioned_monthly:
+ cron_expression: 0 12 * * *
+ incident_grouping:
+ grouping_level: table_dimension_category
+ minimum_severity: warning
+ max_incident_length_days: 60
+ mute_for_days: 60
diff --git a/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/thelook-ecomerce/public.thelook-ecomerce-orders.dqotable.yaml b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/thelook-ecomerce/public.thelook-ecomerce-orders.dqotable.yaml
new file mode 100644
index 0000000000..21c8257422
--- /dev/null
+++ b/examples/data-quality-monitoring/moving-checks-between-environments/production-user-home/sources/thelook-ecomerce/public.thelook-ecomerce-orders.dqotable.yaml
@@ -0,0 +1,70 @@
+# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json
+apiVersion: dqo/v1
+kind: table
+spec:
+ incremental_time_window:
+ daily_partitioning_recent_days: 7
+ monthly_partitioning_recent_months: 1
+ monitoring_checks:
+ daily:
+ volume:
+ daily_row_count:
+ warning:
+ min_count: 100
+ columns:
+ order_id:
+ type_snapshot:
+ column_type: INTEGER
+ nullable: true
+ precision: 0
+ user_id:
+ type_snapshot:
+ column_type: INTEGER
+ nullable: true
+ precision: 0
+ status:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ gender:
+ type_snapshot:
+ column_type: '"CHAR"'
+ nullable: true
+ created_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ error:
+ max_count: 0
+ returned_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ monitoring_checks:
+ daily:
+ nulls:
+ daily_nulls_count:
+ error:
+ max_count: 800
+ shipped_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ delivered_at:
+ type_snapshot:
+ column_type: TEXT
+ nullable: true
+ length: 1073741824
+ num_of_item:
+ type_snapshot:
+ column_type: INTEGER
+ nullable: true
+ precision: 0
diff --git a/lib/pom.xml b/lib/pom.xml
index 9ec6694c89..05061ca1e0 100644
--- a/lib/pom.xml
+++ b/lib/pom.xml
@@ -11,7 +11,7 @@
com.dqops
dqo-lib
-
1.10.0
+
1.10.1
lib
POM for a list of dependencies to libraries that should be distributed in the "lib" folder, especially all JDBC drivers.
jar
diff --git a/mkdocs.yml b/mkdocs.yml
index a8a30cab5b..764dc10e17 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -138,6 +138,7 @@ nav:
- Data quality monitoring:
- Detect empty tables: "examples/data-quality-monitoring/detect-empty-tables.md"
- Running checks with a scheduler: "examples/data-quality-monitoring/running-checks-with-a-scheduler.md"
+ - Moving configured checks between environments: "examples/data-quality-monitoring/moving-configured-checks-between-environments.md"
- Working with DQOps:
- "working-with-dqo/index.md"
- Daily monitoring of data quality: "working-with-dqo/daily-monitoring-of-data-quality.md"
diff --git a/pom.xml b/pom.xml
index 11b6d4a01c..c4682cfa3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
com.dqops
dqo-data-quality-observer
-
1.10.0
+
1.10.1
pom
DQOps Data Quality Operations Center