[Bugfix] Add limit 0 to credits_used and usage_cost to fix Postgres errors

.buildkite/scripts/run_models.sh

@@ -44,6 +44,9 @@ dbt test --target "$db"
 dbt run --vars '{fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
 dbt run --vars '{fivetran_platform__usage_pricing: true}' --target "$db"
 dbt test --target "$db"
+dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true}' --target "$db" --full-refresh
+dbt run --vars '{fivetran_platform__credits_pricing: false, fivetran_platform__usage_pricing: true}' --target "$db"
+dbt test --target "$db"
 dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false}' --target "$db" --full-refresh
 dbt run --vars '{fivetran_platform__usage_pricing: false, fivetran_platform_using_destination_membership: false, fivetran_platform_using_user: false}' --target "$db"
 dbt test --target "$db"

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_fivetran_log v1.4.2
+[PR #109](https://github.com/fivetran/dbt_fivetran_log/pull/109) includes the following updates:
+
+## Bug Fixes
+- Adjusted the `stg_fivetran_platform__credits_used` and `stg_fivetran_platform__usage_cost` models to return empty tables (via a `limit 0`) if the respective `fivetran_platform__credits_pricing` and/or `fivetran_platform__usage_pricing` variables are disabled. This is to avoid Postgres data type errors if those tables are null. 
+
+## Under the Hood
+- Included an additional test case within the integration tests where the `fivetran_platform__credits_pricing` variable is set to false and the `fivetran_platform__usage_pricing` variable is set to true in order to effectively test this scenario.
+- Updated seed files to ensure downstream models properly populate into `fivetran_platform__usage_mar_destination_history`.
+
 # dbt_fivetran_log v1.4.1
 
 [PR #107](https://github.com/fivetran/dbt_fivetran_log/pull/107) includes the following updates:
@@ -10,6 +20,7 @@
 - Updated the maintainer PR template to resemble the most up to date format.
 - Included a `quickstart.yml` file to allow for automated Quickstart data model deployments.
 
+
 # dbt_fivetran_log v1.4.0
 
 ## Feature Updates

DECISIONLOG.md

@@ -3,7 +3,7 @@
 ## Usage Cost vs Credits Used Sources
 This package takes into consideration that the Fivetran pricing model has adjusted over the years. In particular, this package considers the old credit model (captured within the `credits_used` source) and the newer usage model (captured within the `usage_cost` source). By default, this package will dynamically check the mentioned sources in your destination and generate the respective staging models if the source is found. If the source is not found, the package will create a single row of null records in order to imitate the schema structure and ensure downstream transformations succeed. See the [does_table_exist()](macros/does_table_exist.sql) macro for more information on this dynamic functionality.
 
-The below variables control the respective behaviors and may be overridden if desired. If overridden and configured to `false`, the models will still be materialized, but with only a single null row. This ensures the package does not generate records from the source, but still materializes the schema to ensure there is no run failure. The intention is that these variables are not needed to be configured, but if necessary they are available.
+The below variables control the respective behaviors and may be overridden if desired. If overridden and configured to `false`, the models will still be materialized, but it will return no rows. This ensures the package does not generate records from the source, but still materializes the schema to ensure there is no run failure. The intention is that these variables are not needed to be configured, but if necessary they are available.
 
 ```yml
 vars:

README.md

@@ -81,7 +81,7 @@ vars:
 ```
 
 ## Step 4: Disable Models for Non Existent Sources
-If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` sources. To disable the corresponding functionality in the package, you must add the following variable(s) to your root `dbt_project.yml` file. By default, all variables are assumed to be `true`:
+If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` sources. It's also possible you might not have any To disable the corresponding functionality in the package, you must add the following variable(s) to your root `dbt_project.yml` file. By default, all variables are assumed to be `true`:
 
 ```yml
 vars:

integration_tests/ci/sample.profiles.yml

@@ -16,7 +16,7 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: fivetran_platform_integration_tests
+      schema: fivetran_platform_integration_tests 
       threads: 8
     bigquery:
       type: bigquery
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: fivetran_platform_integration_tests
+      schema: fivetran_platform_integration_tests 
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: fivetran_platform_integration_tests
+      schema: fivetran_platform_integration_tests 
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
@@ -58,7 +58,7 @@ integration_tests:
       server: "{{ env_var('CI_SQLSERVER_DBT_SERVER') }}"
       port: 1433
       database: "{{ env_var('CI_SQLSERVER_DBT_DATABASE') }}"
-      schema: fivetran_platform_integration_tests
+      schema: fivetran_platform_integration_tests 
       user: "{{ env_var('CI_SQLSERVER_DBT_USER') }}"
       password: "{{ env_var('CI_SQLSERVER_DBT_PASS') }}"
       threads: 8

integration_tests/seeds/connector.csv

@@ -1,3 +1,3 @@
 connector_id,_fivetran_synced,connecting_user_id,connector_name,connector_type,destination_id,paused,signed_up
-exceedingly_twenty,2021-03-12 21:03:45.994,,s3.s3_testing,s3,ups_trampoline,true,2020-08-31 19:00:22.913630
-intrinsic_departed,2021-03-12 21:03:45.991,off_ence,s3.patrick_star,s3,television_trace,true,2019-03-22 18:49:12.833910
+this_connector,2021-03-12 21:03:45.994,,this_connector,s3,ups_trampoline,true,2020-08-31 19:00:22.913630
+this_connector,2021-03-12 21:03:45.991,off_ence,this_connector,s3,television_trace,true,2019-03-22 18:49:12.833910

integration_tests/seeds/credits_used.csv

@@ -1,3 +1,3 @@
 destination_id,measured_month,_fivetran_synced,credits_consumed
-avert_defy,2020-12,2021-02-17 05:50:35.168,0
-avert_defy,2021-01,2021-02-17 05:50:35.168,0
+ups_trampoline,2022-01,2021-02-17 05:50:35.168,0
+ups_trampoline,2022-02,2021-02-17 05:50:35.168,0

integration_tests/seeds/destination.csv

@@ -1,3 +1,3 @@
 id,_fivetran_synced,account_id,created_at,name,region
 ups_trampoline,2021-03-12 20:20:40.650,1k56c2c4xlti6_acct,2019-07-09 07:21:03.935607,s_test,us-test
-television_trace,2021-03-12 20:20:40.624,1k56c2c4xlti6_acct,2019-07-10 08:07:48.374015,s_gcp,us-test
+television_trace,2021-03-12 20:20:40.624,1k56c2c4xlti6_acct,2019-07-10 08:07:48.374015,s_gcp,us-test

integration_tests/seeds/incremental_mar.csv

@@ -1,11 +1,11 @@
 connector_id,destination_id,free_type,measured_date,schema_name,sync_type,table_name,updated_at,_fivetran_synced,incremental_rows
-this_connector,111eee,PAID,2022-08-31 00:00:00,this_connector,UNKNOWN,my_table,2022-09-01 6:12:13,2022-11-09 20:09:26,46
-that_connector,222rrr,PAID,2022-05-22 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-22 8:29:22,2022-11-07 20:10:24,1
-that_connector,222rrr,PAID,2022-03-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-01 21:52:24,2022-11-07 20:10:15,8
-that_connector,222rrr,PAID,2022-03-04 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-04 16:19:05,2022-11-07 20:10:16,1
-that_connector,222rrr,PAID,2022-07-29 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-29 23:05:17,2022-11-07 20:10:33,4
-that_connector,222rrr,PAID,2022-05-14 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-14 11:07:37,2022-11-07 20:10:24,1
-that_connector,222rrr,PAID,2021-09-19 00:00:00,that_connector,UNKNOWN,drop_table,2021-09-19 12:11:38,2022-11-07 20:09:55,0
-that_connector,222rrr,PAID,2022-06-08 00:00:00,that_connector,UNKNOWN,drop_table,2022-06-08 21:29:07,2022-11-07 20:10:25,2
-that_connector,222rrr,PAID,2022-03-05 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-05 10:55:00,2022-11-07 20:10:16,1
-that_connector,222rrr,PAID,2022-07-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-01 21:56:37,2022-11-07 20:10:30,1
+this_connector,ups_trampoline,PAID,2022-01-31 00:00:00,this_connector,UNKNOWN,my_table,2022-09-01 6:12:13,2022-11-09 20:09:26,46
+this_connector,ups_trampoline,PAID,2022-02-22 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-22 8:29:22,2022-11-07 20:10:24,1
+this_connector,television_trace,PAID,2022-03-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-01 21:52:24,2022-11-07 20:10:15,8
+this_connector,television_trace,PAID,2022-03-04 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-04 16:19:05,2022-11-07 20:10:16,1
+this_connector,television_trace,PAID,2022-07-29 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-29 23:05:17,2022-11-07 20:10:33,4
+this_connector,television_trace,PAID,2022-05-14 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-14 11:07:37,2022-11-07 20:10:24,1
+this_connector,television_trace,PAID,2021-09-19 00:00:00,that_connector,UNKNOWN,drop_table,2021-09-19 12:11:38,2022-11-07 20:09:55,0
+that_connector,television_trace,PAID,2022-06-08 00:00:00,that_connector,UNKNOWN,drop_table,2022-06-08 21:29:07,2022-11-07 20:10:25,2
+that_connector,television_trace,PAID,2022-03-05 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-05 10:55:00,2022-11-07 20:10:16,1
+that_connector,television_trace,PAID,2022-07-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-01 21:56:37,2022-11-07 20:10:30,1

integration_tests/seeds/usage_cost.csv

@@ -1,5 +1,5 @@
 destination_id,measured_month,_fivetran_synced,amount
-romance_agenda,2022-01,2022-03-30T10:46:11.937Z,671.77
-romance_agenda,2022-02,2022-03-30T10:46:11.936Z,193.8
-romance_agenda,2022-03,2022-04-02T00:26:07.402Z,247.24
-romance_agenda,2022-04,2022-04-18T17:15:49.753Z,200.88
+ups_trampoline,2022-01,2022-03-30T10:46:11.937Z,671.77
+ups_trampoline,2022-02,2022-03-30T10:46:11.936Z,193.8
+television_trace,2022-03,2022-04-02T00:26:07.402Z,247.24
+television_trace,2022-04,2022-04-18T17:15:49.753Z,200.88

models/staging/stg_fivetran_platform__credits_used.sql

@@ -21,8 +21,16 @@ from fields
 {% else %}
 
 select
+    {% if target.type in ('sqlserver') %}
+    top 0
+    {% endif %}
+
     cast(null as {{ dbt.type_string() }}) as destination_id,
     cast(null as {{ dbt.type_string() }}) as measured_month,
     cast(null as {{ dbt.type_int() }}) as credits_spent
 
+    {% if target.type not in ('sqlserver') %}
+    limit 0
+    {% endif %}
+
 {% endif %}

models/staging/stg_fivetran_platform__usage_cost.sql

@@ -20,8 +20,17 @@ from fields
 {% else %}
 
 select
+
+    {% if target.type in ('sqlserver') %}
+    top 0
+    {% endif %}
+
     cast(null as {{ dbt.type_string() }}) as destination_id,
     cast(null as {{ dbt.type_string() }}) as measured_month,
     cast(null as {{ dbt.type_int() }}) as dollars_spent
 
+    {% if target.type not in ('sqlserver') %}
+    limit 0
+    {% endif %}
+
 {% endif %}