Merge branch 'main' into dbeatty/snapshots_new_record_mode

dbt-labs · Feb 6, 2025 · 7acbfc8 · 7acbfc8
2 parents 0fd3114 + 9e6516a
commit 7acbfc8
Show file tree

Hide file tree

Showing 27 changed files with 960 additions and 510 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
diff --git a/.changes/unreleased/Features-20241217-181340.yaml b/.changes/unreleased/Features-20241217-181340.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Add IdpTokenAuthPlugin authentication method.
+time: 2024-12-17T18:13:40.281494-08:00
+custom:
+    Author: versusfacit
+    Issue: "898"
diff --git a/.changes/unreleased/Under the Hood-20241204-185729.yaml b/.changes/unreleased/Under the Hood-20241204-185729.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add retry logic for retryable exceptions
+time: 2024-12-04T18:57:29.925299-08:00
+custom:
+    Author: versusfacit colin-rogers-dbt
+    Issue: "960"
diff --git a/.changes/unreleased/Under the Hood-20241207-165918.yaml b/.changes/unreleased/Under the Hood-20241207-165918.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move from setup.py to pyproject.toml and to hatch as a dev tool
+time: 2024-12-07T16:59:18.731819-05:00
+custom:
+  Author: mikealfare
+  Issue: "951"
diff --git a/.changes/unreleased/Under the Hood-20241211-145132.yaml b/.changes/unreleased/Under the Hood-20241211-145132.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Refactor to use new batch context varaibles
+time: 2024-12-11T14:51:32.239224-06:00
+custom:
+  Author: QMalcolm
+  Issue: "966"
diff --git a/.github/scripts/update_dependencies.sh b/.github/scripts/update_dependencies.sh
@@ -2,10 +2,10 @@
 set -e
 
 git_branch=$1
-target_req_file="dev-requirements.txt"
-core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
-postgres_req_sed_pattern="s|dbt-core.git.*#egg=dbt-postgres|dbt-core.git@${git_branch}#egg=dbt-postgres|g"
-tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
+target_req_file="hatch.toml"
+core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${git_branch}#subdirectory=core|g"
+postgres_req_sed_pattern="s|dbt-postgres.git|dbt-postgres.git@${git_branch}"
+tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${git_branch}#subdirectory=dbt-tests-adapter|g"
 if [[ "$OSTYPE" == darwin* ]]; then
  # mac ships with a different version of sed that requires a delimiter arg
  sed -i "" "$core_req_sed_pattern" $target_req_file

diff --git a/.github/scripts/update_dev_dependency_branches.sh b/.github/scripts/update_dev_dependency_branches.sh
@@ -5,8 +5,8 @@ set -e
 dbt_adapters_branch=$1
 dbt_core_branch=$2
 dbt_common_branch=$3
-target_req_file="dev-requirements.txt"
-core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
+target_req_file="hatch.toml"
+core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g"
 adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
 common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
 if [[ "$OSTYPE" == darwin* ]]; then

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -100,8 +100,6 @@ jobs:
             python-version: "3.9"
 
     env:
-      TOXENV: integration-redshift
-      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
       DBT_INVOCATION_ENV: github-actions
       DD_CIVISIBILITY_AGENTLESS_ENABLED: true
       DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
@@ -144,14 +142,9 @@ jobs:
             ${{ inputs.dbt_adapters_branch }} \
             ${{ inputs.dbt_core_branch }} \
             ${{ inputs.dbt_common_branch }}
-          cat dev-requirements.txt
+          cat hatch.toml
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install tox
-          python -m pip --version
-          tox --version
+      - uses: pypa/hatch@install
 
       - name: Create AWS IAM profiles
         run: |
@@ -180,7 +173,7 @@ jobs:
           AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }}
           AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
 
-      - name: Run tox (redshift)
+      - run: hatch run integration-tests tests/functional -m "not flaky"
         env:
           REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
           REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
@@ -196,25 +189,6 @@ jobs:
           DBT_TEST_USER_1: dbt_test_user_1
           DBT_TEST_USER_2: dbt_test_user_2
           DBT_TEST_USER_3: dbt_test_user_3
-        run: tox -- -m "not flaky" --ddtrace
-
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: logs
-          path: ./logs
-
-      - name: Get current date
-        if: always()
-        id: date
-        run: |
-          echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
-
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv
-          path: integration_results.csv
 
   test-flaky:
     name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky
@@ -236,8 +210,6 @@ jobs:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     env:
-      TOXENV: integration-redshift
-      PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv"
       DBT_INVOCATION_ENV: github-actions
       DD_CIVISIBILITY_AGENTLESS_ENABLED: true
       DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
@@ -274,12 +246,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install tox
-          python -m pip --version
-          tox --version
+      - uses: pypa/hatch@install
 
       - name: Update Adapters and Core branches (update dev_requirements.txt)
         if: github.event_name == 'workflow_dispatch'
@@ -288,8 +255,7 @@ jobs:
             ${{ inputs.dbt_adapters_branch }} \
             ${{ inputs.dbt_core_branch }} \
             ${{ inputs.dbt_common_branch }}
-          cat dev-requirements.txt
-
+          cat hatch.toml
 
       - name: Run tox (redshift)
         env:
@@ -307,7 +273,7 @@ jobs:
           DBT_TEST_USER_1: dbt_test_user_1
           DBT_TEST_USER_2: dbt_test_user_2
           DBT_TEST_USER_3: dbt_test_user_3
-        run: tox -- -m flaky --ddtrace
+        run: hatch run integration-tests tests/functional -m flaky -n1 --ddtrace
 
   require-label-comment:
     runs-on: ubuntu-22.04

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -48,19 +48,12 @@ jobs:
           persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install -r dev-requirements.txt
-          python -m pip --version
-          pre-commit --version
-
       - name: pre-commit hooks
-        run: pre-commit run --all-files --show-diff-on-failure
+        uses: pre-commit/action@v3.0.1
 
   unit:
     name: unit test / python ${{ matrix.python-version }}
@@ -72,10 +65,6 @@ jobs:
       matrix:
         python-version: ['3.9', '3.10', '3.11', '3.12']
 
-    env:
-      TOXENV: "unit"
-      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
-
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -88,27 +77,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install tox
-          python -m pip --version
-          tox --version
-
-      - name: Run tox
-        run: tox
-
-      - name: Get current date
-        if: always()
-        id: date
-        #no colons allowed for artifacts
-        run: |
-          echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT
+        uses: pypa/hatch@install
 
-      - uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
-          path: unit_results.csv
+      - run: hatch run unit-tests
 
   build:
     name: build packages
@@ -130,24 +101,16 @@ jobs:
           python-version: '3.9'
 
       - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install --upgrade setuptools wheel twine check-wheel-contents
-          python -m pip --version
+        uses: pypa/hatch@install
 
       - name: Build distributions
-        run: ./scripts/build-dist.sh
+        run: hatch build
 
       - name: Show distributions
         run: ls -lh dist/
 
       - name: Check distribution descriptions
-        run: |
-          twine check dist/*
-
-      - name: Check wheel contents
-        run: |
-          check-wheel-contents dist/*.whl --ignore W007,W008
+        run: hatch run build:check-all
 
       - name: Check if this is an alpha version
         id: check-is-alpha
@@ -156,11 +119,6 @@ jobs:
           if [[ "$(ls -lh dist/)" == *"a1"* ]]; then export is_alpha=1; fi
           echo "is_alpha=$is_alpha" >> $GITHUB_OUTPUT
 
-      - uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist/
-
   test-build:
     name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
 
@@ -175,6 +133,7 @@ jobs:
       matrix:
         os: [ubuntu-22.04, macos-14, windows-2022]
         python-version: ['3.9', '3.10', '3.11', '3.12']
+        dist-type: ["whl", "gz"]
         exclude:
           # psycopg2-binary doesn't have a precompiled wheel for python 3.9 for mac
           - os: macos-14
@@ -186,32 +145,13 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install python dependencies
-        run: |
-          python -m pip install  --user --upgrade pip
-          python -m pip install --upgrade wheel
-          python -m pip --version
-
-      - uses: actions/download-artifact@v3
-        with:
-          name: dist
-          path: dist/
-
       - name: Show distributions
         run: ls -lh dist/
 
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-
-      - name: Check wheel distributions
-        run: |
-          python -c "import dbt.adapters.redshift"
-
-      - name: Install source distributions
+      - name: Install ${{ matrix.dist-type }} distributions
         run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+          find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
 
-      - name: Check source distributions
+      - name: Check ${{ matrix.dist-type }} distributions
         run: |
-          python -c "import dbt.adapters.redshift"
+          python -c "import dbt.adapters.bigquery"
diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml
@@ -50,11 +50,15 @@ jobs:
           commit_sha=$(git rev-parse HEAD)
           echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT
 
-      - name: "Get Current Version Number"
-        id: version-number-sources
-        run: |
-          current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
-          echo "current_version=$current_version" >> $GITHUB_OUTPUT
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      - uses: pypa/hatch@install
+
+      - id: version-number-sources
+        run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT
 
       - name: "Audit Version And Parse Into Parts"
         id: semver