diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 30c8f2190..3476ae2f6 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -44,9 +44,9 @@ jobs: packages: write contents: read steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index be941be8d..815f58056 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,9 +23,9 @@ jobs: job_build_python_whl: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: 3.11 - name: Install dependencies @@ -53,7 +53,7 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to Docker Hub uses: docker/login-action@v2 @@ -96,10 +96,10 @@ jobs: env: PYTHONPATH: home/runner/work/core/ steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 9182c1934..d0803c87c 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Sonarcloud Scan' +name: "Sonarcloud Scan" run-name: ${{ github.event.workflow_run.display_title }} @@ -27,53 +27,53 @@ on: jobs: job_download_pr_artifact: - outputs: - pr_info: ${{ steps.pr.outputs.result }} - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: 'Download Artifact' - uses: actions/github-script@v6 - with: - script: | - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.payload.workflow_run.id, - }); - let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { - return artifact.name == "pr_number" - })[0]; - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); + outputs: + pr_info: ${{ steps.pr.outputs.result }} + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: "Download Artifact" + uses: actions/github-script@v7 + with: + script: | + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == "pr_number" + })[0]; + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_number.zip`, Buffer.from(download.data)); - - name: 'Unzip Artifact' - run: unzip pr_number.zip + - name: "Unzip Artifact" + run: unzip pr_number.zip - - name: 'Read Artifact' - id: pr - uses: actions/github-script@v6 - with: - script: | - let fs = require('fs'); - return fs.readFileSync('./pr_number'); - result-encoding: string + - name: "Read Artifact" + id: pr + uses: actions/github-script@v7 + with: + script: | + let fs = require('fs'); + return fs.readFileSync('./pr_number'); + result-encoding: string job_run_unit_tests_and_sonarqube: - needs: job_download_pr_artifact - uses: rtdip/core/.github/workflows/sonarcloud_reusable.yml@develop - with: - REPO_NAME: ${{ github.event.workflow_run.head_repository.full_name }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - PR_NUMBER: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_number }} - PR_HEAD_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_head_ref }} - PR_BASE_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_base_ref }} - secrets: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + needs: job_download_pr_artifact + uses: rtdip/core/.github/workflows/sonarcloud_reusable.yml@develop + with: + REPO_NAME: ${{ github.event.workflow_run.head_repository.full_name }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + PR_NUMBER: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_number }} + PR_HEAD_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_head_ref }} + PR_BASE_REF: ${{ fromJSON(needs.job_download_pr_artifact.outputs.pr_info).pr_base_ref }} + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/sonarcloud_reusable.yml b/.github/workflows/sonarcloud_reusable.yml index f3a3fd021..884fe15d4 100644 --- a/.github/workflows/sonarcloud_reusable.yml +++ b/.github/workflows/sonarcloud_reusable.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Reusable Sonarcloud Scan' +name: "Reusable Sonarcloud Scan" on: workflow_call: @@ -25,20 +25,20 @@ on: type: string HEAD_SHA: required: true - type: string + type: string PR_NUMBER: required: true - type: string + type: string PR_HEAD_REF: required: true type: string PR_BASE_REF: required: true - type: string + type: string secrets: SONAR_TOKEN: required: true - + jobs: job_test_python_pyspark_latest_version: defaults: @@ -50,19 +50,19 @@ jobs: python-version: ["3.11"] pyspark: ["3.5.1"] delta-spark: ["3.0.0"] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: ${{ inputs.REPO_NAME }} ref: ${{ inputs.HEAD_BRANCH }} fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 + - name: Setup Python + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - + - name: Install Boost run: | sudo apt update @@ -80,7 +80,7 @@ jobs: create-args: >- python=${{ matrix.python-version }} pyspark=${{ matrix.pyspark }} - delta-spark=${{ matrix.delta-spark }} + delta-spark=${{ matrix.delta-spark }} cache-environment: true - name: Test @@ -93,12 +93,11 @@ jobs: coverage xml --omit "venv/**,maintenance/**,xunit-reports/**" -i -o coverage-reports/coverage-unittests.xml echo Coverage `coverage report --omit "venv/**" | grep TOTAL | tr -s ' ' | cut -d" " -f4` - - name: Mkdocs Test run: | mkdocs build --strict - - name: Override Coverage Source Path for Sonar + - name: Override Coverage Source Path for Sonar run: | sed -i "s/\/home\/runner\/work\/core\/core<\/source>/\/github\/workspace<\/source>/g" /home/runner/work/core/core/coverage-reports/coverage-unittests.xml @@ -115,5 +114,5 @@ jobs: -Dsonar.pullrequest.branch=${{ inputs.PR_HEAD_REF }} -Dsonar.pullrequest.base=${{ inputs.PR_BASE_REF }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 192747f41..36333108e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,88 +12,88 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Reusable Test Workflow' +name: "Reusable Test Workflow" on: workflow_call: jobs: job_test_python_pyspark_versions: - defaults: - run: - shell: bash -l {0} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - python-version: ["3.9", "3.10", "3.11"] - pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0"] - exclude: - - pyspark: "3.5.0" - python-version: "3.9" - - pyspark: "3.5.0" - python-version: "3.10" - - pyspark: "3.4.1" - python-version: "3.9" - - pyspark: "3.4.1" - python-version: "3.10" - - pyspark: "3.4.0" - python-version: "3.9" - - pyspark: "3.4.0" - python-version: "3.10" - - pyspark: "3.3.2" - python-version: "3.11" - - pyspark: "3.3.1" - python-version: "3.11" - - pyspark: "3.3.0" - python-version: "3.11" - include: - - pyspark: "3.3.0" - delta-spark: "2.2.0" - - pyspark: "3.3.1" - delta-spark: "2.3.0" - - pyspark: "3.3.2" - delta-spark: "2.3.0" - - pyspark: "3.4.0" - delta-spark: "2.4.0" - - pyspark: "3.4.1" - delta-spark: "2.4.0" - - pyspark: "3.5.0" - delta-spark: "3.0.0" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.9", "3.10", "3.11"] + pyspark: ["3.3.0", "3.3.1", "3.3.2", "3.4.0", "3.4.1", "3.5.0"] + exclude: + - pyspark: "3.5.0" + python-version: "3.9" + - pyspark: "3.5.0" + python-version: "3.10" + - pyspark: "3.4.1" + python-version: "3.9" + - pyspark: "3.4.1" + python-version: "3.10" + - pyspark: "3.4.0" + python-version: "3.9" + - pyspark: "3.4.0" + python-version: "3.10" + - pyspark: "3.3.2" + python-version: "3.11" + - pyspark: "3.3.1" + python-version: "3.11" + - pyspark: "3.3.0" + python-version: "3.11" + include: + - pyspark: "3.3.0" + delta-spark: "2.2.0" + - pyspark: "3.3.1" + delta-spark: "2.3.0" + - pyspark: "3.3.2" + delta-spark: "2.3.0" + - pyspark: "3.4.0" + delta-spark: "2.4.0" + - pyspark: "3.4.1" + delta-spark: "2.4.0" + - pyspark: "3.5.0" + delta-spark: "3.0.0" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Boost - run: | - sudo apt update - sudo apt install -y libboost-all-dev + - name: Install Boost + run: | + sudo apt update + sudo apt install -y libboost-all-dev - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH - - name: Install Conda environment with Micromamba - uses: mamba-org/setup-micromamba@main - with: - environment-file: environment.yml - create-args: >- - python=${{ matrix.python-version }} - pyspark=${{ matrix.pyspark }} - delta-spark=${{ matrix.delta-spark }} - cache-environment: true + - name: Install Conda environment with Micromamba + uses: mamba-org/setup-micromamba@main + with: + environment-file: environment.yml + create-args: >- + python=${{ matrix.python-version }} + pyspark=${{ matrix.pyspark }} + delta-spark=${{ matrix.delta-spark }} + cache-environment: true - - name: Test - run: | - coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests + - name: Test + run: | + coverage run -m pytest --junitxml=xunit-reports/xunit-result-unitttests.xml tests job_test_mkdocs: defaults: @@ -105,19 +105,19 @@ jobs: python-version: ["3.11"] pyspark: ["3.5.0"] delta-spark: ["3.0.0"] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: ${{ inputs.REPO_NAME }} ref: ${{ inputs.HEAD_BRANCH }} fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v4 + - name: Setup Python + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - + - name: Install Boost run: | sudo apt update @@ -135,7 +135,7 @@ jobs: create-args: >- python=${{ matrix.python-version }} pyspark=${{ matrix.pyspark }} - delta-spark=${{ matrix.delta-spark }} + delta-spark=${{ matrix.delta-spark }} cache-environment: true - name: Mkdocs Test @@ -145,5 +145,5 @@ jobs: job_lint_python_black: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: psf/black@stable \ No newline at end of file + - uses: actions/checkout@v4 + - uses: psf/black@stable