From d9df099b42eee2491ebc93161e47c5ccf3b62f6b Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 15:49:21 -0300 Subject: [PATCH 1/8] ci: upgrade actions for workflow Test Changes --- .github/workflows/test-changes.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index 38114967..25410179 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -50,7 +50,7 @@ jobs: echo 'python_eol=yes' >> $GITHUB_ENV - name: Checkout source code - uses: actions/checkout@main + uses: actions/checkout@v4 - name: Install linux tools if: matrix.os == 'ubuntu-latest' @@ -60,13 +60,13 @@ jobs: - name: Set up Python ${{ matrix.python }} if: env.python_eol == 'no' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Set up Python ${{ matrix.python }} discontinued on ${{ matrix.os }} if: env.python_eol == 'yes' - uses: MatteoH2O1999/setup-python@v1 + uses: MatteoH2O1999/setup-python@v3 with: python-version: ${{ matrix.python }} cache: pip @@ -160,10 +160,10 @@ jobs: steps: - name: Checkout source code - uses: actions/checkout@main + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} From 1e9056ee306674520f96055891cb782e4b8999cf Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 15:58:51 -0300 Subject: [PATCH 2/8] ci: narrow triggering of the workflow Test Changes --- .github/workflows/test-changes.yml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index 25410179..db1c1cb3 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -3,7 +3,30 @@ name: Test Changes -on: [push, pull_request] +on: + push: + branches: [ "master" ] + pull_request: + types: [opened, reopened, synchronize, ready_for_review, labeled] + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + type: choice + options: + - info + - warning + - debug + tags: + description: 'Test scenario tags' + required: false + type: boolean + environment: + description: 'Environment to run tests against' + type: environment + required: true jobs: run-guard: From 1f789eee4dd8006dbe6ef56a63fc0731dd2530e2 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 16:09:08 -0300 Subject: [PATCH 3/8] ci: changed the triggering of code analysis workflows --- .github/workflows/codacy-analysis.yml | 17 ++++++++++++++--- .github/workflows/codeql-analysis.yml | 16 ++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/codacy-analysis.yml b/.github/workflows/codacy-analysis.yml index c2e9d0b4..63059008 100644 --- a/.github/workflows/codacy-analysis.yml +++ b/.github/workflows/codacy-analysis.yml @@ -18,10 +18,21 @@ name: Codacy Security Scan on: + schedule: + - cron: '59 11 27 * *' push: - branches: [ master, main ] - pull_request: - branches: [ master, main ] + branches: [ "master" ] + # pull_request: + # # The branches below must be a subset of the branches above + # branches: [ "master" ] + workflow_run: + workflows: [Test Changes] + branches: [ "master" ] + types: + - completed + +permissions: + contents: read jobs: codacy-security-scan: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ea195e1e..83b14633 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,13 +12,17 @@ name: "CodeQL" on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] schedule: - - cron: '44 10 * * 0' + - cron: '59 10 27 * *' + push: + branches: [ "master" ] + # pull_request: + # branches: [ "master" ] + workflow_run: + workflows: [Test Changes] + branches: [ "master" ] + types: + - completed jobs: analyze: From 7f5d94627039fba8673ea178057b84e3ca01564d Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 16:29:03 -0300 Subject: [PATCH 4/8] ci: upgrade actions for code analysis workflows --- .github/workflows/codacy-analysis.yml | 37 +++++++++---------- .github/workflows/codeql-analysis.yml | 52 ++++++++++++++++++--------- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/.github/workflows/codacy-analysis.yml b/.github/workflows/codacy-analysis.yml index 63059008..3a0357f8 100644 --- a/.github/workflows/codacy-analysis.yml +++ b/.github/workflows/codacy-analysis.yml @@ -1,17 +1,13 @@ -# Codacy is an automated code review tool that makes it easy to ensure your team is writing high-quality code - -# This workflow checks out code, performs a Codacy security scan and integrates -# the results with the GitHub Advanced Security code scanning feature. - -# The following scenario is implemented: -# - Integration with GitHub code scanning: -# Analyzes each commit and pull request and uploads the results to GitHub, -# which displays the identified issues under your repository's tab Security. - -# For more information on the Codacy security scan action usage, see: -# - https://github.com/marketplace/actions/codacy-analysis-cli -# - https://github.com/codacy/codacy-analysis-cli-action +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow checks out code, performs a Codacy security scan +# and integrates the results with the +# GitHub Advanced Security code scanning feature. For more information on +# the Codacy security scan action usage and parameters, see +# https://github.com/codacy/codacy-analysis-cli-action. # For more information on Codacy Analysis CLI in general, see # https://github.com/codacy/codacy-analysis-cli. @@ -36,19 +32,22 @@ permissions: jobs: codacy-security-scan: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status name: Codacy Security Scan runs-on: ubuntu-latest steps: # Checkout the repository to the GitHub Actions runner - name: Checkout code - uses: actions/checkout@main + uses: actions/checkout@v3 # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis - name: Run Codacy Analysis CLI - uses: codacy/codacy-analysis-cli-action@master + uses: codacy/codacy-analysis-cli-action@v4 with: - # To get your project token from your Codacy repository check: - # https://github.com/codacy/codacy-analysis-cli#project-token + # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository # You can also omit the token and run the tools that support default configurations project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} verbose: true @@ -62,8 +61,6 @@ jobs: # Upload the SARIF file generated in the previous step - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@main + uses: github/codeql-action/upload-sarif@v2 with: sarif_file: results.sarif - -# end of file # diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 83b14633..8665992d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,45 +27,63 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read strategy: fail-fast: false matrix: language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository - uses: actions/checkout@main + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - #- run: | - # make bootstrap - # make release + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 322df00f1d4bbbd8311b6af620d835c0f8e82f7d Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 16:29:31 -0300 Subject: [PATCH 5/8] ci: upgrade actions for publish workflow --- .github/workflows/publish-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 8bf88fbe..cc9d12be 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source code - uses: actions/checkout@main + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} From 5f4e9cf94b8db80bd1741eb0488b4d1d1dcfbe6a Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 16:34:15 -0300 Subject: [PATCH 6/8] fix: ci fails with version 3.9.2 with python3.12 on macos-latest: PyTables/PyTables#1093 --- petl/test/io/test_pytables.py | 3 +-- requirements-formats.txt | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/petl/test/io/test_pytables.py b/petl/test/io/test_pytables.py index e174f72d..f060d8fd 100644 --- a/petl/test/io/test_pytables.py +++ b/petl/test/io/test_pytables.py @@ -31,8 +31,7 @@ def test_fromhdf5(): # set up a new hdf5 table to work with h5file = tables.open_file(f.name, mode='w', title='Test file') h5file.create_group('/', 'testgroup', 'Test Group') - h5table = h5file.create_table('/testgroup', 'testtable', FooBar, - 'Test Table') + h5table = h5file.create_table('/testgroup', 'testtable', FooBar, 'Test Table') # load some data into the table table1 = (('foo', 'bar'), diff --git a/requirements-formats.txt b/requirements-formats.txt index 04d8fc29..34692330 100644 --- a/requirements-formats.txt +++ b/requirements-formats.txt @@ -12,5 +12,5 @@ fastavro>=0.24.2 ; python_version >= '3.4' fastavro==0.24.2 ; python_version < '3.0' gspread>=3.4.0 ; python_version >= '3.4' -# version 3.7.0 doesn't work yet with python3.11 -tables ; python_version != '3.11' +# version 3.9.2 fails with python3.12 on macos-latest: PyTables/PyTables#1093 +tables ; python_version != '3.12' From 82f66446405be0969f595208b90f93d59444a7b2 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 16:35:50 -0300 Subject: [PATCH 7/8] build: doc throubleshooting on blosc bcolz deps --- requirements-optional.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/requirements-optional.txt b/requirements-optional.txt index b46163e3..007eee7a 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -2,13 +2,20 @@ # Also check: .github/workflows/test-changes.yml # Throubleshooting: -# 1. $ export DISABLE_BLOSC_AVX2=1 -# 2. $ brew install c-blosc +# 1. $ export DISABLE_BLOSC_AVX2=1 + +# 2.1 $ brew install c-blosc # On macOS +# 2.2 $ sudo apt-get install python3-dev # On debian distros +# 2.3 $ sudo dnf install python3-devel # On debian distros + +# 3.1 $ sudo find / -iname "Python.h" +# 3.2 $ export C_INCLUDE_PATH=/usr/include/python3.11/Python.h + blosc ; python_version >= '3.7' # Throubleshooting: -# 1. pip install --prefer-binary -r requirements-optional.txt -# 2. pip install --prefer-binary bcolz +# 1. $ pip install --prefer-binary -r requirements-optional.txt +# 2. $ pip install --prefer-binary bcolz bcolz ; python_version >= '3.7' and python_version < '3.10' From a26359f96dc6c869e7a88c51d70116b3c73c9203 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Thu, 14 Mar 2024 16:48:30 -0300 Subject: [PATCH 8/8] ci: list installed packages for throubleshooting --- .github/workflows/test-changes.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index db1c1cb3..6d79f69d 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -149,6 +149,16 @@ jobs: - name: Setup petl package run: python setup.py sdist bdist_wheel + - name: Install extra packages dependencies for mode full + if: env.testing == 'full' + run: python -m pip install --prefer-binary -r requirements-formats.txt + + - name: List Installed Packages for Throubleshooting + run: | + echo "::group::List Installed Packages for Throubleshooting" + python -m pip list --format freeze + echo "::endgroup::" + - name: Test python source code for mode simple if: env.testing == 'simple' run: pytest --cov=petl petl @@ -156,9 +166,6 @@ jobs: - name: Test documentation inside source code for mode full if: env.testing == 'full' run: | - echo "::group::Install extra packages test dependencies" - python -m pip install --prefer-binary -r requirements-formats.txt - echo "::endgroup::" echo "::group::Perform doctest-modules execution with coverage" pytest --doctest-modules --cov=petl petl echo "::endgroup::"