[DCJ-616] Update unit and connected tests in GitHub Action to latest standards #6768
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit, Smoke, Connected and Integration tests | |
| on: | |
| workflow_dispatch: {} | |
| pull_request: | |
| schedule: | |
| - cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST. | |
| # do not allow concurrent runs of this workflow on the same branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| test_unit: | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: Run unit tests | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS: 'jade-dev-account.json' | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # extract service account credentials | |
| base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| # assemble code, run unit tests, and generate scan | |
| ./gradlew assemble | |
| ./gradlew check --scan jacocoTestReport sonar | |
| test_connected: | |
| name: Connected tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: Run connected tests | |
| env: | |
| # required for azure tests | |
| AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571 | |
| AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408 | |
| AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }} | |
| # required for synapse tests | |
| AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }} | |
| AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }} | |
| AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net | |
| # required for connected tests | |
| GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json | |
| # required data project for snapshotTest | |
| GOOGLE_CLOUD_DATA_PROJECT: broad-jade-integration-data | |
| # required for testAzureBillingProfile | |
| # uses an azure marketplace app with this hardcoded deployment email | |
| JADE_USER_EMAIL: connected-tdr-user@notarealemail.org | |
| # required for rbs tests | |
| RBS_CLIENT_CREDENTIAL_FILE_PATH: rbs-tools-sa.json | |
| # output plain logs instead of json | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| run: | | |
| # extract service account credentials | |
| base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS} | |
| base64 --decode <<< ${{ secrets.B64_RBS_APPLICATION_CREDENTIALS }} > ${RBS_CLIENT_CREDENTIAL_FILE_PATH} | |
| # assemble code and run connected tests | |
| ./gradlew assemble | |
| ./gradlew testConnected --scan | |
| test_integration: | |
| name: Integration tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| services: | |
| postgres: | |
| image: postgres:11 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| ports: | |
| - 5432:5432 | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| env: | |
| # This must be defined for the bash redirection | |
| GOOGLE_APPLICATION_CREDENTIALS: 'jade-dev-account.json' | |
| # This must be defined for the bash redirection | |
| GOOGLE_SA_CERT: 'jade-dev-account.pem' | |
| # Required for locking and deployment to integration namespace | |
| K8_CLUSTER: 'integration-master' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '17' | |
| cache: 'gradle' | |
| - name: Whitelist Runner IP | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gcp_whitelist' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Check for an available namespace to deploy API to and set state lock | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'k8_checknamespace' | |
| k8_namespaces: 'integration-1,integration-2,integration-3,integration-6' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Build docker container via Gradle | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| env: | |
| # Unset the GitHub Action default JAVA_HOME to build with JDK 17 | |
| JAVA_HOME: | |
| with: | |
| actions_subcommand: 'gradlebuild' # creates gcr build with git_hash tag | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Deploy to cluster with Helm | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'helmdeploy' | |
| helm_create_secret_manager_secret_version: 0.0.8 | |
| helm_datarepo_api_chart_version: 0.0.744 | |
| helm_datarepo_ui_chart_version: 0.0.362 | |
| helm_gcloud_sqlproxy_chart_version: 0.19.13 | |
| helm_oidc_proxy_chart_version: 0.0.44 | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Fetch gitHash for deployed integration version | |
| id: configuration | |
| run: | | |
| git_hash=$(git rev-parse --short HEAD) | |
| echo "git_hash=${git_hash}" >> $GITHUB_OUTPUT | |
| echo "Latest git hash for this branch: $git_hash" | |
| - name: Wait for deployment to come back online | |
| uses: broadinstitute/datarepo-actions/actions/wait-for-deployment@0.74.0 | |
| timeout-minutes: 20 | |
| env: | |
| DESIRED_GITHASH: ${{ steps.configuration.outputs.git_hash }} | |
| DEPLOYMENT_TYPE: 'api' | |
| - name: Run test runner smoke tests via Gradle | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gradletestrunnersmoketest' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Run integration tests via Gradle | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| env: | |
| AZURE_CREDENTIALS_APPLICATIONID: ${{ env.AZURE_CREDENTIALS_APPLICATIONID }} | |
| AZURE_CREDENTIALS_HOMETENANTID: ${{ env.AZURE_CREDENTIALS_HOMETENANTID }} | |
| TDR_LOG_APPENDER: 'Console-Standard' | |
| with: | |
| actions_subcommand: 'gradleinttest' | |
| pgport: ${{ job.services.postgres.ports[5432] }} | |
| test_to_run: 'testIntegration' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Clean state lock from used Namespace on API deploy | |
| if: always() | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'k8_checknamespace_clean' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} | |
| - name: Clean whitelisted Runner IP | |
| if: always() | |
| uses: broadinstitute/datarepo-actions/actions/main@0.74.0 | |
| with: | |
| actions_subcommand: 'gcp_whitelist_clean' | |
| sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }} |