Nightly v5 integration tests #483
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly v5 integration tests | |
| on: | |
| schedule: | |
| - cron: "15 9 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| send-slack-message: | |
| description: 'whether to send a message to #daq-release-notifications on completion' | |
| default: 'no' | |
| override-nightly-tag: | |
| description: 'manually specify the nightly tag (optional)' | |
| required: false | |
| candidate-release-tag: | |
| description: 'manually specify a candidate release tag (optional)' | |
| required: false | |
| default: "" | |
| frozen-release-tag: | |
| description: 'manually specify a frozen release tag (optional)' | |
| required: false | |
| default: "" | |
| jobs: | |
| make_variables: | |
| name: create nightly tag, timestamp, etc. | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.create_variables.outputs.release_tag }} | |
| is_candidate_release: ${{ steps.create_variables.outputs.is_candidate }} | |
| is_frozen_release: ${{ steps.create_variables.outputs.is_frozen }} | |
| timestamp: ${{ steps.create_variables.outputs.timestamp }} | |
| persistent_log_dir: ${{ steps.create_variables.outputs.persistent_log_dir }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - id: create_variables | |
| run: | | |
| is_candidate=false | |
| is_frozen=false | |
| if [[ -n "${{ github.event.inputs.candidate-release-tag }}" && -n "${{ github.event.inputs.frozen-release-tag }}" ]]; then | |
| echo "Error: You cannot specify both a candidate tag and a frozen tag. Please choose one and try again. Exiting..." | |
| exit 1 | |
| fi | |
| if [ -n "${{ github.event.inputs.override-nightly-tag }}" ]; then | |
| release_tag="${{ github.event.inputs.override-nightly-tag }}" | |
| if [[ -n "${{ github.event.inputs.candidate-release-tag }}" || -n "${{ github.event.inputs.frozen-release-tag }}" ]]; then | |
| echo "WARNING: The provided nightly tag will supercede the provided candidate or frozen release tag." | |
| fi | |
| elif [ -n "${{ github.event.inputs.candidate-release-tag }}" ]; then | |
| release_tag="${{ github.event.inputs.candidate-release-tag }}" | |
| is_candidate=true | |
| elif [ -n "${{ github.event.inputs.frozen-release-tag }}" ]; then | |
| release_tag="${{ github.event.inputs.frozen-release-tag }}" | |
| is_frozen=true | |
| else | |
| date_tag=$(date +%y%m%d) | |
| release_tag="NFD_DEV_${date_tag}_A9" | |
| fi | |
| timestamp=$(date +"%Y-%m-%d-%H%M%Z") | |
| persistent_log_dir=/home/nfs/dunedaq/github-actions/pytest-logs-${timestamp} | |
| echo "release_tag=${release_tag}" >> "$GITHUB_OUTPUT" | |
| echo "is_candidate=${is_candidate}" >> "$GITHUB_OUTPUT" | |
| echo "is_frozen=${is_frozen}" >> "$GITHUB_OUTPUT" | |
| echo "timestamp=${timestamp}" >> "$GITHUB_OUTPUT" | |
| echo "persistent_log_dir=${persistent_log_dir}" >> "$GITHUB_OUTPUT" | |
| cat "$GITHUB_OUTPUT" | |
| integration_tests: | |
| name: integration_tests | |
| runs-on: daq | |
| timeout-minutes: 30 | |
| needs: make_variables | |
| outputs: | |
| integtest_xml_path: ${{ steps.setup_and_run_test.outputs.INTEGTEST_OUTPUT_PATH }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_name: [ | |
| "listrev_test", | |
| "3ru_1df_multirun_test", | |
| "3ru_3df_multirun_test", | |
| "example_system_test", | |
| "fake_data_producer_test", | |
| "long_window_readout_test", | |
| "minimal_system_quick_test", | |
| "readout_type_scan", | |
| "small_footprint_quick_test", | |
| "tpstream_writing_test" | |
| ] | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: setup release and run tests | |
| env: | |
| RELEASE_TAG: ${{needs.make_variables.outputs.tag}} | |
| IS_CANDIDATE_RELEASE: ${{needs.make_variables.outputs.is_candidate_release}} | |
| IS_FROZEN_RELEASE: ${{needs.make_variables.outputs.is_frozen_release}} | |
| PERSISTENT_LOG_DIR: ${{needs.make_variables.outputs.persistent_log_dir}} | |
| id: setup_and_run_test | |
| run: | | |
| DET=fd | |
| INTEGTEST_OUTPUT_PATH=$GITHUB_WORKSPACE/integration_tests_$RELEASE_TAG | |
| mkdir -p $INTEGTEST_OUTPUT_PATH | |
| cd $INTEGTEST_OUTPUT_PATH | |
| source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh | |
| setup_dbt latest_v5 | |
| if [[ "$IS_CANDIDATE_RELEASE" == "true" ]]; then | |
| echo "dbt-setup-release -b candidate $RELEASE_TAG" | |
| [[ -e /cvmfs/dunedaq-development.opensciencegrid.org/candidates/$RELEASE_TAG/daq_app_rte.sh ]] | |
| dbt-setup-release -b candidate $RELEASE_TAG | |
| elif [[ "$IS_FROZEN_RELEASE" == "true" ]]; then | |
| echo "dbt-setup-release $RELEASE_TAG" | |
| [[ -e /cvmfs/dunedaq.opensciencegrid.org/spack/releases/$RELEASE_TAG/daq_app_rte.sh ]] | |
| dbt-setup-release $RELEASE_TAG | |
| else | |
| echo "dbt-setup-release -n $RELEASE_TAG" | |
| [[ -e /cvmfs/dunedaq-development.opensciencegrid.org/nightly/$RELEASE_TAG/daq_app_rte.sh ]] | |
| dbt-setup-release -n $RELEASE_TAG | |
| fi | |
| TEST_PATH="" | |
| if [[ "${{ matrix.test_name }}" == "listrev_test" ]]; then | |
| TEST_PATH=$LISTREV_SHARE/integtest | |
| else | |
| TEST_PATH=$DAQSYSTEMTEST_SHARE/integtest | |
| fi | |
| echo "INTEGTEST_OUTPUT_PATH=$INTEGTEST_OUTPUT_PATH" >> "$GITHUB_OUTPUT" | |
| pytest_retval=0 | |
| pytest -v -s --junit-xml=${{ matrix.test_name }}_results.xml \ | |
| $TEST_PATH/${{ matrix.test_name }}.py || { pytest_retval=$?; true; } | |
| mkdir -p $PERSISTENT_LOG_DIR | |
| cp -rL $(readlink -f /tmp/pytest-of-dunedaq/pytest-current/) $PERSISTENT_LOG_DIR/${{ matrix.test_name }} | |
| chmod -R 755 $PERSISTENT_LOG_DIR | |
| if [[ $(du -sk $PERSISTENT_LOG_DIR | awk '{print $1}') -gt 10000000 ]]; then | |
| echo "The log directory $PERSISTENT_LOG_DIR has exceeded 10 GB. Deleting and exiting..." | |
| rm -rf $PERSISTENT_LOG_DIR | |
| exit 2 | |
| fi | |
| test $pytest_retval == 0 || false | |
| parse_results: | |
| runs-on: daq | |
| if: always() | |
| needs: [make_variables, integration_tests] | |
| steps: | |
| - name: Checkout daq-release | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: DUNE-DAQ/daq-release | |
| path: daq-release-integtest | |
| - name: Generate summary tables | |
| env: | |
| RELEASE_TAG: ${{needs.make_variables.outputs.tag}} | |
| run: | | |
| cd daq-release-integtest/scripts/github-ci/ | |
| python integtest_xml_parser.py --input-directory ${{ github.workspace }}/integration_tests_$RELEASE_TAG | |
| cat pytest_summary_table.md >> $GITHUB_STEP_SUMMARY | |
| store_and_upload_logs: | |
| runs-on: daq | |
| timeout-minutes: 5 | |
| needs: [make_variables, integration_tests] | |
| if: success() || failure() | |
| outputs: | |
| integtest_log_path: ${{ steps.copy_logs.outputs.PERSISTENT_LOG_DIR }} | |
| steps: | |
| - name: Upload logs as artifact | |
| env: | |
| TIMESTAMP: ${{needs.make_variables.outputs.timestamp}} | |
| PERSISTENT_LOG_DIR: ${{needs.make_variables.outputs.persistent_log_dir}} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pytest-logs-${{ env.TIMESTAMP }} | |
| path: ${{ env.PERSISTENT_LOG_DIR }} | |
| - name: Delete old log directories | |
| run: | | |
| old_dirs=$(find /home/nfs/dunedaq/github-actions -mindepth 1 -maxdepth 1 -type d -mtime +7) | |
| echo "Removing the following directories that are more than 7 days old: $old_dirs" | |
| rm -rf $old_dirs | |
| send_slack_message: | |
| if: (github.event_name != 'workflow_dispatch' && failure()) || (github.event_name == 'workflow_dispatch' && github.event.inputs.send-slack-message == 'yes') | |
| needs: [integration_tests, store_and_upload_logs] | |
| uses: ./.github/workflows/slack-notification.yml | |
| with: | |
| integtest_xml_dir: ${{ needs.integration_tests.outputs.integtest_xml_path }} | |
| integtest_log_dir: ${{ needs.store_and_upload_logs.outputs.integtest_log_path }} | |
| secrets: | |
| slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| cleanup_files: | |
| runs-on: daq | |
| if: always() | |
| needs: [make_variables, parse_results, send_slack_message] | |
| steps: | |
| - name: Remove xml files | |
| env: | |
| RELEASE_TAG: ${{needs.make_variables.outputs.tag}} | |
| run: | | |
| rm -rf ${{ github.workspace }}/integration_tests_$RELEASE_TAG | |
| - name: Remove daq-release | |
| run: | | |
| rm -rf ${{ github.workspace }}/daq-release-integtest/ | |
| # Integration tests can sometimes collide with stale processes, leading to timeout | |
| cleanup_stale_gunicorn_processes: | |
| runs-on: daq | |
| if: always() | |
| needs: integration_tests | |
| steps: | |
| - name: Run cleanup script | |
| run: | | |
| /home/nfs/dunedaq/kill_stale_gunicorn_processes.sh | |