Merge remote-tracking branch 'origin/master' into aarondonahue/io/add…

…_ilev_to_output

.github/workflows/eamxx-sa-coverage.yml

@@ -0,0 +1,58 @@
+name: eamxx-sa-coverage
+
+on:
+  workflow_dispatch:
+
+  # Add schedule trigger for nightly runs at midnight MT (Standard Time)
+  schedule:
+    - cron: '0 7 * * *'  # Runs at 7 AM UTC, which is midnight MT during Standard Time
+
+concurrency:
+  # Two runs are in the same group if they are testing the same git ref
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }}  # Submit to cdash only for nightlies
+
+jobs:
+  gcc-openmp:
+    runs-on:  [self-hosted, ghci-snl-cpu, gcc]
+    name: gcc-openmp / cov
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          show-progress: false
+          submodules: recursive
+      - name: Show action trigger
+        uses: ./.github/actions/show-workflow-trigger
+      - name: Run tests
+        uses: ./.github/actions/test-all-scream
+        with:
+          build_type: cov
+          machine: ghci-snl-cpu
+          generate: false
+          submit: ${{ env.submit }}
+          cmake-configs: Kokkos_ENABLE_OPENMP=ON
+  gcc-cuda:
+    runs-on:  [self-hosted, ghci-snl-cuda, cuda, gcc]
+    name: gcc-cuda / cov
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          show-progress: false
+          submodules: recursive
+      - name: Show action trigger
+        uses: ./.github/actions/show-workflow-trigger
+      - name: Run tests
+        uses: ./.github/actions/test-all-scream
+        with:
+          build_type: cov
+          machine: ghci-snl-cuda
+          generate: false
+          submit: ${{ env.submit }}
+          cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70

.github/workflows/eamxx-sa-sanitizer.yml

@@ -0,0 +1,62 @@
+name: eamxx-sa-sanitizer
+
+on:
+  workflow_dispatch:
+
+  # Add schedule trigger for nightly runs at midnight MT (Standard Time)
+  schedule:
+    - cron: '0 7 * * *'  # Runs at 7 AM UTC, which is midnight MT during Standard Time
+
+concurrency:
+  # Two runs are in the same group if they are testing the same git ref
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }}  # Submit to cdash only for nightlies
+
+jobs:
+  gcc-openmp:
+    runs-on:  [self-hosted, ghci-snl-cpu, gcc]
+    name: gcc-openmp / cov
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          show-progress: false
+          submodules: recursive
+      - name: Show action trigger
+        uses: ./.github/actions/show-workflow-trigger
+      - name: Run tests
+        uses: ./.github/actions/test-all-scream
+        with:
+          build_type: valg
+          machine: ghci-snl-cpu
+          generate: false
+          submit: ${{ env.submit }}
+          cmake-configs: Kokkos_ENABLE_OPENMP=ON
+  gcc-cuda:
+    runs-on:  [self-hosted, ghci-snl-cuda, cuda, gcc]
+    strategy:
+      fail-fast: false
+      matrix:
+        build_type: [csm, csr, csi, css]
+    name: gcc-cuda / ${{ matrix.build_type }}
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          show-progress: false
+          submodules: recursive
+      - name: Show action trigger
+        uses: ./.github/actions/show-workflow-trigger
+      - name: Run tests
+        uses: ./.github/actions/test-all-scream
+        with:
+          build_type: ${{ matrix.build_type }}
+          machine: ghci-snl-cuda
+          generate: false
+          submit: ${{ env.submit }}
+          cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70

.github/workflows/eamxx-sa-testing.yml

@@ -0,0 +1,176 @@
+name: eamxx-sa
+
+on:
+  # Runs on PRs against master
+  pull_request:
+    branches: [ master ]
+    types: [opened, synchronize, ready_for_review, reopened]
+
+  # Manual run is used to bless
+  workflow_dispatch:
+    inputs:
+      job_to_run:
+        description: 'Job to run'
+        required: true
+        type: choice
+        options:
+          - gcc-openmp
+          - gcc-cuda
+          - all
+      bless:
+        description: 'Generate baselines'
+        required: true
+        type: boolean
+
+  # Add schedule trigger for nightly runs at midnight MT (Standard Time)
+  schedule:
+    - cron: '0 7 * * *'  # Runs at 7 AM UTC, which is midnight MT during Standard Time
+
+concurrency:
+  # Two runs are in the same group if they are testing the same git ref
+  #  - if trigger=pull_request, the ref is refs/pull/<PR_NUMBER>/merge
+  #  - for other triggers, the ref is the branch tested
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }}  # Submit to cdash only for nightlies
+
+jobs:
+  pre_process_pr:
+    if: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest  # This job can run anywhere
+    outputs:
+      relevant_paths: ${{ steps.check_paths.outputs.value }}
+      labels: ${{ steps.get_labels.outputs.labels }}
+    steps:
+      - name: Check files modified by PR
+        id: check_paths
+        run: |
+          paths=(
+            components/eamxx
+            components/eam/src/physics/rrtmgp
+            components/eam/src/physics/p3/scream
+            components/eam/src/physics/cam
+            components/eam/src/physics/rrtmgp/external
+            externals/ekat
+            externals/scorpio
+            externals/haero
+            externals/YAKL
+            .github/workflows/eamxx-sa-testing.yml
+          )
+          pattern=$(IFS=\|; echo "${paths[*]}")
+
+          # Use the GitHub API to get the list of changed files
+          response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.number }}/files")
+          changed_files=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')
+
+          # Check for matches and echo the matching files (or "" if none)
+          matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "")
+          if [[ -n "$matching_files" ]]; then
+            echo "Found relevant files: $matching_files"
+            echo "value=true" >> $GITHUB_OUTPUT
+          else
+            echo "No relevant files touched by this PR."
+            echo "value=false" >> $GITHUB_OUTPUT
+          fi
+      - name: Retrieve PR labels
+        id: get_labels
+        run: |
+          labels="${{ join(github.event.pull_request.labels.*.name, ',') }}"
+          echo "labels=${labels}" >> $GITHUB_OUTPUT
+  gcc-openmp:
+    needs: [pre_process_pr]
+    if: |
+      success() && github.event_name == 'schedule' ||
+      (
+        github.event_name == 'pull_request' &&
+        needs.pre_process_pr.outputs.relevant_paths=='true' &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all')
+      ) || (
+        github.event_name == 'workflow_dispatch' &&
+        github.event.inputs.job_to_run == 'gcc-openmp' || 
+        github.event.inputs.job_to_run == 'all'
+      )
+    runs-on:  [self-hosted, ghci-snl-cpu, gcc]
+    strategy:
+      fail-fast: false
+      matrix:
+        build_type: [sp, dbg, fpe, opt]
+    name: gcc-openmp / ${{ matrix.build_type }}
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          show-progress: false
+          submodules: recursive
+      - name: Show action trigger
+        uses: ./.github/actions/show-workflow-trigger
+      - name: Set test-all inputs based on event specs
+        run: |
+          echo "generate=false" >> $GITHUB_ENV
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            if [ "${{ inputs.bless }}" == "true" ]; then
+              echo "generate=true" >> $GITHUB_ENV
+            fi
+          fi
+      - name: Run tests
+        uses: ./.github/actions/test-all-scream
+        with:
+          build_type: ${{ matrix.build_type }}
+          machine: ghci-snl-cpu
+          generate: ${{ env.generate }}
+          submit: ${{ env.submit }}
+          cmake-configs: Kokkos_ENABLE_OPENMP=ON
+  gcc-cuda:
+    needs: [pre_process_pr]
+    if: |
+      success() && github.event_name == 'schedule' ||
+      (
+        github.event_name == 'pull_request' &&
+        needs.pre_process_pr.outputs.relevant_paths=='true' &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') &&
+        !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all')
+      ) || (
+        github.event_name == 'workflow_dispatch' &&
+        github.event.inputs.job_to_run == 'gcc-cuda' ||
+        github.event.inputs.job_to_run == 'all'
+      )
+    runs-on:  [self-hosted, ghci-snl-cuda, cuda, gcc]
+    strategy:
+      fail-fast: false
+      matrix:
+        build_type: [sp, dbg, opt]
+    name: gcc-cuda / ${{ matrix.build_type }}
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          show-progress: false
+          submodules: recursive
+      - name: Show action trigger
+        uses: ./.github/actions/show-workflow-trigger
+      - name: Set test-all inputs based on event specs
+        run: |
+          echo "generate=false" >> $GITHUB_ENV
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            if [ "${{ inputs.bless }}" == "true" ]; then
+              echo "generate=true" >> $GITHUB_ENV
+            fi
+          fi
+      - name: Run tests
+        uses: ./.github/actions/test-all-scream
+        with:
+          build_type: ${{ matrix.build_type }}
+          machine: ghci-snl-cuda
+          generate: ${{ env.generate }}
+          submit: ${{ env.submit }}
+          cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70