ci: detect if runner is ready for integration-parallel tests

codex-storage · Feb 20, 2025 · 9904622 · 9904622
1 parent 992f7bc
commit 9904622
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 7 deletions.
diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml
@@ -17,6 +17,7 @@ env:
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
         include: ${{ fromJson(inputs.matrix) }}
 
@@ -42,6 +43,45 @@ jobs:
           nim_version: ${{ matrix.nim_version }}
           coverage: false
 
+      - name: Check runner resources for parallel integration tests
+        run: |
+          echo "Determining runner"
+          case "${{ matrix.os }}" in
+            linux)    CPU=$(nproc --all)
+                      RAM=$(awk '/MemTotal/ {print int($2 / 1024 / 1024 + 0.5)}' /proc/meminfo)
+                      ;;
+            macos)    CPU=$(sysctl -n hw.ncpu)
+                      RAM=$(sysctl -n hw.memsize | awk '{print $0/1073741824}')
+                      sysctl -n hw.ncpu
+                      ;;
+            windows)  CPU=$NUMBER_OF_PROCESSORS
+                      RAM=$(systeminfo | awk '/Total Physical Memory:/ { gsub(/,/,"."); print int($4 + 0.5) }')
+                      ;;
+            *)        CPU=2
+                      RAM=8
+                      echo "Unknown runner"
+                      ;;
+          esac
+          echo "CPU=${CPU}" >> $GITHUB_ENV
+          echo "RAM=${RAM}" >> $GITHUB_ENV
+          echo "TYPE=${RUNNER_ENVIRONMENT}" >> $GITHUB_ENV
+
+          # Set PARALLEL=1 if the runner has enough resources
+          if [[ ("${{ matrix.os }}" == "linux" || "${{ matrix.os }}" == "windows") && "${CPU}" -ge 16 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          elif [[ "${{ matrix.os }}" == "macos" && "${CPU}" -ge 6 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          else
+            echo "PARALLEL=0" >> $GITHUB_ENV
+          fi
+
+      - name: Show runner information
+        run: |
+          echo "CPU: ${{ env.CPU }}"
+          echo "RAM: ${{ env.RAM }}"
+          echo "TYPE: ${{ env.TYPE }}"
+          echo "PARALLEL: ${{ env.PARALLEL }}"
+
       ## Part 1 Tests ##
       - name: Unit tests
         if: matrix.tests == 'unittest' || matrix.tests == 'all'
@@ -76,12 +116,12 @@ jobs:
 
       ## Part 3 Tests ##
       - name: Integration tests
-        if: matrix.tests == 'integration' || matrix.tests == 'all'
+        if: (matrix.tests == 'integration' || matrix.tests == 'all') && matrix.tests != 'integration-parallel'
         run: make -j${ncpu} PARALLEL=0 testIntegration
 
       - name: Parallel integration tests
-        if: matrix.tests == 'integration-parallel'
-        run: make -j${ncpu} DEBUG=${{ runner.debug }} testIntegration
+        if: (matrix.tests == 'integration-parallel' || matrix.tests == 'all') && matrix.tests != 'integration'
+        run: make -j${ncpu} PARALLEL=${{ env.PARALLEL }} DEBUG=${{ runner.debug }} testIntegration
 
       - name: Upload integration tests log files
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -27,10 +27,15 @@ jobs:
         uses: fabiocaccamo/create-matrix-action@v5
         with:
           matrix: |
-            os {linux},   cpu {amd64}, builder {runner-node-01-linux-03-eu-hel1},   tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {macos},   cpu {amd64}, builder {macos-14-large},       tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {windows}, cpu {amd64}, builder {windows-latest-amd64-32vcpu}, tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {msys2}
-
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},               tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {arm64}, builder {ubuntu-24.04-arm},            tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest-amd64-16vcpu},  tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {amd64}, builder {macos-13},                    tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {amd64}, builder {macos-latest},                tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {amd64}, builder {macos-14-large},              tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {amd64}, builder {macos-14-xlarge},             tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {windows}, cpu {amd64}, builder {windows-latest},              tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {msys2}
+            os {windows}, cpu {amd64}, builder {windows-latest-amd64-16vcpu}, tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {msys2}
 
   build:
     needs: matrix