ci: check runner for parallel integration tests and use value from vars

codex-storage · Feb 21, 2025 · 50b6bee · 50b6bee
1 parent 992f7bc
commit 50b6bee
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 13 deletions.
diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml
@@ -42,6 +42,46 @@ jobs:
           nim_version: ${{ matrix.nim_version }}
           coverage: false
 
+      - name: Check runner resources for parallel integration tests
+        run: |
+          echo "Determining runner"
+          case "${{ matrix.os }}" in
+            linux)    CPU=$(nproc --all)
+                      RAM=$(awk '/MemTotal/ {print int($2 / 1024 / 1024 + 0.5)}' /proc/meminfo)
+                      ;;
+            macos)    CPU=$(sysctl -n hw.ncpu)
+                      RAM=$(sysctl -n hw.memsize | awk '{print $0/1073741824}')
+                      sysctl -n hw.ncpu
+                      ;;
+            windows)  CPU=$NUMBER_OF_PROCESSORS
+                      RAM=$(systeminfo | awk '/Total Physical Memory:/ { gsub(/,/,"."); print int($4 + 0.5) }')
+                      ;;
+            *)        CPU=2
+                      RAM=8
+                      echo "Unknown runner"
+                      ;;
+          esac
+          echo "CPU=${CPU}" >> $GITHUB_ENV
+          echo "RAM=${RAM}" >> $GITHUB_ENV
+          echo "TYPE=${RUNNER_ENVIRONMENT}" >> $GITHUB_ENV
+
+          # Set PARALLEL=1 if the runner has enough resources
+          if [[ ("${{ matrix.os }}" == "linux" || "${{ matrix.os }}" == "windows") && "${CPU}" -ge 16 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          elif [[ "${{ matrix.os }}" == "macos" && "${CPU}" -ge 6 ]]; then
+            echo "PARALLEL=1" >> $GITHUB_ENV
+          else
+            echo "PARALLEL=0" >> $GITHUB_ENV
+          fi
+
+      - name: Show runner information
+        run: |
+          echo "OS: ${{ matrix.os }}"
+          echo "CPU: ${{ env.CPU }}"
+          echo "RAM: ${{ env.RAM }} GB"
+          echo "TYPE: ${{ env.TYPE }}"
+          echo "PARALLEL: ${{ env.PARALLEL }}"
+
       ## Part 1 Tests ##
       - name: Unit tests
         if: matrix.tests == 'unittest' || matrix.tests == 'all'
@@ -54,7 +94,7 @@ jobs:
           node-version: 18.15
 
       - name: Install Ethereum node dependencies
-        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'integration-parallel' || matrix.tests == 'tools' || matrix.tests == 'all'
+        if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all'
         working-directory: vendor/codex-contracts-eth
         env:
           MSYS2_PATH_TYPE: inherit
@@ -77,17 +117,13 @@ jobs:
       ## Part 3 Tests ##
       - name: Integration tests
         if: matrix.tests == 'integration' || matrix.tests == 'all'
-        run: make -j${ncpu} PARALLEL=0 testIntegration
-
-      - name: Parallel integration tests
-        if: matrix.tests == 'integration-parallel'
-        run: make -j${ncpu} DEBUG=${{ runner.debug }} testIntegration
+        run: make -j${ncpu} PARALLEL=${{ env.PARALLEL }} DEBUG=${{ runner.debug }} testIntegration
 
       - name: Upload integration tests log files
         uses: actions/upload-artifact@v4
-        if: (matrix.tests == 'integration' || matrix.tests == 'integration-parallel' || matrix.tests == 'all') && always()
+        if: (matrix.tests == 'integration' || matrix.tests == 'all') && always()
         with:
-          name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-integration-tests-logs
+          name: ${{ matrix.os }}-${{ matrix.cpu }}-${{ matrix.nim_version }}-${{ matrix.tests }}-tests-logs
           path: tests/integration/logs/
           retention-days: 1
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,27 +10,38 @@ on:
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
   nim_version: v2.0.14
+  builder_integration_from_vars: true
+  builder_integration_linux: runner-node-01-linux-03-eu-hel1
+  builder_integration_macos: macos-14-large
+  builder_integration_windows: windows-latest-amd64-32vcpu
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   matrix:
+    name: Compute matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
       cache_nonce: ${{ env.cache_nonce }}
     steps:
+      - name: Compute builders
+        if: env.builder_integration_from_vars
+        run: |
+          if [[ -n "${{ vars.builder_integration_linux }}" ]]; then echo "builder_integration_linux=${{ vars.builder_integration_linux }}"; fi >> $GITHUB_ENV
+          if [[ -n "${{ vars.builder_integration_macos }}" ]]; then echo "builder_integration_macos=${{ vars.builder_integration_macos }}"; fi >> $GITHUB_ENV
+          if [[ -n "${{ vars.builder_integration_windows }}" ]]; then echo "builder_integration_windows=${{ vars.builder_integration_windows }}"; fi >> $GITHUB_ENV
+
       - name: Compute matrix
         id: matrix
         uses: fabiocaccamo/create-matrix-action@v5
         with:
           matrix: |
-            os {linux},   cpu {amd64}, builder {runner-node-01-linux-03-eu-hel1},   tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {macos},   cpu {amd64}, builder {macos-14-large},       tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {windows}, cpu {amd64}, builder {windows-latest-amd64-32vcpu}, tests {integration-parallel}, nim_version {${{ env.nim_version }}}, shell {msys2}
-
+            os {linux},   cpu {amd64}, builder {${{ env.builder_integration_linux }}},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {amd64}, builder {${{ env.builder_integration_macos }}},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {windows}, cpu {amd64}, builder {${{ env.builder_integration_windows }}}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2}
 
   build:
     needs: matrix
@@ -51,4 +62,3 @@ jobs:
           options: "codex/ tests/"
           fail: true
           suggest: true
-