Merge remote-tracking branch 'origin/main' into ay/autoformat_cleanup

.github/scripts/data_analysis/print_gtest_annotations.py

@@ -1,19 +1,10 @@
 import argparse
-import xmltodict
+import xml.etree.ElementTree as ET
 import glob
 import os
 from typing import Union
 
 
-def _guaranteed_list(x):
-    if not x:
-        return []
-    elif isinstance(x, list):
-        return x
-    else:
-        return [x]
-
-
 def _build_workflow_command(
     command_name: str,
     file: str,
@@ -61,29 +52,22 @@ def _escape(s: str) -> str:
 
     # Iterate through each XML file
     for xml_file in xml_files:
-        with open(xml_file, "r") as f:
-            results = xmltodict.parse(f.read())
-
-        # Check for failed tests
-        failed_tests = []
-        for testsuite in _guaranteed_list(results["testsuites"]["testsuite"]):
-            for testcase in _guaranteed_list(testsuite["testcase"]):
-                if "failure" in testcase:
-                    failed_tests.append(testcase)
-
-        # Create error annotations for each failed test
-        for failed_test in failed_tests:
-            failure_messages = _guaranteed_list(failed_test["failure"])
-            if failure_messages:
-                # first message is often enough
-                failure_message = failure_messages[0]["@message"]
-            else:
-                failure_message = "unknown_failure_message"
-
-            msg = _build_workflow_command(
-                command_name="error",
-                file=failed_test["@file"].lstrip("/work/"),
-                line=int(failed_test["@line"]),
-                message=failure_message,
-            )
-            print(msg)
+        tree = ET.parse(xml_file)
+        root = tree.getroot()
+        for testsuite in root.findall("testsuite"):
+            # Skip printing out pytest information
+            # It's already handled by pytest-github-actions-annotate-failures plugin
+            if testsuite.attrib.get("name") == "pytest":
+                continue
+            for testcase in testsuite.findall("testcase"):
+                failure = testcase.find("failure")
+                # If failure exists, print the failure message
+                if failure is not None:
+                    failure_message = failure.attrib.get("message")
+                    msg = _build_workflow_command(
+                        command_name="error",
+                        file=testcase.attrib.get("file", "").lstrip("/work/"),
+                        line=int(testcase.attrib["line"]),
+                        message=failure_message,
+                    )
+                    print(msg)

.github/workflows/_produce-data.yaml

@@ -20,22 +20,23 @@ on:
     workflows:
       - "All post-commit tests"
       - "(Single-card) Model perf tests"
-      - "(Single-card) Device perf tests"
+      - "(Single-card) Device perf regressions"
       - "(Single-card) Demo tests"
       - "(Single-card) Tests for new models"
-      - "Nightly fast dispatch tests"
+      - "(Single-card) Fast dispatch frequent tests"
       - "(Single-card) Nightly model and ttnn tests"
-      - "(Single-card) Tests for new models"
       - "(T3K) T3000 demo tests"
+      - "(T3K) T3000 frequent tests"
       - "(T3K) T3000 model perf tests"
+      - "(T3K) T3000 nightly tests"
       - "(T3K) T3000 perplexity tests"
-      - "(T3K) T3000 model perf tests"
       - "(T3K) T3000 profiler tests"
       - "(T3K) T3000 unit tests"
       - "(TG) TG unit tests"
       - "(TG) TG demo tests"
       - "(TG) TG frequent tests"
       - "(TG) TG model perf tests"
+      - "(TG) TG nightly tests"
       - "(TGG) TGG model perf tests"
       - "(TGG) TGG unit tests"
       - "(TGG) TGG demo tests"
@@ -44,6 +45,7 @@ on:
       - "Blackhole post-commit tests"
       - "Custom test dispatch"
       - "PR Gate"
+      - "Nightly tt-metal L2 tests"
     types:
       - completed
 
@@ -143,6 +145,7 @@ jobs:
       - name: Upload workflow run data, even on failure
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
+        timeout-minutes: 10
         with:
           name: workflow-run-data
           path: |

.github/workflows/_test-wheels-impl.yaml

@@ -35,6 +35,7 @@ jobs:
         with:
           os: ${{ matrix.os }}
       - uses: actions/download-artifact@v4
+        timeout-minutes: 10
         with:
           name: eager-dist-${{ matrix.os }}-any
       - name: Set up end-to-end tests environment

.github/workflows/all-post-commit-workflows.yaml

@@ -32,6 +32,7 @@ permissions:
   pages: write
   id-token: write
   packages: write
+  checks: write
 
 run-name: All post-commit tests${{ (github.event_name == 'workflow_dispatch' && inputs.with-retries) && ' (with retries)' || ''}}
 jobs:
@@ -48,8 +49,17 @@ jobs:
     uses: ./.github/workflows/build-artifact.yaml
     with:
       build-type: ${{ inputs.build-type || 'Release' }}
+      build-wheel: true
       tracy: true
     secrets: inherit
+
+  smoke-tests:
+    needs: build-artifact
+    uses: ./.github/workflows/zzz_smoke.yaml
+    with:
+      docker-image: ${{ needs.build-artifact.outputs.ci-build-docker-image }}
+      package-artifact-name: ${{ needs.build-artifact.outputs.packages-artifact-name }}
+
   # Slow Dispatch Unit Tests
   sd-unit-tests:
     needs: build-artifact
@@ -175,6 +185,9 @@ jobs:
     with:
       arch: ${{ matrix.test-group.arch}}
       runner-label: ${{ matrix.test-group.runner-label}}
+      docker-image: ${{ needs.build-artifact-profiler.outputs.ci-build-docker-image }}
+      build-artifact-name: ${{ needs.build-artifact-profiler.outputs.build-artifact-name }}
+      wheel-artifact-name: ${{ needs.build-artifact-profiler.outputs.wheel-artifact-name }}
   build-docs:
     needs: build-artifact
     uses: ./.github/workflows/docs-latest-public.yaml

.github/workflows/bisect-dispatch.yaml

@@ -81,6 +81,7 @@ jobs:
         run: |
           echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
       - uses: actions/download-artifact@v4
+        timeout-minutes: 10
         with:
           name: ${{ needs.build-artifact.outputs.build-artifact-name }}
       - name: Extract files

.github/workflows/blackhole-post-commit.yaml

@@ -54,6 +54,7 @@ jobs:
     secrets: inherit
     with:
       build-type: ${{ inputs.build-type || 'Release' }}
+      build-wheel: true
       tracy: true
       version: "20.04"
   run-profiler-regression:
@@ -63,6 +64,9 @@ jobs:
     with:
       arch: "blackhole"
       runner-label: ${{ inputs.runner-label || 'BH' }}
+      docker-image: ${{ needs.build-artifact-profiler.outputs.ci-build-docker-image }}
+      build-artifact-name: ${{ needs.build-artifact-profiler.outputs.build-artifact-name }}
+      wheel-artifact-name: ${{ needs.build-artifact-profiler.outputs.wheel-artifact-name }}
   umd-unit-tests:
     secrets: inherit
     uses: ./.github/workflows/umd-unit-tests.yaml

.github/workflows/build-and-unit-tests.yaml

@@ -100,6 +100,7 @@ jobs:
           slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
           owner: U06CXU895AP # Michael Chiou
       - uses: ./.github/actions/upload-artifact-with-job-uuid
+        timeout-minutes: 10
         if: ${{ !cancelled() }}
         with:
           path: |

.github/workflows/build-artifact.yaml

@@ -194,7 +194,7 @@ jobs:
 
           args_fixme=$([ "${{ inputs.skip-tt-train }}" = "true" ] && echo "--build-metal-tests --build-ttnn-tests --build-programming-examples" || echo "--build-all")
           echo "Args: ${args_fixme}"
-          build_command="./build_metal.sh --build-type ${{ inputs.build-type }} --toolchain-path ${{ inputs.toolchain }} ${args_fixme} --enable-ccache --configure-only --cpm-use-local-packages"
+          build_command="./build_metal.sh --build-dir build --build-type ${{ inputs.build-type }} --toolchain-path ${{ inputs.toolchain }} ${args_fixme} --enable-ccache --configure-only --cpm-use-local-packages"
           echo "Build tracy: ${{ inputs.tracy }}"
           if [ "${{ inputs.tracy }}" = "true" ]; then
             build_command="$build_command --enable-profiler"
@@ -226,6 +226,7 @@ jobs:
 
       - name: ☁️ Upload packages
         uses: actions/upload-artifact@v4
+        timeout-minutes: 10
         with:
           name: ${{ env.ARTIFACT_NAME }}
           path: |
@@ -244,6 +245,7 @@ jobs:
       - name: ☁️ Upload wheel
         if: ${{ inputs.build-wheel }}
         uses: actions/upload-artifact@v4
+        timeout-minutes: 10
         with:
           name: ${{ env.wheel_artifact_name }}
           path: /work/dist/
@@ -263,6 +265,7 @@ jobs:
       - name: ☁️ Upload tarball
         if: ${{ inputs.publish-artifact }}
         uses: actions/upload-artifact@v4
+        timeout-minutes: 10
         with:
           name: ${{ env.build_artifact_name }}
           path: /work/ttm_any.tar

.github/workflows/cpp-post-commit.yaml

@@ -105,6 +105,7 @@ jobs:
           slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
           owner: U06CXU895AP # Michael Chiou
       - uses: ./.github/actions/upload-artifact-with-job-uuid
+        timeout-minutes: 10
         if: ${{ !cancelled() }}
         with:
           path: |

.github/workflows/docs-latest-public-wrapper.yaml

@@ -3,6 +3,10 @@ name: "[post-commit] Docs build and deploy to GitHub pages on main"
 on:
   workflow_dispatch:
 
+permissions:
+  id-token: write
+  pages: write
+
 jobs:
   build-docker-artifact:
     uses: ./.github/workflows/build-docker-artifact.yaml

.github/workflows/docs-latest-public.yaml

@@ -44,6 +44,7 @@ jobs:
         with:
           os: ubuntu-20.04
       - uses: actions/download-artifact@v4
+        timeout-minutes: 10
         with:
           name: TTMetal_build_any
       - name: Extract files

.github/workflows/fabric-build-and-unit-tests-wrapper.yaml

@@ -2,6 +2,8 @@ name: "[post-commit] Fabric unit tests"
 
 on:
   workflow_dispatch:
+  schedule:
+    - cron: "0 * * * *" # Run every hour
 
 jobs:
   build-artifact:
@@ -15,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         test-group: [
-          { arch: wormhole_b0, runner-label: N300 },
+          { arch: wormhole_b0, runner-label: tt-ubuntu-2204-n300-large-stable },
         ]
     uses: ./.github/workflows/fabric-build-and-unit-tests.yaml
     with: