Merge branch 'master' into patch-25

mlcommons · Feb 7, 2025 · 80b084c · 80b084c
2 parents 9da134e + e1d1a4f
commit 80b084c
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 16 deletions.
diff --git a/.github/workflows/auto-update-dev.yml b/.github/workflows/auto-update-dev.yml
@@ -1,21 +1,25 @@
-name: Auto-Update Dev Branch from Master
+name: Auto-Update Dev Branches from Master
 
 on:
   push:
     branches:
       - master  # Trigger workflow on commits to 'dev' branch
 
 jobs:
-  update-main:
+  update-dev:
+    strategy:
+      matrix: 
+        branch: [ "dev", "docs" ]
     runs-on: ubuntu-latest
+
     permissions:
       contents: write  # Required to push to protected branches
 
     steps:
       - name: Checkout Main Branch
         uses: actions/checkout@v4
         with:
-          ref: dev
+          ref: ${{ matrix.branch }}
           fetch-depth: 0
           ssh-key: ${{ secrets.DEPLOY_KEY }}
 
@@ -24,11 +28,12 @@ jobs:
           git config user.name "github-actions"
           git config user.email "github-actions@github.com"
 
-      - name: Merge auto-update into dev
+      - name: Merge auto-update into ${{ matrix.branch }}
         run: |
           git fetch origin master:master
           git merge --no-ff master -m "Auto-merge updates from master branch"
 
-      - name: Push Changes to Main
+      - name: Push Changes to ${{ matrix.branch }}
         run: |
-          git push origin dev
+          git push origin ${{ matrix.branch }}
+     
diff --git a/.github/workflows/test-submission-checker.yml b/.github/workflows/test-submission-checker.yml
@@ -31,8 +31,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cm4mlops
+        python3 -m pip install mlc-scripts
         git clone https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --depth 1
     - name: Test MLPerf inference submission checker
       run: |
-        cm run script --tags=run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/mlperf_inference_unofficial_submissions_v5.0 --src_version=v5.0 --extra_args=" --skip-extra-files-in-root-check --skip-extra-accuracy-files-check" --quiet 
+        mlcr run,mlperf,inference,submission,checker --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --input=`pwd`/mlperf_inference_unofficial_submissions_v5.0 --src_version=v5.0 --extra_args=" --skip-extra-files-in-root-check --skip-extra-accuracy-files-check" --quiet 
diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py
@@ -735,19 +735,19 @@
 ACC_PATTERN = {
     "acc": r"^(?:\{\"accuracy|accuracy)[\": ]*=?\s*([\d\.]+).*",
     "AUC": r"^AUC=([\d\.]+).*",
-    "mAP": r".*'(?:mAP|Total)':\s*([\d\.]+)",
+    "mAP": r".*(?:mAP=|'Total':)\s*([\d.]+)",
     "bleu": r"^BLEU\:\s*([\d\.]+).*",
     "F1": r"^{[\"\']exact_match[\"\']\:\s*[\d\.]+,\s*[\"\']f1[\"\']\:\s*([\d\.]+)}",
     "WER": r"Word Error Rate\:.*, accuracy=([0-9\.]+)%",
     "DICE": r"Accuracy\:\s*mean\s*=\s*([\d\.]+).*",
-    "ROUGE1": r".*'rouge1':\s([\d.]+).*",
-    "ROUGE2": r".*'rouge2':\s([\d.]+).*",
-    "ROUGEL": r".*'rougeL':\s([\d.]+).*",
+    "ROUGE1": r".*'rouge1':\s+'?([\d.]+)'?.*",
+    "ROUGE2": r".*'rouge2':\s+'?([\d.]+)'?.*",
+    "ROUGEL": r".*'rougeL':\s+'?([\d.]+)'?.*",
     "ROUGELSUM": r".*'rougeLsum':\s([\d.]+).*",
     "GEN_LEN": r".*'gen_len':\s([\d.]+).*",
     "TOKENS_PER_SAMPLE": r".*'tokens_per_sample':\s([\d.]+).*",
-    "CLIP_SCORE": r".*'CLIP_SCORE':\s([\d.]+).*",
-    "FID_SCORE": r".*'FID_SCORE':\s([\d.]+).*",
+    "CLIP_SCORE": r".*'CLIP_SCORE':\s+'?([\d.]+).*",
+    "FID_SCORE": r".*'FID_SCORE':\s+'?([\d.]+).*",
     "gsm8k_accuracy": r".*'gsm8k':\s([\d.]+).*",
     "mbxp_accuracy": r".*'mbxp':\s([\d.]+).*",
     "exact_match": r".*'exact_match':\s([\d.]+).*"
@@ -2407,7 +2407,7 @@ def log_result(
                                 .replace('"', "")
                                 .replace("{", "")
                                 .replace("}", "")
-                            )
+                            ).strip()
                             if mlperf_model in REQUIRED_ACC_BENCHMARK:
                                 if (
                                         config.version
@@ -2425,7 +2425,7 @@ def log_result(
                                             missing_files,
                                         )
                                         accuracy_is_valid = False
-                            if not accuracy_is_valid and not is_closed_or_network:
+                            if not accuracy_is_valid and acc and not is_closed_or_network:
                                 if debug:
                                     log.warning(
                                         "%s, accuracy not valid but taken for open",