CI: Split test wf to 2 parallel wfs

.github/workflows/TestCommands.yml

@@ -0,0 +1,29 @@
+name: Test Commands
+
+on:
+  push:
+    paths-ignore:
+      - "README.md"
+      - "LICENSE"
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+
+      - name: Build
+        run: go build -v ./
+
+      - name: Test dataset_tokenizer
+        uses: robherley/go-test-action@v0
+        with:
+          testArguments: ./
+          moduleDirectory: ./cmd/dataset_tokenizer

.github/workflows/TestGPT_BPE.yml

@@ -5,6 +5,7 @@ on:
     paths-ignore:
       - "README.md"
       - "LICENSE"
+      - "cmd/**"
   workflow_dispatch:
 
 jobs:
@@ -13,41 +14,16 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
 
       - name: Build
         run: go build -v ./
 
-      - name: Check for changes to code
-        id: check_changes_repo
-        run: |
-          changed_files=$(git diff --name-only ${{ github.event.before }})
-          if echo "$changed_files" | grep -q '^cmd/dataset_tokenizer/'; then
-            echo "Dataset Tokenizer files have changed"
-            echo "run_dataset_tests=true" >> $GITHUB_ENV
-          fi
-
-          # Other changes
-          if  echo "$changed_files" | grep -qv '^cmd/'; then
-            echo "Other files have changed"
-            echo "run_other_tests=true" >> $GITHUB_ENV
-          fi
-
       - name: Test gpt_bpe
-        if: env.run_other_tests == 'true'
-        uses: robherley/go-test-action@v0
-        with:
-          testArguments: ./
-
-      - name: Test dataset_tokenizer
-        if: env.run_dataset_tests == 'true'
         uses: robherley/go-test-action@v0
         with:
           testArguments: ./
-          moduleDirectory: ./cmd/dataset_tokenizer

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: Publish
 
 on:
   workflow_run:
-    workflows: ["Test gpt_bpe_test.go"]
+    workflows: ["Test Commands"]
     types:
       - completed