From 763b9ac9e212f3cb3d5ed42cd8f3e2076b6770e9 Mon Sep 17 00:00:00 2001
From: rexwang8 <rexkingsbackyard@gmail.com>
Date: Fri, 18 Oct 2024 11:02:27 -0400
Subject: [PATCH] CI: Test Dataset Tokenizer Selectively

---
 .github/workflows/TestGPT_BPE.yml | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/TestGPT_BPE.yml b/.github/workflows/TestGPT_BPE.yml
index 89e4aad..9172bf1 100644
--- a/.github/workflows/TestGPT_BPE.yml
+++ b/.github/workflows/TestGPT_BPE.yml
@@ -2,6 +2,9 @@ name: Test gpt_bpe_test.go
 
 on:
   push:
+    paths-ignore:
+      - "README.md"
+      - "LICENSE"
   workflow_dispatch:
 
 jobs:
@@ -19,7 +22,30 @@ jobs:
       - name: Build
         run: go build -v ./
 
-      - name: Test
+      - name: Check for changes to code
+        id: check_changes_repo
+        run: |
+          changed_files=$(git diff --name-only ${{ github.event.before }})
+          if echo "$changed_files" | grep -q '^cmd/dataset_tokenizer/'; then
+            echo "Dataset Tokenizer files have changed"
+            echo "run_dataset_tests=true" >> $GITHUB_ENV
+          fi
+
+          # Other changes
+          if  echo "$changed_files" | grep -qv '^cmd/'; then
+            echo "Other files have changed"
+            echo "run_other_tests=true" >> $GITHUB_ENV
+          fi
+
+      - name: Test gpt_bpe
+        if: env.run_other_tests == 'true'
+        uses: robherley/go-test-action@v0
+        with:
+          testArguments: ./
+
+      - name: Test dataset_tokenizer
+        if: env.run_dataset_tests == 'true'
         uses: robherley/go-test-action@v0
         with:
           testArguments: ./
+          moduleDirectory: ./cmd/dataset_tokenizer