From 628cdc755c58440d439cce8324a984d502f59fae Mon Sep 17 00:00:00 2001
From: rexwang8 <rexkingsbackyard@gmail.com>
Date: Wed, 16 Oct 2024 15:10:36 -0400
Subject: [PATCH] feat: autotest wf

---
 .github/workflows/TestGPT_BPE.yml | 43 +++++++++++++++++++++++++++++++
 gpt_bpe_test.go                   | 10 +++----
 2 files changed, 48 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/TestGPT_BPE.yml

diff --git a/.github/workflows/TestGPT_BPE.yml b/.github/workflows/TestGPT_BPE.yml
new file mode 100644
index 0000000..bf5d75d
--- /dev/null
+++ b/.github/workflows/TestGPT_BPE.yml
@@ -0,0 +1,43 @@
+name: Test gpt_bpe_test.go
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
+      - name: Install dependencies
+        run: go get github.com/wbrown/gpt_bpe/resources
+
+      - name: Build
+        run: go build
+
+      - name: Test
+        run: go test -json -race -shuffle=on -count=1 -v github.com/wbrown/gpt_bpe >> test_results.json
+
+      - name: Output to Markdown
+        run: |
+          jq -r ' \
+          . as $line | \
+          [$line.Time, $line.Action, $line.Package, ($line.Test // "N/A"), ($line.Output // "N/A")] | \
+          @csv' test_results.json | \
+          sed 's/,/|/g; s/^/|/; s/$/|/'| \
+          sed 's/|\|"|/|/' | \
+          sed 's/^|"|$//g' | \
+          sed '/^[[:space:]]*$/d' | \
+          awk 'BEGIN {print "| Time | Action | Package | Test | Output |"; print "|------|--------|---------|------|--------|"} {print $0}' > output.md
+
+      - name: Output to Job Summary
+        run: |
+          echo "## Test Results" >> $GITHUB_STEP_SUMMARY
+          cat output.md >> $GITHUB_STEP_SUMMARY
diff --git a/gpt_bpe_test.go b/gpt_bpe_test.go
index 88ef7b0..5e3be07 100644
--- a/gpt_bpe_test.go
+++ b/gpt_bpe_test.go
@@ -1483,7 +1483,7 @@ func TestModelDownloadLlama(t *testing.T) {
 	// Check that the model files are there
 	// We want to check for the presence of the following files:
 	// config.json, pytorch_model.bin,
-	// tokenizer.json, vocab.json
+	// tokenizer.model, vocab.json
 
 	// Check for pytorch_model.bin
 	singleModelPattern := regexp.MustCompile(`pytorch_model\.bin$`)
@@ -1518,7 +1518,7 @@ func TestModelDownloadLlama(t *testing.T) {
 	}
 
 	// Check for additional metadata files
-	metaFiles := []string{"tokenizer.json", "vocab.json", "config.json"}
+	metaFiles := []string{"tokenizer.model", "vocab.json", "config.json"}
 	for _, metaFile := range metaFiles {
 		metaPath := destPath + "/" + metaFile
 		assertFileExists(t, metaPath)
@@ -1542,10 +1542,10 @@ func TestModelDownloadMistral(t *testing.T) {
 	// Check that the model files are there
 	// We want to check for the presence of the following files:
 	// config.json, pytorch_model.bin,
-	// tokenizer.json, vocab.json
+	// tokenizer.model
 
 	// Check for additional metadata files
-	metaFiles := []string{"tokenizer.json", "vocab.json, config.json", "pytorch_model-00001-of-00002.bin"}
+	metaFiles := []string{"tokenizer.model", "config.json", "pytorch_model-00001-of-00002.bin"}
 	for _, metaFile := range metaFiles {
 		metaPath := destPath + "/" + metaFile
 		assertFileExists(t, metaPath)
@@ -1574,7 +1574,7 @@ func TestModelDownloadFairseq(t *testing.T) {
 	// vocab, config. merges, pytorch_model
 
 	// Check for additional metadata files
-	metaFiles := []string{"tokenizer.json", "vocab.json, config.json", "pytorch_model.bin", "merges.txt"}
+	metaFiles := []string{"vocab.json, config.json", "pytorch_model.bin", "merges.txt"}
 	for _, metaFile := range metaFiles {
 		metaPath := destPath + "/" + metaFile
 		assertFileExists(t, metaPath)