feat: autotest wf

.github/workflows/TestGPT_BPE.yml

@@ -0,0 +1,43 @@
+name: Test gpt_bpe_test.go
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
+      - name: Install dependencies
+        run: go get github.com/wbrown/gpt_bpe/resources
+
+      - name: Build
+        run: go build
+
+      - name: Test
+        run: go test -json -race -shuffle=on -count=1 -v github.com/wbrown/gpt_bpe >> test_results.json
+
+      - name: Output to Markdown
+        run: |
+          jq -r ' \
+          . as $line | \
+          [$line.Time, $line.Action, $line.Package, ($line.Test // "N/A"), ($line.Output // "N/A")] | \
+          @csv' test_results.json | \
+          sed 's/,/|/g; s/^/|/; s/$/|/'| \
+          sed 's/|\|"|/|/' | \
+          sed 's/^|"|$//g' | \
+          sed '/^[[:space:]]*$/d' | \
+          awk 'BEGIN {print "| Time | Action | Package | Test | Output |"; print "|------|--------|---------|------|--------|"} {print $0}' > output.md
+
+      - name: Output to Job Summary
+        run: |
+          echo "## Test Results" >> $GITHUB_STEP_SUMMARY
+          cat output.md >> $GITHUB_STEP_SUMMARY

gpt_bpe_test.go

@@ -1483,7 +1483,7 @@ func TestModelDownloadLlama(t *testing.T) {
 	// Check that the model files are there
 	// We want to check for the presence of the following files:
 	// config.json, pytorch_model.bin,
-	// tokenizer.json, vocab.json
+	// tokenizer.model, vocab.json
 
 	// Check for pytorch_model.bin
 	singleModelPattern := regexp.MustCompile(`pytorch_model\.bin$`)
@@ -1518,7 +1518,7 @@ func TestModelDownloadLlama(t *testing.T) {
 	}
 
 	// Check for additional metadata files
-	metaFiles := []string{"tokenizer.json", "vocab.json", "config.json"}
+	metaFiles := []string{"tokenizer.model", "vocab.json", "config.json"}
 	for _, metaFile := range metaFiles {
 		metaPath := destPath + "/" + metaFile
 		assertFileExists(t, metaPath)
@@ -1542,10 +1542,10 @@ func TestModelDownloadMistral(t *testing.T) {
 	// Check that the model files are there
 	// We want to check for the presence of the following files:
 	// config.json, pytorch_model.bin,
-	// tokenizer.json, vocab.json
+	// tokenizer.model
 
 	// Check for additional metadata files
-	metaFiles := []string{"tokenizer.json", "vocab.json, config.json", "pytorch_model-00001-of-00002.bin"}
+	metaFiles := []string{"tokenizer.model", "config.json", "pytorch_model-00001-of-00002.bin"}
 	for _, metaFile := range metaFiles {
 		metaPath := destPath + "/" + metaFile
 		assertFileExists(t, metaPath)
@@ -1574,7 +1574,7 @@ func TestModelDownloadFairseq(t *testing.T) {
 	// vocab, config. merges, pytorch_model
 
 	// Check for additional metadata files
-	metaFiles := []string{"tokenizer.json", "vocab.json, config.json", "pytorch_model.bin", "merges.txt"}
+	metaFiles := []string{"vocab.json, config.json", "pytorch_model.bin", "merges.txt"}
 	for _, metaFile := range metaFiles {
 		metaPath := destPath + "/" + metaFile
 		assertFileExists(t, metaPath)