Versioning issue with BDN test files, fixed now

.github/workflows/ci-bdnbenchmark.yml

@@ -6,13 +6,15 @@ env:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
   DOTNET_NOLOGO: true
 
+permissions:
+  actions: write
+  deployments: write  #permission to deploy GitHub pages website
+  contents: write  # permission to update benchmark contents in gh-pages branch
+
 jobs:
   changes:
     name: Check for changes
     runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code
-    permissions:
-      pull-requests: read
-      contents: read
     steps:
     - name: Check out code
       uses: actions/checkout@v4
@@ -34,12 +36,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        #ORIGINALos: [ ubuntu-latest, windows-latest ]
-        os: [ windows-latest ]
+        os: [ ubuntu-latest, windows-latest ]
         framework: [ 'net8.0' ]
         configuration: [ 'Release' ]
-        #ORIGINAL test: [ 'CI_BDN_Config_RespParseStress', 'CI_BDN_Config_RespTsavoriteStress', 'CI_BDN_Config_RespClusterBench', 'CI_BDN_Config_RespClusterMigrateBench', 'CI_BDN_Config_Lua' ]
-        test: [ 'CI_BDN_Config_RespParseStress' ]
+        test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ]   
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -53,6 +53,7 @@ jobs:
       - name: Run BDN.benchmark Perf Test
         run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }}
         shell: pwsh
+        continue-on-error: true
 
       - name: Upload test results to artifacts
         uses: actions/upload-artifact@v4
@@ -61,6 +62,7 @@ jobs:
           path: |
              ./test/BDNPerfTests/results
         if: ${{ always() }}
+
       - name: Upload Error Log to artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -69,36 +71,29 @@ jobs:
             ./test/BDNPerfTests/errorlog
         if: ${{ always() }}
 
-      # Download previous benchmark result from cache (if exists)
-      - name: Download previous benchmark data
-        uses: actions/cache@v4
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
-
-      # Run `github-action-benchmark` action
-      - name: Store benchmark result
+      # Run `github-action-benchmark` action to get Commit Comment
+      - name: Store benchmark result for commit comment
         uses: benchmark-action/github-action-benchmark@v1
         with:
-          # What benchmark tool created output file
           tool: 'benchmarkdotnet'
-          # Where the output from the benchmark tool is stored
-          # This can't find the file
-          #output-file-path: ./test/BDNPerfTests/results/Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}.results
-
-          # This gives error: Error: Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: Unexpected token '/', "// Validat"... is not valid JSON
-          #output-file-path: ./test/BDNPerfTests/results/CI_BDN_Config_RespParseStress_Windows.results
-
-          # Error: Command 'git' failed with args '-c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com -c http.https://github.com/.extraheader= switch gh-pages': fatal: invalid reference: gh-pages
-          output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.Resp.RespParseStress-report-full-compressed.json
-
-          # Where the previous data file is stored
-          external-data-json-path: ./cache/benchmark-data.json
-          # Workflow will fail when an alert happens
+          output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: '130%' 
+          comment-on-alert: true
           fail-on-alert: true
+          alert-comment-cc-users: '@darrenge'
+          summary-always: false
+          auto-push: true
+
+      # Run `github-action-benchmark` action for the Continuous Benchmarking Charts
+      - name: Store benchmark result for charts
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: ${{matrix.test}} (${{matrix.os}}  ${{matrix.framework}} ${{matrix.configuration}})
+          tool: 'benchmarkdotnet'
+          output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          summary-always: true
+          auto-push: true
 
-          # Take these out as not part of minimal
-          # github-token: ${{ secrets.GITHUB_TOKEN }}
-          # auto-push: false
-          # skip-fetch-gh-pages: true
-          # summary-always: true
+

test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json → test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json

@@ -1,7 +1,4 @@
 {
-    "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES",
-    "ExpectedCoresToTestOn_win": 2,
-    "ExpectedCoresToTestOn_linux": 2,
     "configuration": "Release",
     "framework": "net8.0",
     "filter": "*RespClusterBench*",
@@ -12,8 +9,9 @@
     "expectedGETMeanValue_win": 24,
     "expectedSETMeanValue_win": 24,
     "expectedMGETMeanValue_win": 19,
-    "expectedMSETMeanValue_win": 17,
-    "acceptableMeanRange": 10,
+    "expectedMSETMeanValue_win": 19,
+
+    "acceptableMeanRange": 15,
     "acceptableError": ".1",
     "acceptableStdDev": ".1"
 }

test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json → test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json

@@ -1,7 +1,4 @@
 {
-    "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES",
-    "ExpectedCoresToTestOn_win": 2,
-    "ExpectedCoresToTestOn_linux": 2,
     "configuration": "Release",
     "framework": "net8.0",
     "filter": "*RespClusterMigrateBench*",
@@ -13,7 +10,8 @@
     "expectedSETMeanValue_win": 47,
     "expectedMGETMeanValue_win": 40,
     "expectedMSETMeanValue_win": 37,
-    "acceptableMeanRange": 10,
+
+    "acceptableMeanRange": 15,
     "acceptableError": ".1",
     "acceptableStdDev": ".1"
 }

test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json

@@ -0,0 +1,30 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "*LuaRunner*",
+    "expectedBasicLuaRunner1MeanValue_linux": 182,
+    "expectedBasicLuaRunner2MeanValue_linux": 325,
+    "expectedBasicLuaRunner3MeanValue_linux": 1473,
+    "expectedBasicLuaRunner4MeanValue_linux": 3754,
+
+    "expectedBasicLuaRunner1MeanValue_win": 115,
+    "expectedBasicLuaRunner2MeanValue_win": 180,
+    "expectedBasicLuaRunner3MeanValue_win": 686,
+    "expectedBasicLuaRunner4MeanValue_win": 2711,
+
+
+    "expectedBasicLuaRunner1AllocatedValue_linux": 24,
+    "expectedBasicLuaRunner2AllocatedValue_linux": 144,
+    "expectedBasicLuaRunner3AllocatedValue_linux": 240,
+    "expectedBasicLuaRunner4AllocatedValue_linux": 776,
+
+    "expectedBasicLuaRunner1AllocatedValue_win": 24,
+    "expectedBasicLuaRunner2AllocatedValue_win": 144,
+    "expectedBasicLuaRunner3AllocatedValue_win": 240,
+    "expectedBasicLuaRunner4AllocatedValue_win": 776,
+
+    "acceptableMeanRange": 15,
+    "acceptableAllocatedRange": 10,    
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json

@@ -0,0 +1,30 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "*LuaStress*",
+
+    "expectedBasicLuaStress1MeanValue_linux": 175,
+    "expectedBasicLuaStress2MeanValue_linux": 284,
+    "expectedBasicLuaStress3MeanValue_linux": 1407,
+    "expectedBasicLuaStress4MeanValue_linux": 2733,
+
+    "expectedBasicLuaStress1AllocatedValue_linux": 24,
+    "expectedBasicLuaStress2AllocatedValue_linux": 144,
+    "expectedBasicLuaStress3AllocatedValue_linux": 328,
+    "expectedBasicLuaStress4AllocatedValue_linux": 832,
+
+    "expectedBasicLuaStress1MeanValue_win": 107,
+    "expectedBasicLuaStress2MeanValue_win": 180,
+    "expectedBasicLuaStress3MeanValue_win": 660,
+    "expectedBasicLuaStress4MeanValue_win": 1805,
+
+    "expectedBasicLuaStress1AllocatedValue_win": 24,
+    "expectedBasicLuaStress2AllocatedValue_win": 144,
+    "expectedBasicLuaStress3AllocatedValue_win": 328,
+    "expectedBasicLuaStress4AllocatedValue_win": 832,
+
+    "acceptableMeanRange": 15,
+    "acceptableAllocatedRange": 10,    
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json

@@ -0,0 +1,43 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "*RespParseStress*",
+    "expectedInLinePingMeanValue_linux": 2.2,
+    "expectedSETMeanValue_linux": 19,
+    "expectedSETEXMeanValue_linux": 26,
+    "expectedGETMeanValue_linux": 13,
+    "expectedIncrementMeanValue_linux": 27,
+    "expectedZAddRemMeanValue_linux": 186,
+    "expectedLPushPopMeanValue_linux": 166,
+    "expectedSAddRemMeanValue_linux": 164,
+    "expectedHSetDelMeanValue_linux": 196,
+    "expectedMyDictSetGetMeanValue_linux": 248,
+
+    "expectedZAddRemAllocatedValue_linux": 23552,
+    "expectedLPushPopAllocatedValue_linux": 18432,
+    "expectedSAddRemAllocatedValue_linux": 16384,
+    "expectedHSetDelAllocatedValue_linux": 55297,
+    "expectedMyDictSetGetAllocatedValue_linux": 30720,
+
+    "expectedInLinePingMeanValue_win": 2.2,
+    "expectedSETMeanValue_win": 18,
+    "expectedSETEXMeanValue_win": 25,
+    "expectedGETMeanValue_win": 14,
+    "expectedIncrementMeanValue_win": 24,
+    "expectedZAddRemMeanValue_win": 144,
+    "expectedLPushPopMeanValue_win": 128,
+    "expectedSAddRemMeanValue_win": 124,
+    "expectedHSetDelMeanValue_win": 169,
+    "expectedMyDictSetGetMeanValue_win": 250,
+
+    "expectedZAddRemAllocatedValue_win": 23552,
+    "expectedLPushPopAllocatedValue_win": 18432,
+    "expectedSAddRemAllocatedValue_win": 16384,
+    "expectedHSetDelAllocatedValue_win": 55297,
+    "expectedMyDictSetGetAllocatedValue_win": 30720,
+
+    "acceptableMeanRange": 15,
+    "acceptableAllocatedRange": 10,    
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json

@@ -0,0 +1,16 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "*RespTsavoriteStress*",
+    "expectedGETMeanValue_linux": 21,
+    "expectedSETMeanValue_linux": 19,
+    "expectedIncrementMeanValue_linux": 27,
+    "expectedGETMeanValue_win": 20,
+    "expectedSETMeanValue_win": 17,
+    "expectedIncrementMeanValue_win": 26,
+
+    "acceptableMeanRange": 15,
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}
+