From a81a31ebe70098278b3d9cb551c4bd7d13c2a93d Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Sep 2024 13:22:27 -0700 Subject: [PATCH 01/44] The CI scripts, YML and config files to check perf using BDN benchmarking. Also, a fix in ClusterBench for and renamed Lua Runner benchmarking --- .github/workflows/ci-bdnbenchmark.yml | 44 ++ .../BDN.benchmark/Cluster/RespClusterBench.cs | 2 + .../BDN.benchmark/Resp/RespLuaRunnerStress.cs | 8 +- .../ConfigFiles/CI_BDN_Config_Lua.json | 27 ++ .../CI_BDN_Config_RespClusterBench.json | 19 + ...CI_BDN_Config_RespClusterMigrateBench.json | 19 + .../CI_BDN_Config_RespParseStress.json | 29 ++ .../CI_BDN_Config_RespTsavoriteStress.json | 18 + test/BDNPerfTests/run_bdnperftest.ps1 | 440 ++++++++++++++++++ 9 files changed, 602 insertions(+), 4 deletions(-) create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json create mode 100644 test/BDNPerfTests/run_bdnperftest.ps1 diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index faa2d90598..2511c36c55 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -25,3 +25,47 @@ jobs: - 'website/**' bdnbenchmark: - '!((*.md)|(website/**))' + + # Job to build and run BDN.benchmark + build-run-bdnbenchmark: + name: BDNBenchmark + needs: [changes] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest ] + framework: [ 'net8.0' ] + configuration: [ 'Release' ] + test: [ 'CI_BDN_Config_RespParseStress.json', 'CI_BDN_Config_RespTsavoriteStress.json', 'CI_BDN_Config_RespClusterBench.json', 'CI_BDN_Config_RespClusterMigrateBench.json', 'CI_BDN_Config_Lua.json' ] + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.0.x + - name: Install dependencies + run: dotnet restore + - name: Build Garnet + run: dotnet build --configuration ${{ matrix.configuration }} + + - name: Run BDN.benchmark Perf Test + run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }} + shell: pwsh + + - name: Upload test results to artifacts + uses: actions/upload-artifact@v4 + with: + name: ResultsDir-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + path: | + ./test/BDNPerfTests/results + if: ${{ always() }} + - name: Upload Error Log to artifacts + uses: actions/upload-artifact@v4 + with: + name: ErrorLogDir-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + path: | + ./test/BDNPerfTests/errorlog + if: ${{ always() }} + diff --git a/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs b/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs index 02b04dc1e5..e9af4ba3f2 100644 --- a/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs +++ b/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using BenchmarkDotNet.Attributes; using Embedded.perftest; @@ -48,6 +49,7 @@ public void SetupSingleInstance() EnableCluster = true, Port = Port, CleanClusterConfig = true, + CheckpointDir = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "/tmp" : null }; server = new EmbeddedRespServer(opt); session = server.GetRespSession(); diff --git a/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs b/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs index 006b9f02f0..866dedddda 100644 --- a/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs +++ b/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs @@ -35,19 +35,19 @@ public void GlobalCleanup() } [Benchmark] - public void BasicLua1() + public void BasicLuaRunner1() => r1.RunVoid(); [Benchmark] - public void BasicLua2() + public void BasicLuaRunner2() => r2.Run(); [Benchmark] - public void BasicLua3() + public void BasicLuaRunner3() => r3.Run(keys, null); [Benchmark] - public void BasicLua4() + public void BasicLuaRunner4() => r4.Run(keys, null); } } \ No newline at end of file diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json new file mode 100644 index 0000000000..34d67ebcda --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json @@ -0,0 +1,27 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*Lua*", + "expectedBasicLuaRunner1MeanValue_linux": 186, + "expectedBasicLuaRunner2MeanValue_linux": 305, + "expectedBasicLuaRunner3MeanValue_linux": 1473, + "expectedBasicLuaRunner4MeanValue_linux": 3754, + "expectedBasicLuaRunner1MeanValue_win": 140, + "expectedBasicLuaRunner2MeanValue_win": 173, + "expectedBasicLuaRunner3MeanValue_win": 686, + "expectedBasicLua4RunnerMeanValue_win": 2711, + "expectedBasicLua1MeanValue_linux": 169, + "expectedBasicLua2MeanValue_linux": 284, + "expectedBasicLua3MeanValue_linux": 1407, + "expectedBasicLua4MeanValue_linux": 2733, + "expectedBasicLua1MeanValue_win": 107, + "expectedBasicLua2MeanValue_win": 180, + "expectedBasicLua3MeanValue_win": 660, + "expectedBasicLua4MeanValue_win": 1805, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json new file mode 100644 index 0000000000..4b38979781 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json @@ -0,0 +1,19 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespClusterBench*", + "expectedGETMeanValue_linux": 27, + "expectedSETMeanValue_linux": 25, + "expectedMGETMeanValue_linux": 20, + "expectedMSETMeanValue_linux": 20, + "expectedGETMeanValue_win": 24, + "expectedSETMeanValue_win": 24, + "expectedMGETMeanValue_win": 19, + "expectedMSETMeanValue_win": 17, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json new file mode 100644 index 0000000000..5f4387832b --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json @@ -0,0 +1,19 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespClusterMigrateBench*", + "expectedGETMeanValue_linux": 46, + "expectedSETMeanValue_linux": 49, + "expectedMGETMeanValue_linux": 40, + "expectedMSETMeanValue_linux": 39, + "expectedGETMeanValue_win": 46, + "expectedSETMeanValue_win": 47, + "expectedMGETMeanValue_win": 40, + "expectedMSETMeanValue_win": 37, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json new file mode 100644 index 0000000000..a7199eecc6 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json @@ -0,0 +1,29 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespParseStress*", + "expectedInLinePingMeanValue_linux": 2.4, + "expectedSETMeanValue_linux": 19, + "expectedSETEXMeanValue_linux": 26, + "expectedGETMeanValue_linux": 14, + "expectedZAddRemMeanValue_linux": 186, + "expectedLPushPopMeanValue_linux": 200, + "expectedSAddRemMeanValue_linux": 172, + "expectedHSetDelMeanValue_linux": 200, + "expectedMyDictSetGetMeanValue_linux": 235, + "expectedInLinePingMeanValue_win": 2.2, + "expectedSETMeanValue_win": 19, + "expectedSETEXMeanValue_win": 26, + "expectedGETMeanValue_win": 14, + "expectedZAddRemMeanValue_win": 141, + "expectedLPushPopMeanValue_win": 145, + "expectedSAddRemMeanValue_win": 118, + "expectedHSetDelMeanValue_win": 160, + "expectedMyDictSetGetMeanValue_win": 242, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json new file mode 100644 index 0000000000..ff0c0333c1 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json @@ -0,0 +1,18 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespTsavoriteStress*", + "expectedGETMeanValue_linux": 21, + "expectedSETMeanValue_linux": 19, + "expectedIncrMeanValue_linux": 29, + "expectedGETMeanValue_win": 20, + "expectedSETMeanValue_win": 17, + "expectedIncrMeanValue_win": 26, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} + diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 new file mode 100644 index 0000000000..fea41aec92 --- /dev/null +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -0,0 +1,440 @@ +<#$f +.SYNOPSIS + This script is designed for to run BDN Benchmarking and use the results as a gate in the GitHub CIs to ensure performance isn't declining + +.DESCRIPTION + + Script to test for performance regressions using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. + + CI_BDN_Config_RespParseStress.json + + NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. + NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% + +.EXAMPLE + ./run_bdnperftest.ps1 + ./run_bdnperftest.ps1 CI_BDN_Config_RespParseStress.json +#> + + +# Send the config file for the benchmark. Defaults to a simple one +param ( + [string]$configFile = "CI_BDN_Config_RespParseStress.json" +) + +$OFS = "`r`n" + +# ******** FUNCTION DEFINITIONS ********* + +################## AnalyzeResult ##################### +# +# Takes the result and verifies it falls in the acceptable range (tolerance) based on the percentage. +# +###################################################### +function AnalyzeResult { + param ($foundResultValue, $expectedResultValue, $acceptablePercentRange, $warnonly) + + # Calculate the lower and upper bounds of the expected value + [double] $Tolerance = $acceptablePercentRange / 100 + [double] $LowerBound = $expectedResultValue * (1 - $Tolerance) + [double] $UpperBound = $expectedResultValue * (1 + $Tolerance) + [double] $dblfoundResultValue = $foundResultValue + + # Check if the actual value is within the bounds + if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { + Write-Host "** ** PASS! ** Mean Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** " + return $true # the values are close enough + } + else { + if ($warnonly) { + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Mean Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** " + } + else { + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Mean Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** " + } + return $false # the values are too different + } + } + +######### ParseValueFromResults ########### +# +# Takes the line from the results file and returns the value from the requested column +# strips off all the characters and just to return the actual value +# +# NOTE: Example of ResultsLine from BDN benchmark: "| InlinePing | 2.343 us | 0.0135 us | 0.0113 us | - |" +# NOTE: columnNum is zero based +# +###################################################### +function ParseValueFromResults { +param ($ResultsLine, $columnNum) + + # Remove the leading and trailing pipes and split the string by '|' + $columns = $ResultsLine.Trim('|').Split('|') + $column = $columns | ForEach-Object { $_.Trim() } + $foundValue = $column[$columnNum].Trim(' us') + $foundValue = $foundValue.Trim(' ns') + + return $foundValue +} + + + + +# ******** BEGIN MAIN ********* +# Get base path since paths can differ from machine to machine +$pathstring = $pwd.Path +if ($pathstring.Contains("test")) { + $position = $pathString.IndexOf("test") + $basePath = $pathstring.Substring(0,$position-1) # take off slash off end as well +} else { + $basePath = $pathstring # already in base as not in test + Set-Location .\test\BDNPerfTests\ +} + +# Read the test config file and convert the JSON to a PowerShell object +$fullConfiFileAndPath = "ConfigFiles/$configFile" +if (-not (Test-Path -Path $fullConfiFileAndPath)) { + Write-Error -Message "The test config file $fullConfiFileAndPath does not exist." -Category ObjectNotFound + exit +} +$json = Get-Content -Raw $fullConfiFileAndPath +$object = $json | ConvertFrom-Json + +# Use this in the file name to separate outputs when running in ADO +$CurrentOS = "Windows" +if ($IsLinux) { + $CurrentOS = "Linux" +} + + # Calculate number of cores on the test machine - used to verify the config file settings as the config settings will vary based on machine config +if ($IsLinux) { + $sockets = [int]$(lscpu | grep -E '^Socket' | awk '{print $2}') + $coresPerSocket = [int]$(lscpu | grep -E '^Core' | awk '{print $4}') + $NumberOfCores = ($sockets * $coresPerSocket) + + $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_linux +} +else { + $NumberOfCores = (Get-CimInstance -ClassName Win32_Processor).NumberOfCores | Measure-Object -Sum | Select-Object -ExpandProperty Sum + + $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_win +} + +# When Remove these, then set up the actual error so it fails out if cores do not match +Write-Host "---- DEBUG --- Number of Cores Found: $NumberOfCores " +Write-Host "---- DEBUG --- Expected Number of Cores: $ExpectedCoresToTestOn " + +# To get accurate comparison of found vs expected values, double check to make sure config settings for Number of Cores of the test machine are what is specified in the test config file +if ($ExpectedCoresToTestOn -ne $NumberOfCores) { +# Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." +# exit + Write-Host "WARNING!!!! The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." +} + +Write-Host "************** Start BDN.benchmark ********************" +Write-Host " " + +# Set all the config options (args to benchmark app) based on values from config json file +$configuration = $object.configuration +$framework = $object.framework +$filter = $object.filter +$meanColumn = "1" + +# Set the expected values based on the OS +if ($IsLinux) { + # Linux expected values + $expectedInLinePingMeanValue = $object.expectedInLinePingMeanValue_linux + $expectedSetMeanValue = $object.expectedSETMeanValue_linux + $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_linux + $expectedGetMeanValue = $object.expectedGETMeanValue_linux + $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_linux + $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_linux + $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_linux + $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_linux + $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_linux + $expectedMGetMeanValue = $object.expectedMGETMeanValue_linux + $expectedMSetMeanValue = $object.expectedMSETMeanValue_linux + $expectedIncrMeanValue = $object.expectedIncrMeanValue_linux + $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_linux + $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_linux + $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_linux + $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_linux + $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_linux + $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_linux + $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux +} +else { + # Windows expected values + $expectedInLinePingMeanValue = $object.expectedInLinePingMeanValue_win + $expectedSetMeanValue = $object.expectedSETMeanValue_win + $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_win + $expectedGetMeanValue = $object.expectedGETMeanValue_win + $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_win + $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_win + $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_win + $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_win + $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_win + $expectedMGetMeanValue = $object.expectedMGETMeanValue_win + $expectedMSetMeanValue = $object.expectedMSETMeanValue_win + $expectedIncrMeanValue = $object.expectedIncrMeanValue_win + $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_win + $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_win + $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_win + $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_win + $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_win + $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_win + $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_win + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_win +} + +# percent allowed variance when comparing expected vs actual found value - same for linux and windows. +$acceptableMeanRange = $object.acceptableMeanRange + +# Set up the results dir and errorlog dir +$resultsDir = "$basePath/test/BDNPerfTests/results" +if (-not (Test-Path -Path $resultsDir)) { + New-Item -Path $resultsDir -ItemType Directory +} +$errorLogDir = "$basePath/test/BDNPerfTests/errorlog" +if (-not (Test-Path -Path $errorLogDir)) { + New-Item -Path $errorLogDir -ItemType Directory +} + +# Run the BDN.benchmark +$BDNbenchmarkPath = "$basePath/benchmark/BDN.benchmark" + +# Create Results and all the log files using the the config file name as part of the name of the results \ logs +$justResultsFileNameNoExt = $configFile -replace ".{5}$" # strip off the .json +$resultsFileName = $justResultsFileNameNoExt + "_" + $CurrentOS + ".results" +$resultsFile = "$resultsDir/$resultsFileName" +$BDNbenchmarkErrorFile = "$errorLogDir/$justResultsFileNameNoExt" + "_StandardError_" +$CurrentOS+".log" + +Write-Output "------------ DEBUG #########################" +Write-Output "*#*#* Configuration $configuration" +Write-Output "*#*#* framework $framework" +Write-Output "*#*#* filter $filter" +Write-Output "*#*#* Standard Out (Results) $resultsFile" +Write-Output "*#*#* Standard Error $BDNbenchmarkErrorFile" +Write-Output "*#*#* Working Dir (BDNBM Path) $BDNbenchmarkPath" +Write-Output "*------------ DEBUG #########################" + +Write-Output "** Start BDN Benchmark: $filter" +Write-Output " " +Write-Output "** Start: dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath > $resultsFile 2> $BDNbenchmarkErrorFile" +dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath > $resultsFile 2> $BDNbenchmarkErrorFile + +Write-Output "** BDN Benchmark for $filter finished" +Write-Output " " + +Write-Output "**** EVALUATE THE RESULTS FILE $resultsFile ****" + +# First check if file is there and if not, error out gracefully +if (-not (Test-Path -Path $resultsFile)) { + Write-Error -Message "The test results file $resultsFile does not exist. Check to make sure the test was ran." -Category ObjectNotFound + exit +} + +# Check see if results file size 0 +$resultsFileSizeBytes = (Get-Item -Path $resultsFile).Length +if ($resultsFileSizeBytes -eq 0) { + Write-Error -Message "The test results file $resultsFile is empty." + exit +} + +Write-Output "************************" +Write-Output "** RESULTS " +Write-Output "** " + +# Set the test suite to pass and if any one fails, then mark the suite as fail - just one result failure will mark the whole test as failed +$testSuiteResult = $true + +# Read the results file line by line and pull out the specific results if exists +Get-Content $resultsFile | ForEach-Object { + $line = $_ + switch -Wildcard ($line) { + "*| InlinePing*" { + Write-Output "** InlinePing Mean Value test" + $foundInLinePingMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundInLinePingMeanValue $expectedInLinePingMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + # This one a bit different as need extra space in the check so doesn't pick up other Set* calls + "*| Set *" { + Write-Host "** Set Mean Value test" + $foundSetMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundSetMeanValue $expectedSetMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| SetEx*" { + Write-Host "** SetEx Mean Value test" + $foundSetExMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundSetExMeanValue $expectedSETEXMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Get*" { + Write-Host "** Get Mean Value test" + $foundGetMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundGetMeanValue $expectedGetMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| ZAddRem*" { + Write-Host "** ZAddRem Mean Value test" + $foundZAddRemMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundZAddRemMeanValue $expectedZAddRemMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| LPushPop*" { + Write-Host "** LPushPop Mean Value test" + $foundLPushPopMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundLPushPopMeanValue $expectedLPushPopMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| SAddRem*" { + Write-Host "** SAddRem Mean Value test" + $foundSAddRemMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundSAddRemMeanValue $expectedSAddRemMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| HSetDel*" { + Write-Host "** HSetDel Mean Value test" + $foundHSetDelMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundHSetDelMeanValue $expectedHSetDelMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MyDictSetGet*" { + Write-Host "** MyDictSetGet Mean Value test" + $foundMyDictSetGetMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundMyDictSetGetMeanValue $expectedMyDictSetGetMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Incr*" { + Write-Host "** Incr Mean Value test" + $foundIncrMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundIncrMeanValue $expectedIncrMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MGet*" { + Write-Host "** MGet Mean Value test" + $foundMGetMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundMGetMeanValue $expectedMGetMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MSet*" { + Write-Host "** MSet Mean Value test" + $foundMSetMeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundMSetMeanValue $expectedMSetMeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLua1*" { + Write-Host "** BasicLua1 Mean Value test" + $foundBasicLua1MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua1MeanValue $expectedBasicLua1MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLua2*" { + Write-Host "** BasicLua2 Mean Value test" + $foundBasicLua2MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua2MeanValue $expectedBasicLua2MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + <# + # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. + "*| BasicLua3*" { + Write-Host "** BasicLua3 Mean Value test" + $foundBasicLua3MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua3MeanValue $expectedBasicLua3MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLua4*" { + Write-Host "** BasicLua4 Mean Value test" + $foundBasicLua4MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua4MeanValue $expectedBasicLua4MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + #> + "*| BasicLuaRunner1*" { + Write-Host "** BasicLuaRunner1 Mean Value test" + $foundBasicLuaRunner1MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner1MeanValue $expectedBasicLuaRunner1MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner2*" { + Write-Host "** BasicLuaRunner2 Mean Value test" + $foundBasicLuaRunner2MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner2MeanValue $expectedBasicLuaRunner2MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + <# + # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. + "*| BasicLuaRunner3*" { + Write-Host "** BasicLuaRunner3 Mean Value test" + $foundBasicLuaRunner3MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner3MeanValue $expectedBasicLuaRunner3MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner4*" { + Write-Host "** BasicLuaRunner4 Mean Value test" + $foundBasicLuaRunner4MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner4MeanValue $expectedBasicLuaRunner4MeanValue $acceptableMeanRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + #> + } +} + +Write-Output "** " +Write-Output "************************" +Write-Output "** Final summary:" +Write-Output "** " +if ($testSuiteResult) { + Write-Output "** PASS! All tests passed " +} else { + Write-Error -Message "** BDN Benchmark PERFORMANCE REGRESSION FAIL! At least one test had benchmark value outside of expected range. NOTE: Expected results are based on CI machine and may differ from the machine that this was ran on." +} +Write-Output "** " +Write-Output "************************" From f66f63f169c3882b741082f103fff9265d600b44 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Sep 2024 13:34:16 -0700 Subject: [PATCH 02/44] Cleaned up the debug message and updated couple expected values --- .../ConfigFiles/CI_BDN_Config_Lua.json | 2 +- .../CI_BDN_Config_RespParseStress.json | 2 +- test/BDNPerfTests/run_bdnperftest.ps1 | 18 ++---------------- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json index 34d67ebcda..d40425bf3a 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json @@ -9,7 +9,7 @@ "expectedBasicLuaRunner2MeanValue_linux": 305, "expectedBasicLuaRunner3MeanValue_linux": 1473, "expectedBasicLuaRunner4MeanValue_linux": 3754, - "expectedBasicLuaRunner1MeanValue_win": 140, + "expectedBasicLuaRunner1MeanValue_win": 111, "expectedBasicLuaRunner2MeanValue_win": 173, "expectedBasicLuaRunner3MeanValue_win": 686, "expectedBasicLua4RunnerMeanValue_win": 2711, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json index a7199eecc6..02ec7867af 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json @@ -14,7 +14,7 @@ "expectedSAddRemMeanValue_linux": 172, "expectedHSetDelMeanValue_linux": 200, "expectedMyDictSetGetMeanValue_linux": 235, - "expectedInLinePingMeanValue_win": 2.2, + "expectedInLinePingMeanValue_win": 2.4, "expectedSETMeanValue_win": 19, "expectedSETEXMeanValue_win": 26, "expectedGETMeanValue_win": 14, diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index fea41aec92..44fccfca3b 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -123,15 +123,10 @@ else { $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_win } -# When Remove these, then set up the actual error so it fails out if cores do not match -Write-Host "---- DEBUG --- Number of Cores Found: $NumberOfCores " -Write-Host "---- DEBUG --- Expected Number of Cores: $ExpectedCoresToTestOn " - # To get accurate comparison of found vs expected values, double check to make sure config settings for Number of Cores of the test machine are what is specified in the test config file if ($ExpectedCoresToTestOn -ne $NumberOfCores) { -# Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." -# exit - Write-Host "WARNING!!!! The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." + Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." + exit } Write-Host "************** Start BDN.benchmark ********************" @@ -213,15 +208,6 @@ $resultsFileName = $justResultsFileNameNoExt + "_" + $CurrentOS + ".results" $resultsFile = "$resultsDir/$resultsFileName" $BDNbenchmarkErrorFile = "$errorLogDir/$justResultsFileNameNoExt" + "_StandardError_" +$CurrentOS+".log" -Write-Output "------------ DEBUG #########################" -Write-Output "*#*#* Configuration $configuration" -Write-Output "*#*#* framework $framework" -Write-Output "*#*#* filter $filter" -Write-Output "*#*#* Standard Out (Results) $resultsFile" -Write-Output "*#*#* Standard Error $BDNbenchmarkErrorFile" -Write-Output "*#*#* Working Dir (BDNBM Path) $BDNbenchmarkPath" -Write-Output "*------------ DEBUG #########################" - Write-Output "** Start BDN Benchmark: $filter" Write-Output " " Write-Output "** Start: dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath > $resultsFile 2> $BDNbenchmarkErrorFile" From b0c8cc4cbb670a755764ea2a4494a9c97db1e698 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Sep 2024 14:35:28 -0700 Subject: [PATCH 03/44] Updated SAddRem - windows - to 130 from 118. Also removed unncesssary build statement in YML --- .github/workflows/ci-bdnbenchmark.yml | 2 -- .../BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 2511c36c55..d47b1e7b87 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -47,8 +47,6 @@ jobs: dotnet-version: 8.0.x - name: Install dependencies run: dotnet restore - - name: Build Garnet - run: dotnet build --configuration ${{ matrix.configuration }} - name: Run BDN.benchmark Perf Test run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json index 02ec7867af..0995827e35 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json @@ -20,7 +20,7 @@ "expectedGETMeanValue_win": 14, "expectedZAddRemMeanValue_win": 141, "expectedLPushPopMeanValue_win": 145, - "expectedSAddRemMeanValue_win": 118, + "expectedSAddRemMeanValue_win": 130, "expectedHSetDelMeanValue_win": 160, "expectedMyDictSetGetMeanValue_win": 242, "acceptableMeanRange": 10, From 2b0ec1ff7777454734d864b918e47b548233f542 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Sep 2024 16:07:25 -0700 Subject: [PATCH 04/44] Proof of concept for Performance run on ADO using BDN --- .../pipelines/azure-pipelines-performance.yml | 237 +----------------- 1 file changed, 11 insertions(+), 226 deletions(-) diff --git a/.azure/pipelines/azure-pipelines-performance.yml b/.azure/pipelines/azure-pipelines-performance.yml index 6806f286e0..8829756c3a 100644 --- a/.azure/pipelines/azure-pipelines-performance.yml +++ b/.azure/pipelines/azure-pipelines-performance.yml @@ -3,13 +3,6 @@ trigger: include: - refs/heads/main -schedules: -- cron: "0 7 * * *" # 7:00 UTC is 11:00PM Pacific - branches: - include: - - refs/heads/main - always: true - resources: repositories: - repository: self @@ -49,124 +42,26 @@ jobs: - script: dotnet restore displayName: .NET Restore - - task: DotNetCoreCLI@2 - displayName: '.NET Core build Cluster binaries and tests $(buildConfiguration)' - inputs: - command: 'build' - projects: '**/Garnet.test.cluster.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: DotNetCoreCLI@2 - displayName: '.NET Core build Garnet binaries and tests $(buildConfiguration)' - inputs: - command: 'build' - projects: '**/Garnet.test.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: DotNetCoreCLI@2 - displayName: '.NET Core build Garnet Server' - inputs: - command: 'build' - projects: '**/GarnetServer.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: DotNetCoreCLI@2 - displayName: '.NET Core build Resp.benchmark' - inputs: - command: 'build' - projects: '**/Resp.benchmark.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: PowerShell@2 - displayName: '1: Resp.Benchmark ONLINE; Get Set; 1 Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_GetSet_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '2: Resp.Benchmark ONLINE; Get Set; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_GetSet_MaxThr.json' - workingDirectory: test/PerfRegressionTesting - - task: PowerShell@2 - displayName: '3: Resp.Benchmark ONLINE; ZADD ZREM; 1 Thread' + displayName: '1: BDN.benchmark ' continueOnError: true enabled: true inputs: targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '4: Resp.Benchmark ONLINE; ZADD ZREM; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_MaxThr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '5: Resp.Benchmark OFFLINE; Get; 1 Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_Get_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '6: Resp.Benchmark OFFLINE; Get; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_Get_MaxThr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '7: Resp.Benchmark OFFLINE; ZADDREM; 1 Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '8: Resp.Benchmark OFFLINE; ZADDREM; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_MaxThr.json' - workingDirectory: test/PerfRegressionTesting + filePath: './test/BDNPerfTests/run_bdnperftest.ps1' + arguments: 'CI_BDN_Config_RespParseStress.json' + workingDirectory: test/BDNPerfTests - task: CopyFiles@2 displayName: 'Copy Results Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/PerfRegressionTesting/results/**/*' + Contents: '**/test/BDNPerfTests/results/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: CopyFiles@2 displayName: 'Copy ErrorLog Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/PerfRegressionTesting/errorlog/**/*' + Contents: '**/test/BDNPerfTests/errorlog/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: PublishBuildArtifacts@1 @@ -201,136 +96,26 @@ jobs: - task: NuGetAuthenticate@1 displayName: 'NuGet Authenticate' - - task: DotNetCoreCLI@2 - displayName: 'Build Garnet Cluster binaries and tests $(buildConfiguration)' - inputs: - command: 'build' - projects: '**/Garnet.test.cluster.csproj' - arguments: '--configuration $(buildConfiguration)' - - - script: nuget restore - displayName: Nuget Restore - - - task: DotNetCoreCLI@2 - displayName: 'Build Garnet binaries and tests $(buildConfiguration)' - inputs: - command: 'build' - projects: '**/Garnet.test.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: DotNetCoreCLI@2 - displayName: 'Build Garnet Server $(buildConfiguration)' - inputs: - command: 'build' - projects: '**/GarnetServer.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: DotNetCoreCLI@2 - displayName: '.NET Core build Resp.benchmark' - inputs: - command: 'build' - projects: '**/Resp.benchmark.csproj' - arguments: '--configuration $(buildConfiguration)' - - - task: PowerShell@2 - displayName: 'Run Tsavorite Benchmark Test' - enabled: false - inputs: - targetType: filePath - filePath: './libs/storage/Tsavorite/cs/benchmark/scripts/run_benchmark.ps1' - arguments: '$(Build.SourcesDirectory)' - workingDirectory: libs/storage/Tsavorite/cs/benchmark/scripts - - task: PowerShell@2 displayName: '1: Resp.Benchmark ONLINE; Get Set; 1 Thread' continueOnError: true enabled: true inputs: targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_GetSet_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '2: Resp.Benchmark ONLINE; Get Set; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_GetSet_MaxThr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '3: Resp.Benchmark ONLINE; ZADD ZREM; 1 Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '4: Resp.Benchmark ONLINE; ZADD ZREM; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_MaxThr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '5: Resp.Benchmark OFFLINE; Get; 1 Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_Get_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '6: Resp.Benchmark OFFLINE; Get; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_Get_MaxThr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '7: Resp.Benchmark OFFLINE; ZADDREM; 1 Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_1Thr.json' - workingDirectory: test/PerfRegressionTesting - - - task: PowerShell@2 - displayName: '8: Resp.Benchmark OFFLINE; ZADDREM; Max Thread' - continueOnError: true - enabled: true - inputs: - targetType: filePath - filePath: './test/PerfRegressionTesting/run_benchmark.ps1' - arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_MaxThr.json' - workingDirectory: test/PerfRegressionTesting + filePath: './test/BDNPerfTests/run_bdnperftest.ps1' + arguments: 'CI_BDN_Config_RespParseStress.json' + workingDirectory: test/BDNPerfTests - task: CopyFiles@2 displayName: 'Copy Results Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/PerfRegressionTesting/results/**/*' + Contents: '**/test/BDNPerfTests/results/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: CopyFiles@2 displayName: 'Copy ErrorLog Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/PerfRegressionTesting/errorlog/**/*' + Contents: '**/test/BDNPerfTests/errorlog/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: PublishBuildArtifacts@1 From ebd13e404d2292a96b31d0f003558b48d8d8202a Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Sep 2024 16:16:58 -0700 Subject: [PATCH 05/44] Comment out number of Cores check so can try proof of concept on ADO perf machines --- test/BDNPerfTests/run_bdnperftest.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 44fccfca3b..7834785dc8 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -125,8 +125,9 @@ else { # To get accurate comparison of found vs expected values, double check to make sure config settings for Number of Cores of the test machine are what is specified in the test config file if ($ExpectedCoresToTestOn -ne $NumberOfCores) { - Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." - exit + # DEBUG ---- Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." + # DEBUG ---- exit + Write-Host "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." } Write-Host "************** Start BDN.benchmark ********************" From 37d12df216409634ba388f12bdeb0448c91672a6 Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 8 Oct 2024 15:35:19 -0700 Subject: [PATCH 06/44] Getting close to having BDN working --- .github/workflows/ci-bdnbenchmark.yml | 37 ++- benchmark/BDN.benchmark/Resp/RespLuaStress.cs | 8 +- ..._BDN_Config_Cluster.RespClusterBench.json} | 4 +- ...nfig_Cluster.RespClusterMigrateBench.json} | 4 +- .../ConfigFiles/CI_BDN_Config_Lua.json | 27 -- ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 28 ++ .../CI_BDN_Config_Resp.RespLuaStress.json | 28 ++ ...> CI_BDN_Config_Resp.RespParseStress.json} | 19 +- ..._BDN_Config_Resp.RespTsavoriteStress.json} | 6 +- test/BDNPerfTests/run_bdnperftest.ps1 | 256 ++++++++++++++---- 10 files changed, 309 insertions(+), 108 deletions(-) rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_RespClusterBench.json => CI_BDN_Config_Cluster.RespClusterBench.json} (67%) rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_RespClusterMigrateBench.json => CI_BDN_Config_Cluster.RespClusterMigrateBench.json} (68%) delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_RespParseStress.json => CI_BDN_Config_Resp.RespParseStress.json} (63%) rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_RespTsavoriteStress.json => CI_BDN_Config_Resp.RespTsavoriteStress.json} (58%) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index d47b1e7b87..ef6a38e3f1 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -12,7 +12,8 @@ jobs: runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code permissions: pull-requests: read - contents: read + deployments: write #permission to deploy GitHub pages website + contents: write # permission to update benchmark contents in gh-pages branch steps: - name: Check out code uses: actions/checkout@v4 @@ -37,7 +38,7 @@ jobs: os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] - test: [ 'CI_BDN_Config_RespParseStress.json', 'CI_BDN_Config_RespTsavoriteStress.json', 'CI_BDN_Config_RespClusterBench.json', 'CI_BDN_Config_RespClusterMigrateBench.json', 'CI_BDN_Config_Lua.json' ] + test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] steps: - name: Check out code uses: actions/checkout@v4 @@ -55,15 +56,41 @@ jobs: - name: Upload test results to artifacts uses: actions/upload-artifact@v4 with: - name: ResultsDir-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + name: Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} path: | ./test/BDNPerfTests/results if: ${{ always() }} - name: Upload Error Log to artifacts uses: actions/upload-artifact@v4 with: - name: ErrorLogDir-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + name: ErrorLog-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} path: | ./test/BDNPerfTests/errorlog if: ${{ always() }} - + + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool created output file + tool: 'benchmarkdotnet' + + # Need for both commit comment and charts + output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + + # This is for commit comment & Job Summary + external-data-json-path: ./cache/benchmark-data.json + fail-on-alert: true + comment-on-alert: true + summary-always: true + + # This is for Charts on GitHub pages + auto-push: false diff --git a/benchmark/BDN.benchmark/Resp/RespLuaStress.cs b/benchmark/BDN.benchmark/Resp/RespLuaStress.cs index 45cdcefbf7..fef825366f 100644 --- a/benchmark/BDN.benchmark/Resp/RespLuaStress.cs +++ b/benchmark/BDN.benchmark/Resp/RespLuaStress.cs @@ -58,19 +58,19 @@ LuaFunction CreateFunction(string source) } [Benchmark] - public void BasicLua1() + public void BasicLuaStress1() => f1.Call(); [Benchmark] - public void BasicLua2() + public void BasicLuaStress2() => f2.Call(); [Benchmark] - public void BasicLua3() + public void BasicLuaStress3() => f3.Call(); [Benchmark] - public void BasicLua4() + public void BasicLuaStress4() => f4.Call(); } } \ No newline at end of file diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json similarity index 67% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json index 4b38979781..530c9c6135 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterBench*", @@ -13,6 +10,7 @@ "expectedSETMeanValue_win": 24, "expectedMGETMeanValue_win": 19, "expectedMSETMeanValue_win": 17, + "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json similarity index 68% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json index 5f4387832b..9caf6bd3f3 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterMigrateBench*", @@ -13,6 +10,7 @@ "expectedSETMeanValue_win": 47, "expectedMGETMeanValue_win": 40, "expectedMSETMeanValue_win": 37, + "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json deleted file mode 100644 index d40425bf3a..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, - "configuration": "Release", - "framework": "net8.0", - "filter": "*Lua*", - "expectedBasicLuaRunner1MeanValue_linux": 186, - "expectedBasicLuaRunner2MeanValue_linux": 305, - "expectedBasicLuaRunner3MeanValue_linux": 1473, - "expectedBasicLuaRunner4MeanValue_linux": 3754, - "expectedBasicLuaRunner1MeanValue_win": 111, - "expectedBasicLuaRunner2MeanValue_win": 173, - "expectedBasicLuaRunner3MeanValue_win": 686, - "expectedBasicLua4RunnerMeanValue_win": 2711, - "expectedBasicLua1MeanValue_linux": 169, - "expectedBasicLua2MeanValue_linux": 284, - "expectedBasicLua3MeanValue_linux": 1407, - "expectedBasicLua4MeanValue_linux": 2733, - "expectedBasicLua1MeanValue_win": 107, - "expectedBasicLua2MeanValue_win": 180, - "expectedBasicLua3MeanValue_win": 660, - "expectedBasicLua4MeanValue_win": 1805, - "acceptableMeanRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json new file mode 100644 index 0000000000..daefa3193c --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -0,0 +1,28 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "*LuaRunner*", + "expectedBasicLuaRunner1MeanValue_linux": 186, + "expectedBasicLuaRunner2MeanValue_linux": 305, + "expectedBasicLuaRunner3MeanValue_linux": 1473, + "expectedBasicLuaRunner4MeanValue_linux": 3754, + "expectedBasicLuaRunner1MeanValue_win": 111, + "expectedBasicLuaRunner2MeanValue_win": 173, + "expectedBasicLuaRunner3MeanValue_win": 686, + "expectedBasicLuaRunner4MeanValue_win": 2711, + + "expectedBasicLuaRunner1AllocatedValue_linux": 24, + "expectedBasicLuaRunner2AllocatedValue_linux": 144, + "expectedBasicLuaRunner3AllocatedValue_linux": 328, + "expectedBasicLuaRunner4AllocatedValue_linux": 864, + "expectedBasicLuaRunner1AllocatedValue_win": 24, + "expectedBasicLuaRunner2AllocatedValue_win": 144, + "expectedBasicLuaRunner3AllocatedValue_win": 328, + "expectedBasicLuaRunner4AllocatedValue_win": 864, + + + "acceptableMeanRange": 10, + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json new file mode 100644 index 0000000000..97749bd111 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -0,0 +1,28 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "*LuaStress*", + + "expectedBasicLuaStress1MeanValue_linux": 169, + "expectedBasicLuaStress2MeanValue_linux": 284, + "expectedBasicLuaStress3MeanValue_linux": 1407, + "expectedBasicLuaStress4MeanValue_linux": 2733, + "expectedBasicLuaStress1AllocatedValue_linux": 24, + "expectedBasicLuaStress2AllocatedValue_linux": 144, + "expectedBasicLuaStress3AllocatedValue_linux": 328, + "expectedBasicLuaStress4AllocatedValue_linux": 832, + + "expectedBasicLuaStress1MeanValue_win": 107, + "expectedBasicLuaStress2MeanValue_win": 180, + "expectedBasicLuaStress3MeanValue_win": 660, + "expectedBasicLuaStress4MeanValue_win": 1805, + "expectedBasicLuaStress1AllocatedValue_win": 24, + "expectedBasicLuaStress2AllocatedValue_win": 144, + "expectedBasicLuaStress3AllocatedValue_win": 328, + "expectedBasicLuaStress4AllocatedValue_win": 832, + + "acceptableMeanRange": 10, + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json similarity index 63% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json index 0995827e35..07c85ff150 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespParseStress*", @@ -14,6 +11,14 @@ "expectedSAddRemMeanValue_linux": 172, "expectedHSetDelMeanValue_linux": 200, "expectedMyDictSetGetMeanValue_linux": 235, + + "expectedZAddRemAllocatedValue_linux": 23552, + "expectedLPushPopAllocatedValue_linux": 30721, + "expectedSAddRemAllocatedValue_linux": 16384, + "expectedHSetDelAllocatedValue_linux": 55297, + "expectedMyDictSetGetAllocatedValue_linux": 30720, + + "expectedInLinePingMeanValue_win": 2.4, "expectedSETMeanValue_win": 19, "expectedSETEXMeanValue_win": 26, @@ -23,7 +28,15 @@ "expectedSAddRemMeanValue_win": 130, "expectedHSetDelMeanValue_win": 160, "expectedMyDictSetGetMeanValue_win": 242, + + "expectedZAddRemAllocatedValue_win": 23552, + "expectedLPushPopAllocatedValue_win": 30721, + "expectedSAddRemAllocatedValue_win": 16384, + "expectedHSetDelAllocatedValue_win": 55297, + "expectedMyDictSetGetAllocatedValue_win": 30720, + "acceptableMeanRange": 10, + "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json similarity index 58% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json index ff0c0333c1..cdb2326ae6 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json @@ -1,16 +1,14 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespTsavoriteStress*", "expectedGETMeanValue_linux": 21, "expectedSETMeanValue_linux": 19, - "expectedIncrMeanValue_linux": 29, + "expectedIncrMeanValue_linux": 27, "expectedGETMeanValue_win": 20, "expectedSETMeanValue_win": 17, "expectedIncrMeanValue_win": 26, + "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 7834785dc8..0aeeced8e9 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -6,20 +6,20 @@ Script to test for performance regressions using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. - CI_BDN_Config_RespParseStress.json + CI_BDN_Config_Resp.RespParseStress.json NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% .EXAMPLE ./run_bdnperftest.ps1 - ./run_bdnperftest.ps1 CI_BDN_Config_RespParseStress.json + ./run_bdnperftest.ps1 CI_BDN_Config_Resp.RespParseStress.json #> # Send the config file for the benchmark. Defaults to a simple one param ( - [string]$configFile = "CI_BDN_Config_RespParseStress.json" + [string]$configFile = "CI_BDN_Config_Resp.RespParseStress.json" ) $OFS = "`r`n" @@ -39,20 +39,20 @@ function AnalyzeResult { [double] $LowerBound = $expectedResultValue * (1 - $Tolerance) [double] $UpperBound = $expectedResultValue * (1 + $Tolerance) [double] $dblfoundResultValue = $foundResultValue - + # Check if the actual value is within the bounds if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Mean Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Mean Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Mean Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } return $false # the values are too different @@ -74,15 +74,16 @@ param ($ResultsLine, $columnNum) # Remove the leading and trailing pipes and split the string by '|' $columns = $ResultsLine.Trim('|').Split('|') $column = $columns | ForEach-Object { $_.Trim() } - $foundValue = $column[$columnNum].Trim(' us') + $foundValue = $column[$columnNum] + + $foundValue = $foundValue.Trim(' us') $foundValue = $foundValue.Trim(' ns') + $foundValue = $foundValue.Trim(' B') return $foundValue } - - # ******** BEGIN MAIN ********* # Get base path since paths can differ from machine to machine $pathstring = $pwd.Path @@ -94,6 +95,16 @@ if ($pathstring.Contains("test")) { Set-Location .\test\BDNPerfTests\ } +# Add .json to end if not already there +if ($configFile -notlike "*.json") { + $configFile += ".json" +} + +# This is special case that allows passing test without specifying CI_BDN_Confi_ at the beginning - need for perf test +if ($configFile -notlike "CI_BDN_Config_*") { + $configFile = "CI_BDN_Config_" + $configFile +} + # Read the test config file and convert the JSON to a PowerShell object $fullConfiFileAndPath = "ConfigFiles/$configFile" if (-not (Test-Path -Path $fullConfiFileAndPath)) { @@ -109,27 +120,6 @@ if ($IsLinux) { $CurrentOS = "Linux" } - # Calculate number of cores on the test machine - used to verify the config file settings as the config settings will vary based on machine config -if ($IsLinux) { - $sockets = [int]$(lscpu | grep -E '^Socket' | awk '{print $2}') - $coresPerSocket = [int]$(lscpu | grep -E '^Core' | awk '{print $4}') - $NumberOfCores = ($sockets * $coresPerSocket) - - $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_linux -} -else { - $NumberOfCores = (Get-CimInstance -ClassName Win32_Processor).NumberOfCores | Measure-Object -Sum | Select-Object -ExpandProperty Sum - - $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_win -} - -# To get accurate comparison of found vs expected values, double check to make sure config settings for Number of Cores of the test machine are what is specified in the test config file -if ($ExpectedCoresToTestOn -ne $NumberOfCores) { - # DEBUG ---- Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." - # DEBUG ---- exit - Write-Host "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." -} - Write-Host "************** Start BDN.benchmark ********************" Write-Host " " @@ -138,6 +128,12 @@ $configuration = $object.configuration $framework = $object.framework $filter = $object.filter $meanColumn = "1" +$allocatedRespParseColumn = "5" # Windows adds an additional column before the allocated column +$allocatedLuaColumn = "5" +if ($IsLinux) { + $allocatedRespParseColumn = "4" + $allocatedLuaColumn = "4" +} # Set the expected values based on the OS if ($IsLinux) { @@ -146,22 +142,39 @@ if ($IsLinux) { $expectedSetMeanValue = $object.expectedSETMeanValue_linux $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_linux $expectedGetMeanValue = $object.expectedGETMeanValue_linux + $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_linux $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_linux $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_linux $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_linux $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_linux + $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_linux + $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_linux + $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_linux + $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_linux + $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_linux + $expectedMGetMeanValue = $object.expectedMGETMeanValue_linux $expectedMSetMeanValue = $object.expectedMSETMeanValue_linux $expectedIncrMeanValue = $object.expectedIncrMeanValue_linux - $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_linux - $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_linux - $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_linux - $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_linux + + $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_linux + $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_linux + $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_linux + $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_linux + $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_linux + $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_linux + $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_linux + $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_linux + $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_linux $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_linux $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_linux + $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_linux + $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_linux + $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_linux + $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_linux } else { # Windows expected values @@ -169,26 +182,44 @@ else { $expectedSetMeanValue = $object.expectedSETMeanValue_win $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_win $expectedGetMeanValue = $object.expectedGETMeanValue_win + $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_win $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_win $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_win $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_win $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_win + $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_win + $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_win + $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_win + $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_win + $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_win + $expectedMGetMeanValue = $object.expectedMGETMeanValue_win $expectedMSetMeanValue = $object.expectedMSETMeanValue_win $expectedIncrMeanValue = $object.expectedIncrMeanValue_win - $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_win - $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_win - $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_win - $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_win + + $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_win + $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_win + $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_win + $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_win + $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_win + $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_win + $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_win + $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_win + $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_win $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_win $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_win - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_win + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_win + $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_win + $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_win + $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_win + $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_win } # percent allowed variance when comparing expected vs actual found value - same for linux and windows. $acceptableMeanRange = $object.acceptableMeanRange +$acceptableAllocatedRange = $object.acceptableAllocatedRange # Set up the results dir and errorlog dir $resultsDir = "$basePath/test/BDNPerfTests/results" @@ -211,8 +242,8 @@ $BDNbenchmarkErrorFile = "$errorLogDir/$justResultsFileNameNoExt" + "_StandardEr Write-Output "** Start BDN Benchmark: $filter" Write-Output " " -Write-Output "** Start: dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath > $resultsFile 2> $BDNbenchmarkErrorFile" -dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath > $resultsFile 2> $BDNbenchmarkErrorFile +Write-Output "** Start: dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath --exporters json > $resultsFile 2> $BDNbenchmarkErrorFile" +dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath --exporters json > $resultsFile 2> $BDNbenchmarkErrorFile Write-Output "** BDN Benchmark for $filter finished" Write-Output " " @@ -316,6 +347,46 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } + "*| ZAddRem*" { + Write-Host "** ZAddRem Allocated Value test" + $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $currentResults = AnalyzeResult $foundZAddRemAllocatedValue $expectedZAddRemAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| LPushPop*" { + Write-Host "** LPushPop Allocated Value test" + $foundLPushPopAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $currentResults = AnalyzeResult $foundLPushPopAllocatedValue $expectedLPushPopAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| SAddRem*" { + Write-Host "** SAddRem Allocated Value test" + $foundSAddRemAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $currentResults = AnalyzeResult $foundSAddRemAllocatedValue $expectedSAddRemAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| HSetDel*" { + Write-Host "** HSetDel Allocated Value test" + $foundHSetDelAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $currentResults = AnalyzeResult $foundHSetDelAllocatedValue $expectedHSetDelAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MyDictSetGet*" { + Write-Host "** MyDictSetGet Allocated Value test" + $foundMyDictSetGetAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $currentResults = AnalyzeResult $foundMyDictSetGetAllocatedValue $expectedMyDictSetGetAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } "*| Incr*" { Write-Host "** Incr Mean Value test" $foundIncrMeanValue = ParseValueFromResults $line $meanColumn @@ -340,41 +411,42 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } - "*| BasicLua1*" { - Write-Host "** BasicLua1 Mean Value test" - $foundBasicLua1MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua1MeanValue $expectedBasicLua1MeanValue $acceptableMeanRange $true + "*| BasicLuaStress1*" { + Write-Host "** BasicLuaStress1 Mean Value test" + $foundBasicLuaStress1MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress1MeanValue $expectedBasicLuaStress1MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLua2*" { - Write-Host "** BasicLua2 Mean Value test" - $foundBasicLua2MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua2MeanValue $expectedBasicLua2MeanValue $acceptableMeanRange $true + "*| BasicLuaStress2*" { + Write-Host "** BasicLuaStress2 Mean Value test" + $foundBasicLuaStress2MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress2MeanValue $expectedBasicLuaStress2MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } <# # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. - "*| BasicLua3*" { - Write-Host "** BasicLua3 Mean Value test" - $foundBasicLua3MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua3MeanValue $expectedBasicLua3MeanValue $acceptableMeanRange $true + "*| BasicLuaStress3*" { + Write-Host "** BasicLuaStress3 Mean Value test" + $foundBasicLuaStress3MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress3MeanValue $expectedBasicLuaStress3MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLua4*" { - Write-Host "** BasicLua4 Mean Value test" - $foundBasicLua4MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua4MeanValue $expectedBasicLua4MeanValue $acceptableMeanRange $true + "*| BasicLuaStress4*" { + Write-Host "** BasicLuaStress4 Mean Value test" + $foundBasicLuaStress4MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress4MeanValue $expectedBasicLuaStress4MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } #> + "*| BasicLuaRunner1*" { Write-Host "** BasicLuaRunner1 Mean Value test" $foundBasicLuaRunner1MeanValue = ParseValueFromResults $line $meanColumn @@ -411,6 +483,72 @@ Get-Content $resultsFile | ForEach-Object { } } #> + + "*| BasicLuaStress1*" { + Write-Host "** BasicLuaStress1 Allocated Value test" + $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaStress1AllocatedValue $expectedBasicLuaStress1AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaStress2*" { + Write-Host "** BasicLuaStress2 Allocated Value test" + $foundBasicLuaStress2AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaStress2AllocatedValue $expectedBasicLuaStress2AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaStress3*" { + Write-Host "** BasicLuaStress3 Allocated Value test" + $foundBasicLuaStress3AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaStress3AllocatedValue $expectedBasicLuaStress3AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaStress4*" { + Write-Host "** BasicLuaStress4 Allocated Value test" + $foundBasicLuaStress4AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaStress4AllocatedValue $expectedBasicLuaStress4AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner1*" { + Write-Host "** BasicLuaRunner1 Allocated Value test" + $foundBasicLuaRunner1AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner1AllocatedValue $expectedBasicLuaRunner1AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner2*" { + Write-Host "** BasicLuaRunner2 Allocated Value test" + $foundBasicLuaRunner2AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner2AllocatedValue $expectedBasicLuaRunner2AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + "*| BasicLuaRunner3*" { + Write-Host "** BasicLuaRunner3 Allocated Value test" + $foundBasicLuaRunner3AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner3AllocatedValue $expectedBasicLuaRunner3AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner4*" { + Write-Host "** BasicLuaRunner4 Allocated Value test" + $foundBasicLuaRunner4AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner4AllocatedValue $expectedBasicLuaRunner4AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } } } From b97f807e6902b53e11d06959cc8244ac166471d2 Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 8 Oct 2024 15:56:15 -0700 Subject: [PATCH 07/44] Fixed RespLuaStress --- benchmark/BDN.benchmark/Resp/RespLuaStress.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/BDN.benchmark/Resp/RespLuaStress.cs b/benchmark/BDN.benchmark/Resp/RespLuaStress.cs index fef825366f..d305f806f4 100644 --- a/benchmark/BDN.benchmark/Resp/RespLuaStress.cs +++ b/benchmark/BDN.benchmark/Resp/RespLuaStress.cs @@ -70,7 +70,7 @@ public void BasicLuaStress3() => f3.Call(); [Benchmark] - public void BasicLuaStress4() + public void BasicLuaStress4() => f4.Call(); } } \ No newline at end of file From ba450916f7d692cc9f716234f2eb9710900e8f92 Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 8 Oct 2024 16:29:38 -0700 Subject: [PATCH 08/44] Fixed issue in Lua Runner Stress --- benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs b/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs index 866dedddda..aec4994525 100644 --- a/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs +++ b/benchmark/BDN.benchmark/Resp/RespLuaRunnerStress.cs @@ -10,7 +10,7 @@ namespace BDN.benchmark.Resp public unsafe class RespLuaRunnerStress { LuaRunner r1, r2, r3, r4; - readonly string[] keys = ["key1", "key2"]; + readonly string[] keys = ["key1"]; [GlobalSetup] public void GlobalSetup() @@ -36,7 +36,7 @@ public void GlobalCleanup() [Benchmark] public void BasicLuaRunner1() - => r1.RunVoid(); + => r1.Run(); [Benchmark] public void BasicLuaRunner2() From bff5a215541dbb267422cf86b1602b7a2ef464a2 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 9 Oct 2024 15:04:26 -0700 Subject: [PATCH 09/44] Updated config values, set charting and commit comment as separate tasks --- .github/workflows/ci-bdnbenchmark.yml | 17 +++++++-- ...I_BDN_Config_Cluster.RespClusterBench.json | 2 +- ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 18 +++++---- .../CI_BDN_Config_Resp.RespLuaStress.json | 2 +- .../CI_BDN_Config_Resp.RespParseStress.json | 37 ++++++++++--------- ...I_BDN_Config_Resp.RespTsavoriteStress.json | 4 +- test/BDNPerfTests/run_bdnperftest.ps1 | 37 ++++++++----------- 7 files changed, 63 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index ef6a38e3f1..aab153b91c 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -76,7 +76,7 @@ jobs: key: ${{ runner.os }}-benchmark # Run `github-action-benchmark` action - - name: Store benchmark result + - name: Store benchmark result for commit comment uses: benchmark-action/github-action-benchmark@v1 with: # What benchmark tool created output file @@ -86,11 +86,22 @@ jobs: output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - # This is for commit comment & Job Summary external-data-json-path: ./cache/benchmark-data.json fail-on-alert: true comment-on-alert: true summary-always: true + auto-push: false + + # Run `github-action-benchmark` action + - name: Store benchmark result for charts + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool created output file + tool: 'benchmarkdotnet' + + # Need for both commit comment and charts + output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} # This is for Charts on GitHub pages - auto-push: false + auto-push: true diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json index 530c9c6135..53bca030b5 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json @@ -9,7 +9,7 @@ "expectedGETMeanValue_win": 24, "expectedSETMeanValue_win": 24, "expectedMGETMeanValue_win": 19, - "expectedMSETMeanValue_win": 17, + "expectedMSETMeanValue_win": 19, "acceptableMeanRange": 10, "acceptableError": ".1", diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json index daefa3193c..3ee86d9ac5 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -2,24 +2,26 @@ "configuration": "Release", "framework": "net8.0", "filter": "*LuaRunner*", - "expectedBasicLuaRunner1MeanValue_linux": 186, - "expectedBasicLuaRunner2MeanValue_linux": 305, + "expectedBasicLuaRunner1MeanValue_linux": 200, + "expectedBasicLuaRunner2MeanValue_linux": 337, "expectedBasicLuaRunner3MeanValue_linux": 1473, "expectedBasicLuaRunner4MeanValue_linux": 3754, - "expectedBasicLuaRunner1MeanValue_win": 111, + + "expectedBasicLuaRunner1MeanValue_win": 115, "expectedBasicLuaRunner2MeanValue_win": 173, "expectedBasicLuaRunner3MeanValue_win": 686, "expectedBasicLuaRunner4MeanValue_win": 2711, + "expectedBasicLuaRunner1AllocatedValue_linux": 24, "expectedBasicLuaRunner2AllocatedValue_linux": 144, - "expectedBasicLuaRunner3AllocatedValue_linux": 328, - "expectedBasicLuaRunner4AllocatedValue_linux": 864, + "expectedBasicLuaRunner3AllocatedValue_linux": 240, + "expectedBasicLuaRunner4AllocatedValue_linux": 776, + "expectedBasicLuaRunner1AllocatedValue_win": 24, "expectedBasicLuaRunner2AllocatedValue_win": 144, - "expectedBasicLuaRunner3AllocatedValue_win": 328, - "expectedBasicLuaRunner4AllocatedValue_win": 864, - + "expectedBasicLuaRunner3AllocatedValue_win": 240, + "expectedBasicLuaRunner4AllocatedValue_win": 776, "acceptableMeanRange": 10, "acceptableAllocatedRange": 10, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json index 97749bd111..67dda1726d 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -3,7 +3,7 @@ "framework": "net8.0", "filter": "*LuaStress*", - "expectedBasicLuaStress1MeanValue_linux": 169, + "expectedBasicLuaStress1MeanValue_linux": 187, "expectedBasicLuaStress2MeanValue_linux": 284, "expectedBasicLuaStress3MeanValue_linux": 1407, "expectedBasicLuaStress4MeanValue_linux": 2733, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json index 07c85ff150..cc17e7871b 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json @@ -2,35 +2,36 @@ "configuration": "Release", "framework": "net8.0", "filter": "*RespParseStress*", - "expectedInLinePingMeanValue_linux": 2.4, + "expectedInLinePingMeanValue_linux": 2.2, "expectedSETMeanValue_linux": 19, "expectedSETEXMeanValue_linux": 26, - "expectedGETMeanValue_linux": 14, + "expectedGETMeanValue_linux": 13, + "expectedIncrementMeanValue_linux": 27, "expectedZAddRemMeanValue_linux": 186, - "expectedLPushPopMeanValue_linux": 200, - "expectedSAddRemMeanValue_linux": 172, - "expectedHSetDelMeanValue_linux": 200, - "expectedMyDictSetGetMeanValue_linux": 235, + "expectedLPushPopMeanValue_linux": 166, + "expectedSAddRemMeanValue_linux": 164, + "expectedHSetDelMeanValue_linux": 196, + "expectedMyDictSetGetMeanValue_linux": 248, "expectedZAddRemAllocatedValue_linux": 23552, - "expectedLPushPopAllocatedValue_linux": 30721, + "expectedLPushPopAllocatedValue_linux": 18432, "expectedSAddRemAllocatedValue_linux": 16384, "expectedHSetDelAllocatedValue_linux": 55297, "expectedMyDictSetGetAllocatedValue_linux": 30720, - - "expectedInLinePingMeanValue_win": 2.4, - "expectedSETMeanValue_win": 19, - "expectedSETEXMeanValue_win": 26, - "expectedGETMeanValue_win": 14, - "expectedZAddRemMeanValue_win": 141, - "expectedLPushPopMeanValue_win": 145, - "expectedSAddRemMeanValue_win": 130, - "expectedHSetDelMeanValue_win": 160, - "expectedMyDictSetGetMeanValue_win": 242, + "expectedInLinePingMeanValue_win": 2.2, + "expectedSETMeanValue_win": 18, + "expectedSETEXMeanValue_win": 22, + "expectedGETMeanValue_win": 12, + "expectedIncrementMeanValue_win": 24, + "expectedZAddRemMeanValue_win": 121, + "expectedLPushPopMeanValue_win": 105, + "expectedSAddRemMeanValue_win": 101, + "expectedHSetDelMeanValue_win": 130, + "expectedMyDictSetGetMeanValue_win": 187, "expectedZAddRemAllocatedValue_win": 23552, - "expectedLPushPopAllocatedValue_win": 30721, + "expectedLPushPopAllocatedValue_win": 18432, "expectedSAddRemAllocatedValue_win": 16384, "expectedHSetDelAllocatedValue_win": 55297, "expectedMyDictSetGetAllocatedValue_win": 30720, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json index cdb2326ae6..337a3268a7 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json @@ -4,10 +4,10 @@ "filter": "*RespTsavoriteStress*", "expectedGETMeanValue_linux": 21, "expectedSETMeanValue_linux": 19, - "expectedIncrMeanValue_linux": 27, + "expectedIncrementMeanValue_linux": 27, "expectedGETMeanValue_win": 20, "expectedSETMeanValue_win": 17, - "expectedIncrMeanValue_win": 26, + "expectedIncrementMeanValue_win": 26, "acceptableMeanRange": 10, "acceptableError": ".1", diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 0aeeced8e9..f58687302c 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -128,12 +128,7 @@ $configuration = $object.configuration $framework = $object.framework $filter = $object.filter $meanColumn = "1" -$allocatedRespParseColumn = "5" # Windows adds an additional column before the allocated column -$allocatedLuaColumn = "5" -if ($IsLinux) { - $allocatedRespParseColumn = "4" - $allocatedLuaColumn = "4" -} +$allocatedColumn = "-1" # Number of columns can differ a bit, but know the allocated one is always the last one # Set the expected values based on the OS if ($IsLinux) { @@ -156,7 +151,7 @@ if ($IsLinux) { $expectedMGetMeanValue = $object.expectedMGETMeanValue_linux $expectedMSetMeanValue = $object.expectedMSETMeanValue_linux - $expectedIncrMeanValue = $object.expectedIncrMeanValue_linux + $expectedIncrMeanValue = $object.expectedIncrementMeanValue_linux $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_linux $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_linux @@ -196,7 +191,7 @@ else { $expectedMGetMeanValue = $object.expectedMGETMeanValue_win $expectedMSetMeanValue = $object.expectedMSETMeanValue_win - $expectedIncrMeanValue = $object.expectedIncrMeanValue_win + $expectedIncrMeanValue = $object.expectedIncrementMeanValue_win $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_win $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_win @@ -349,7 +344,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| ZAddRem*" { Write-Host "** ZAddRem Allocated Value test" - $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundZAddRemAllocatedValue $expectedZAddRemAllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -357,7 +352,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| LPushPop*" { Write-Host "** LPushPop Allocated Value test" - $foundLPushPopAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $foundLPushPopAllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundLPushPopAllocatedValue $expectedLPushPopAllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -365,7 +360,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| SAddRem*" { Write-Host "** SAddRem Allocated Value test" - $foundSAddRemAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $foundSAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundSAddRemAllocatedValue $expectedSAddRemAllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -373,7 +368,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| HSetDel*" { Write-Host "** HSetDel Allocated Value test" - $foundHSetDelAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $foundHSetDelAllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundHSetDelAllocatedValue $expectedHSetDelAllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -381,7 +376,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| MyDictSetGet*" { Write-Host "** MyDictSetGet Allocated Value test" - $foundMyDictSetGetAllocatedValue = ParseValueFromResults $line $allocatedRespParseColumn + $foundMyDictSetGetAllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundMyDictSetGetAllocatedValue $expectedMyDictSetGetAllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -486,7 +481,7 @@ Get-Content $resultsFile | ForEach-Object { "*| BasicLuaStress1*" { Write-Host "** BasicLuaStress1 Allocated Value test" - $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaStress1AllocatedValue $expectedBasicLuaStress1AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -494,7 +489,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| BasicLuaStress2*" { Write-Host "** BasicLuaStress2 Allocated Value test" - $foundBasicLuaStress2AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaStress2AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaStress2AllocatedValue $expectedBasicLuaStress2AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -502,7 +497,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| BasicLuaStress3*" { Write-Host "** BasicLuaStress3 Allocated Value test" - $foundBasicLuaStress3AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaStress3AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaStress3AllocatedValue $expectedBasicLuaStress3AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -510,7 +505,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| BasicLuaStress4*" { Write-Host "** BasicLuaStress4 Allocated Value test" - $foundBasicLuaStress4AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaStress4AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaStress4AllocatedValue $expectedBasicLuaStress4AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -518,7 +513,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| BasicLuaRunner1*" { Write-Host "** BasicLuaRunner1 Allocated Value test" - $foundBasicLuaRunner1AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaRunner1AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaRunner1AllocatedValue $expectedBasicLuaRunner1AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -526,7 +521,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| BasicLuaRunner2*" { Write-Host "** BasicLuaRunner2 Allocated Value test" - $foundBasicLuaRunner2AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaRunner2AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaRunner2AllocatedValue $expectedBasicLuaRunner2AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -535,7 +530,7 @@ Get-Content $resultsFile | ForEach-Object { "*| BasicLuaRunner3*" { Write-Host "** BasicLuaRunner3 Allocated Value test" - $foundBasicLuaRunner3AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaRunner3AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaRunner3AllocatedValue $expectedBasicLuaRunner3AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false @@ -543,7 +538,7 @@ Get-Content $resultsFile | ForEach-Object { } "*| BasicLuaRunner4*" { Write-Host "** BasicLuaRunner4 Allocated Value test" - $foundBasicLuaRunner4AllocatedValue = ParseValueFromResults $line $allocatedLuaColumn + $foundBasicLuaRunner4AllocatedValue = ParseValueFromResults $line $allocatedColumn $currentResults = AnalyzeResult $foundBasicLuaRunner4AllocatedValue $expectedBasicLuaRunner4AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false From 83169ef3fd5f05e496036f4511094ce6302ff3f5 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 9 Oct 2024 15:26:19 -0700 Subject: [PATCH 10/44] Updated config values --- .../CI_BDN_Config_Resp.RespLuaRunnerStress.json | 6 +++--- .../CI_BDN_Config_Resp.RespLuaStress.json | 2 +- .../CI_BDN_Config_Resp.RespParseStress.json | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json index 3ee86d9ac5..9f3538e7d6 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -2,13 +2,13 @@ "configuration": "Release", "framework": "net8.0", "filter": "*LuaRunner*", - "expectedBasicLuaRunner1MeanValue_linux": 200, - "expectedBasicLuaRunner2MeanValue_linux": 337, + "expectedBasicLuaRunner1MeanValue_linux": 182, + "expectedBasicLuaRunner2MeanValue_linux": 301, "expectedBasicLuaRunner3MeanValue_linux": 1473, "expectedBasicLuaRunner4MeanValue_linux": 3754, "expectedBasicLuaRunner1MeanValue_win": 115, - "expectedBasicLuaRunner2MeanValue_win": 173, + "expectedBasicLuaRunner2MeanValue_win": 219, "expectedBasicLuaRunner3MeanValue_win": 686, "expectedBasicLuaRunner4MeanValue_win": 2711, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json index 67dda1726d..fdb14629c0 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -3,7 +3,7 @@ "framework": "net8.0", "filter": "*LuaStress*", - "expectedBasicLuaStress1MeanValue_linux": 187, + "expectedBasicLuaStress1MeanValue_linux": 162, "expectedBasicLuaStress2MeanValue_linux": 284, "expectedBasicLuaStress3MeanValue_linux": 1407, "expectedBasicLuaStress4MeanValue_linux": 2733, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json index cc17e7871b..80593e8cf1 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json @@ -21,14 +21,14 @@ "expectedInLinePingMeanValue_win": 2.2, "expectedSETMeanValue_win": 18, - "expectedSETEXMeanValue_win": 22, - "expectedGETMeanValue_win": 12, + "expectedSETEXMeanValue_win": 25, + "expectedGETMeanValue_win": 14, "expectedIncrementMeanValue_win": 24, - "expectedZAddRemMeanValue_win": 121, - "expectedLPushPopMeanValue_win": 105, - "expectedSAddRemMeanValue_win": 101, - "expectedHSetDelMeanValue_win": 130, - "expectedMyDictSetGetMeanValue_win": 187, + "expectedZAddRemMeanValue_win": 144, + "expectedLPushPopMeanValue_win": 128, + "expectedSAddRemMeanValue_win": 124, + "expectedHSetDelMeanValue_win": 169, + "expectedMyDictSetGetMeanValue_win": 250, "expectedZAddRemAllocatedValue_win": 23552, "expectedLPushPopAllocatedValue_win": 18432, From cd17683ba99dd2dc7e36ec3d88b3c09392fca072 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 9 Oct 2024 15:37:38 -0700 Subject: [PATCH 11/44] Minor change to YML (remove pull requests) and updated LuaRunner config value --- .github/workflows/ci-bdnbenchmark.yml | 3 --- .../ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index aab153b91c..ae028e3342 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -11,7 +11,6 @@ jobs: name: Check for changes runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code permissions: - pull-requests: read deployments: write #permission to deploy GitHub pages website contents: write # permission to update benchmark contents in gh-pages branch steps: @@ -102,6 +101,4 @@ jobs: # Need for both commit comment and charts output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - - # This is for Charts on GitHub pages auto-push: true diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json index 9f3538e7d6..575c36cb3a 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -8,7 +8,7 @@ "expectedBasicLuaRunner4MeanValue_linux": 3754, "expectedBasicLuaRunner1MeanValue_win": 115, - "expectedBasicLuaRunner2MeanValue_win": 219, + "expectedBasicLuaRunner2MeanValue_win": 180, "expectedBasicLuaRunner3MeanValue_win": 686, "expectedBasicLuaRunner4MeanValue_win": 2711, From b334f65fbb21b7e78384bcd20563ca072dcc2761 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 9 Oct 2024 15:55:39 -0700 Subject: [PATCH 12/44] One more config value fix --- .../ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json index fdb14629c0..227232581b 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -3,7 +3,7 @@ "framework": "net8.0", "filter": "*LuaStress*", - "expectedBasicLuaStress1MeanValue_linux": 162, + "expectedBasicLuaStress1MeanValue_linux": 175, "expectedBasicLuaStress2MeanValue_linux": 284, "expectedBasicLuaStress3MeanValue_linux": 1407, "expectedBasicLuaStress4MeanValue_linux": 2733, From 4df29ee584128627e7d378891c196f0520ba3dea Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 9 Oct 2024 16:21:23 -0700 Subject: [PATCH 13/44] Added a task to do the push --- .github/workflows/ci-bdnbenchmark.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index ae028e3342..cf55dbae5a 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -101,4 +101,8 @@ jobs: # Need for both commit comment and charts output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true + auto-push: false + + # Run `github-action-benchmark` action + - name: Push benchmark results + run: git push 'https://darrenge:${{ secrets.GITHUB_TOKEN }}@github.com/microsoft/garnet.git' gh-pages:gh-pages From 82f6f15dc28a23999f557c9a4a7d1fdcc208d1f7 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 9 Oct 2024 17:14:40 -0700 Subject: [PATCH 14/44] Putting back to push chart in the chart task and not a separate one --- .github/workflows/ci-bdnbenchmark.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index cf55dbae5a..ae028e3342 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -101,8 +101,4 @@ jobs: # Need for both commit comment and charts output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: false - - # Run `github-action-benchmark` action - - name: Push benchmark results - run: git push 'https://darrenge:${{ secrets.GITHUB_TOKEN }}@github.com/microsoft/garnet.git' gh-pages:gh-pages + auto-push: true From 58038d1ea21c0ffa556d4a6f0eff858b0ab21075 Mon Sep 17 00:00:00 2001 From: darrenge Date: Thu, 10 Oct 2024 10:32:02 -0700 Subject: [PATCH 15/44] Moved the permissions to full workflow --- .github/workflows/ci-bdnbenchmark.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index ae028e3342..95e09a7586 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -6,13 +6,14 @@ env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_NOLOGO: true +permissions: + deployments: write #permission to deploy GitHub pages website + contents: write # permission to update benchmark contents in gh-pages branch + jobs: changes: name: Check for changes runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code - permissions: - deployments: write #permission to deploy GitHub pages website - contents: write # permission to update benchmark contents in gh-pages branch steps: - name: Check out code uses: actions/checkout@v4 @@ -51,6 +52,7 @@ jobs: - name: Run BDN.benchmark Perf Test run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }} shell: pwsh + continue-on-error: true - name: Upload test results to artifacts uses: actions/upload-artifact@v4 From 5875d87449a8706163ce3120c42009af605faf71 Mon Sep 17 00:00:00 2001 From: darrenge Date: Thu, 10 Oct 2024 10:53:07 -0700 Subject: [PATCH 16/44] Just minor to add commits --- .github/workflows/ci-bdnbenchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 95e09a7586..211a63ed6f 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -7,7 +7,7 @@ env: DOTNET_NOLOGO: true permissions: - deployments: write #permission to deploy GitHub pages website + deployments: write # permission to deploy GitHub pages website contents: write # permission to update benchmark contents in gh-pages branch jobs: From d69a07b30fe9500cf7cd13d175f016fd14f4f78b Mon Sep 17 00:00:00 2001 From: darrenge Date: Thu, 10 Oct 2024 14:28:01 -0700 Subject: [PATCH 17/44] Adding actions: write permit since actions default to read only --- .github/workflows/ci-bdnbenchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 211a63ed6f..bd3e7fbe51 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -7,6 +7,7 @@ env: DOTNET_NOLOGO: true permissions: + actions: write deployments: write # permission to deploy GitHub pages website contents: write # permission to update benchmark contents in gh-pages branch From 68d0972ccb4255528d1174fb769ee0270d599155 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 14 Oct 2024 14:02:46 -0700 Subject: [PATCH 18/44] Set up the commit comment for BDN as well as summary to show current vs previous BDN perf numbers --- .github/workflows/ci-bdnbenchmark.yml | 20 +++++++++---------- ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 2 +- .../CI_BDN_Config_Resp.RespLuaStress.json | 2 ++ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index bd3e7fbe51..982eb7531b 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -8,7 +8,7 @@ env: permissions: actions: write - deployments: write # permission to deploy GitHub pages website + deployments: write #permission to deploy GitHub pages website contents: write # permission to update benchmark contents in gh-pages branch jobs: @@ -62,6 +62,7 @@ jobs: path: | ./test/BDNPerfTests/results if: ${{ always() }} + - name: Upload Error Log to artifacts uses: actions/upload-artifact@v4 with: @@ -81,27 +82,24 @@ jobs: - name: Store benchmark result for commit comment uses: benchmark-action/github-action-benchmark@v1 with: - # What benchmark tool created output file tool: 'benchmarkdotnet' - - # Need for both commit comment and charts output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - + alert-threshold: '200%' external-data-json-path: ./cache/benchmark-data.json - fail-on-alert: true comment-on-alert: true - summary-always: true - auto-push: false + fail-on-alert: true + alert-comment-cc-users: '@darrenge' + summary-always: false + auto-push: false # needs to be false if using external-data-json-path # Run `github-action-benchmark` action - name: Store benchmark result for charts uses: benchmark-action/github-action-benchmark@v1 with: - # What benchmark tool created output file + name: BDNBenchmark - ${{matrix.test}} (os ${{matrix.os}}, framework ${{matrix.framework}}, configuration ${{matrix.configuration}}) tool: 'benchmarkdotnet' - - # Need for both commit comment and charts output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} + summary-always: true auto-push: true diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json index 575c36cb3a..5953e785f1 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -3,7 +3,7 @@ "framework": "net8.0", "filter": "*LuaRunner*", "expectedBasicLuaRunner1MeanValue_linux": 182, - "expectedBasicLuaRunner2MeanValue_linux": 301, + "expectedBasicLuaRunner2MeanValue_linux": 325, "expectedBasicLuaRunner3MeanValue_linux": 1473, "expectedBasicLuaRunner4MeanValue_linux": 3754, diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json index 227232581b..18e559ba25 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -7,6 +7,7 @@ "expectedBasicLuaStress2MeanValue_linux": 284, "expectedBasicLuaStress3MeanValue_linux": 1407, "expectedBasicLuaStress4MeanValue_linux": 2733, + "expectedBasicLuaStress1AllocatedValue_linux": 24, "expectedBasicLuaStress2AllocatedValue_linux": 144, "expectedBasicLuaStress3AllocatedValue_linux": 328, @@ -16,6 +17,7 @@ "expectedBasicLuaStress2MeanValue_win": 180, "expectedBasicLuaStress3MeanValue_win": 660, "expectedBasicLuaStress4MeanValue_win": 1805, + "expectedBasicLuaStress1AllocatedValue_win": 24, "expectedBasicLuaStress2AllocatedValue_win": 144, "expectedBasicLuaStress3AllocatedValue_win": 328, From f8ddb6cefd5c0c6ccc561469c8f6a16bd2c9c055 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 14 Oct 2024 16:42:25 -0700 Subject: [PATCH 19/44] A few fixes --- .github/workflows/ci-bdnbenchmark.yml | 24 ++++++++++++------------ test/BDNPerfTests/run_bdnperftest.ps1 | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 982eb7531b..40f68938a6 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -79,27 +79,27 @@ jobs: key: ${{ runner.os }}-benchmark # Run `github-action-benchmark` action - - name: Store benchmark result for commit comment + - name: Store benchmark result for charts uses: benchmark-action/github-action-benchmark@v1 with: + name: BDNBenchmark - ${{matrix.test}} (os ${{matrix.os}}, framework ${{matrix.framework}}, configuration ${{matrix.configuration}}) tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - external-data-json-path: ./cache/benchmark-data.json - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@darrenge' - summary-always: false - auto-push: false # needs to be false if using external-data-json-path + summary-always: true + auto-push: true # Run `github-action-benchmark` action - - name: Store benchmark result for charts + - name: Store benchmark result for commit comment uses: benchmark-action/github-action-benchmark@v1 with: - name: BDNBenchmark - ${{matrix.test}} (os ${{matrix.os}}, framework ${{matrix.framework}}, configuration ${{matrix.configuration}}) tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - summary-always: true - auto-push: true + alert-threshold: '115%' + external-data-json-path: ./cache/benchmark-data.json + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@darrenge' + summary-always: false + auto-push: false # needs to be false if using external-data-json-path diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index f58687302c..ca7255589a 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -42,7 +42,7 @@ function AnalyzeResult { # Check if the actual value is within the bounds if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " Write-Host "** " return $true # the values are close enough } @@ -79,6 +79,7 @@ param ($ResultsLine, $columnNum) $foundValue = $foundValue.Trim(' us') $foundValue = $foundValue.Trim(' ns') $foundValue = $foundValue.Trim(' B') + $foundValue = $foundValue.Trim(' m') return $foundValue } From 8e058650200541b30be506a3eca8445ee1cfd2e0 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 13:08:16 -0700 Subject: [PATCH 20/44] Updated BDN perf YML, config files and ps1 script --- .github/workflows/ci-bdnbenchmark.yml | 53 ++-- .../ConfigFiles/CI_BDN_Config_Lua.json | 27 ++ ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 30 --- .../CI_BDN_Config_Resp.RespLuaStress.json | 30 --- .../CI_BDN_Config_Resp.RespParseStress.json | 43 --- ...on => CI_BDN_Config_RespClusterBench.json} | 6 +- ...I_BDN_Config_RespClusterMigrateBench.json} | 4 +- .../CI_BDN_Config_RespParseStress.json | 29 ++ ...=> CI_BDN_Config_RespTsavoriteStress.json} | 8 +- test/BDNPerfTests/run_bdnperftest.ps1 | 254 +++++------------- 10 files changed, 160 insertions(+), 324 deletions(-) create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_Cluster.RespClusterBench.json => CI_BDN_Config_RespClusterBench.json} (62%) rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_Cluster.RespClusterMigrateBench.json => CI_BDN_Config_RespClusterMigrateBench.json} (68%) create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_Resp.RespTsavoriteStress.json => CI_BDN_Config_RespTsavoriteStress.json} (52%) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 40f68938a6..acccfe4bc2 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -6,15 +6,13 @@ env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_NOLOGO: true -permissions: - actions: write - deployments: write #permission to deploy GitHub pages website - contents: write # permission to update benchmark contents in gh-pages branch - jobs: changes: name: Check for changes runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code + permissions: + pull-requests: read + contents: read steps: - name: Check out code uses: actions/checkout@v4 @@ -36,10 +34,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-latest, windows-latest ] + #ORIGINALos: [ ubuntu-latest, windows-latest ] + os: [ windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] - test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] + #ORIGINAL test: [ 'CI_BDN_Config_RespParseStress', 'CI_BDN_Config_RespTsavoriteStress', 'CI_BDN_Config_RespClusterBench', 'CI_BDN_Config_RespClusterMigrateBench', 'CI_BDN_Config_Lua' ] + test: [ 'CI_BDN_Config_RespParseStress' ] steps: - name: Check out code uses: actions/checkout@v4 @@ -53,7 +53,6 @@ jobs: - name: Run BDN.benchmark Perf Test run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }} shell: pwsh - continue-on-error: true - name: Upload test results to artifacts uses: actions/upload-artifact@v4 @@ -62,7 +61,6 @@ jobs: path: | ./test/BDNPerfTests/results if: ${{ always() }} - - name: Upload Error Log to artifacts uses: actions/upload-artifact@v4 with: @@ -79,27 +77,28 @@ jobs: key: ${{ runner.os }}-benchmark # Run `github-action-benchmark` action - - name: Store benchmark result for charts + - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - name: BDNBenchmark - ${{matrix.test}} (os ${{matrix.os}}, framework ${{matrix.framework}}, configuration ${{matrix.configuration}}) + # What benchmark tool created output file tool: 'benchmarkdotnet' - output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - summary-always: true - auto-push: true + # Where the output from the benchmark tool is stored + # This can't find the file + #output-file-path: ./test/BDNPerfTests/results/Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}.results + + # This gives error: Error: Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: Unexpected token '/', "// Validat"... is not valid JSON + #output-file-path: ./test/BDNPerfTests/results/CI_BDN_Config_RespParseStress_Windows.results - # Run `github-action-benchmark` action - - name: Store benchmark result for commit comment - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'benchmarkdotnet' - output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '115%' + # Error: Command 'git' failed with args '-c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com -c http.https://github.com/.extraheader= switch gh-pages': fatal: invalid reference: gh-pages + output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.Resp.RespParseStress-report-full-compressed.json + + # Where the previous data file is stored external-data-json-path: ./cache/benchmark-data.json - comment-on-alert: true + # Workflow will fail when an alert happens fail-on-alert: true - alert-comment-cc-users: '@darrenge' - summary-always: false - auto-push: false # needs to be false if using external-data-json-path + + # Take these out as not part of minimal + # github-token: ${{ secrets.GITHUB_TOKEN }} + # auto-push: false + # skip-fetch-gh-pages: true + # summary-always: true diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json new file mode 100644 index 0000000000..d40425bf3a --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json @@ -0,0 +1,27 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*Lua*", + "expectedBasicLuaRunner1MeanValue_linux": 186, + "expectedBasicLuaRunner2MeanValue_linux": 305, + "expectedBasicLuaRunner3MeanValue_linux": 1473, + "expectedBasicLuaRunner4MeanValue_linux": 3754, + "expectedBasicLuaRunner1MeanValue_win": 111, + "expectedBasicLuaRunner2MeanValue_win": 173, + "expectedBasicLuaRunner3MeanValue_win": 686, + "expectedBasicLua4RunnerMeanValue_win": 2711, + "expectedBasicLua1MeanValue_linux": 169, + "expectedBasicLua2MeanValue_linux": 284, + "expectedBasicLua3MeanValue_linux": 1407, + "expectedBasicLua4MeanValue_linux": 2733, + "expectedBasicLua1MeanValue_win": 107, + "expectedBasicLua2MeanValue_win": 180, + "expectedBasicLua3MeanValue_win": 660, + "expectedBasicLua4MeanValue_win": 1805, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json deleted file mode 100644 index 5953e785f1..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*LuaRunner*", - "expectedBasicLuaRunner1MeanValue_linux": 182, - "expectedBasicLuaRunner2MeanValue_linux": 325, - "expectedBasicLuaRunner3MeanValue_linux": 1473, - "expectedBasicLuaRunner4MeanValue_linux": 3754, - - "expectedBasicLuaRunner1MeanValue_win": 115, - "expectedBasicLuaRunner2MeanValue_win": 180, - "expectedBasicLuaRunner3MeanValue_win": 686, - "expectedBasicLuaRunner4MeanValue_win": 2711, - - - "expectedBasicLuaRunner1AllocatedValue_linux": 24, - "expectedBasicLuaRunner2AllocatedValue_linux": 144, - "expectedBasicLuaRunner3AllocatedValue_linux": 240, - "expectedBasicLuaRunner4AllocatedValue_linux": 776, - - "expectedBasicLuaRunner1AllocatedValue_win": 24, - "expectedBasicLuaRunner2AllocatedValue_win": 144, - "expectedBasicLuaRunner3AllocatedValue_win": 240, - "expectedBasicLuaRunner4AllocatedValue_win": 776, - - "acceptableMeanRange": 10, - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json deleted file mode 100644 index 18e559ba25..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*LuaStress*", - - "expectedBasicLuaStress1MeanValue_linux": 175, - "expectedBasicLuaStress2MeanValue_linux": 284, - "expectedBasicLuaStress3MeanValue_linux": 1407, - "expectedBasicLuaStress4MeanValue_linux": 2733, - - "expectedBasicLuaStress1AllocatedValue_linux": 24, - "expectedBasicLuaStress2AllocatedValue_linux": 144, - "expectedBasicLuaStress3AllocatedValue_linux": 328, - "expectedBasicLuaStress4AllocatedValue_linux": 832, - - "expectedBasicLuaStress1MeanValue_win": 107, - "expectedBasicLuaStress2MeanValue_win": 180, - "expectedBasicLuaStress3MeanValue_win": 660, - "expectedBasicLuaStress4MeanValue_win": 1805, - - "expectedBasicLuaStress1AllocatedValue_win": 24, - "expectedBasicLuaStress2AllocatedValue_win": 144, - "expectedBasicLuaStress3AllocatedValue_win": 328, - "expectedBasicLuaStress4AllocatedValue_win": 832, - - "acceptableMeanRange": 10, - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json deleted file mode 100644 index 80593e8cf1..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespParseStress*", - "expectedInLinePingMeanValue_linux": 2.2, - "expectedSETMeanValue_linux": 19, - "expectedSETEXMeanValue_linux": 26, - "expectedGETMeanValue_linux": 13, - "expectedIncrementMeanValue_linux": 27, - "expectedZAddRemMeanValue_linux": 186, - "expectedLPushPopMeanValue_linux": 166, - "expectedSAddRemMeanValue_linux": 164, - "expectedHSetDelMeanValue_linux": 196, - "expectedMyDictSetGetMeanValue_linux": 248, - - "expectedZAddRemAllocatedValue_linux": 23552, - "expectedLPushPopAllocatedValue_linux": 18432, - "expectedSAddRemAllocatedValue_linux": 16384, - "expectedHSetDelAllocatedValue_linux": 55297, - "expectedMyDictSetGetAllocatedValue_linux": 30720, - - "expectedInLinePingMeanValue_win": 2.2, - "expectedSETMeanValue_win": 18, - "expectedSETEXMeanValue_win": 25, - "expectedGETMeanValue_win": 14, - "expectedIncrementMeanValue_win": 24, - "expectedZAddRemMeanValue_win": 144, - "expectedLPushPopMeanValue_win": 128, - "expectedSAddRemMeanValue_win": 124, - "expectedHSetDelMeanValue_win": 169, - "expectedMyDictSetGetMeanValue_win": 250, - - "expectedZAddRemAllocatedValue_win": 23552, - "expectedLPushPopAllocatedValue_win": 18432, - "expectedSAddRemAllocatedValue_win": 16384, - "expectedHSetDelAllocatedValue_win": 55297, - "expectedMyDictSetGetAllocatedValue_win": 30720, - - "acceptableMeanRange": 10, - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json similarity index 62% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json index 53bca030b5..4b38979781 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json @@ -1,4 +1,7 @@ { + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterBench*", @@ -9,8 +12,7 @@ "expectedGETMeanValue_win": 24, "expectedSETMeanValue_win": 24, "expectedMGETMeanValue_win": 19, - "expectedMSETMeanValue_win": 19, - + "expectedMSETMeanValue_win": 17, "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json similarity index 68% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json index 9caf6bd3f3..5f4387832b 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json @@ -1,4 +1,7 @@ { + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterMigrateBench*", @@ -10,7 +13,6 @@ "expectedSETMeanValue_win": 47, "expectedMGETMeanValue_win": 40, "expectedMSETMeanValue_win": 37, - "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json new file mode 100644 index 0000000000..0995827e35 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json @@ -0,0 +1,29 @@ +{ + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespParseStress*", + "expectedInLinePingMeanValue_linux": 2.4, + "expectedSETMeanValue_linux": 19, + "expectedSETEXMeanValue_linux": 26, + "expectedGETMeanValue_linux": 14, + "expectedZAddRemMeanValue_linux": 186, + "expectedLPushPopMeanValue_linux": 200, + "expectedSAddRemMeanValue_linux": 172, + "expectedHSetDelMeanValue_linux": 200, + "expectedMyDictSetGetMeanValue_linux": 235, + "expectedInLinePingMeanValue_win": 2.4, + "expectedSETMeanValue_win": 19, + "expectedSETEXMeanValue_win": 26, + "expectedGETMeanValue_win": 14, + "expectedZAddRemMeanValue_win": 141, + "expectedLPushPopMeanValue_win": 145, + "expectedSAddRemMeanValue_win": 130, + "expectedHSetDelMeanValue_win": 160, + "expectedMyDictSetGetMeanValue_win": 242, + "acceptableMeanRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json similarity index 52% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json index 337a3268a7..ff0c0333c1 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json @@ -1,14 +1,16 @@ { + "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", + "ExpectedCoresToTestOn_win": 2, + "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespTsavoriteStress*", "expectedGETMeanValue_linux": 21, "expectedSETMeanValue_linux": 19, - "expectedIncrementMeanValue_linux": 27, + "expectedIncrMeanValue_linux": 29, "expectedGETMeanValue_win": 20, "expectedSETMeanValue_win": 17, - "expectedIncrementMeanValue_win": 26, - + "expectedIncrMeanValue_win": 26, "acceptableMeanRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index ca7255589a..b4ca30f493 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -6,20 +6,20 @@ Script to test for performance regressions using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. - CI_BDN_Config_Resp.RespParseStress.json + CI_BDN_Config_RespParseStress.json NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% .EXAMPLE ./run_bdnperftest.ps1 - ./run_bdnperftest.ps1 CI_BDN_Config_Resp.RespParseStress.json + ./run_bdnperftest.ps1 CI_BDN_Config_RespParseStress.json #> # Send the config file for the benchmark. Defaults to a simple one param ( - [string]$configFile = "CI_BDN_Config_Resp.RespParseStress.json" + [string]$configFile = "CI_BDN_Config_RespParseStress.json" ) $OFS = "`r`n" @@ -39,20 +39,20 @@ function AnalyzeResult { [double] $LowerBound = $expectedResultValue * (1 - $Tolerance) [double] $UpperBound = $expectedResultValue * (1 + $Tolerance) [double] $dblfoundResultValue = $foundResultValue - + # Check if the actual value is within the bounds if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** ** PASS! ** Mean Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Mean Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Mean Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } return $false # the values are too different @@ -74,17 +74,15 @@ param ($ResultsLine, $columnNum) # Remove the leading and trailing pipes and split the string by '|' $columns = $ResultsLine.Trim('|').Split('|') $column = $columns | ForEach-Object { $_.Trim() } - $foundValue = $column[$columnNum] - - $foundValue = $foundValue.Trim(' us') + $foundValue = $column[$columnNum].Trim(' us') $foundValue = $foundValue.Trim(' ns') - $foundValue = $foundValue.Trim(' B') - $foundValue = $foundValue.Trim(' m') return $foundValue } + + # ******** BEGIN MAIN ********* # Get base path since paths can differ from machine to machine $pathstring = $pwd.Path @@ -101,11 +99,6 @@ if ($configFile -notlike "*.json") { $configFile += ".json" } -# This is special case that allows passing test without specifying CI_BDN_Confi_ at the beginning - need for perf test -if ($configFile -notlike "CI_BDN_Config_*") { - $configFile = "CI_BDN_Config_" + $configFile -} - # Read the test config file and convert the JSON to a PowerShell object $fullConfiFileAndPath = "ConfigFiles/$configFile" if (-not (Test-Path -Path $fullConfiFileAndPath)) { @@ -121,6 +114,27 @@ if ($IsLinux) { $CurrentOS = "Linux" } + # Calculate number of cores on the test machine - used to verify the config file settings as the config settings will vary based on machine config +if ($IsLinux) { + $sockets = [int]$(lscpu | grep -E '^Socket' | awk '{print $2}') + $coresPerSocket = [int]$(lscpu | grep -E '^Core' | awk '{print $4}') + $NumberOfCores = ($sockets * $coresPerSocket) + + $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_linux +} +else { + $NumberOfCores = (Get-CimInstance -ClassName Win32_Processor).NumberOfCores | Measure-Object -Sum | Select-Object -ExpandProperty Sum + + $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_win +} + +# To get accurate comparison of found vs expected values, double check to make sure config settings for Number of Cores of the test machine are what is specified in the test config file +if ($ExpectedCoresToTestOn -ne $NumberOfCores) { + Write-Output -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." + ##&#&# DEBUG #*#*#* Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." +# exit +} + Write-Host "************** Start BDN.benchmark ********************" Write-Host " " @@ -129,7 +143,10 @@ $configuration = $object.configuration $framework = $object.framework $filter = $object.filter $meanColumn = "1" -$allocatedColumn = "-1" # Number of columns can differ a bit, but know the allocated one is always the last one + +#*#*#* DEBUG +# TO DO: Add "Allocated" to values that are checked +#*#*#* DEBUG # Set the expected values based on the OS if ($IsLinux) { @@ -138,39 +155,22 @@ if ($IsLinux) { $expectedSetMeanValue = $object.expectedSETMeanValue_linux $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_linux $expectedGetMeanValue = $object.expectedGETMeanValue_linux - $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_linux $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_linux $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_linux $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_linux $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_linux - $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_linux - $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_linux - $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_linux - $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_linux - $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_linux - $expectedMGetMeanValue = $object.expectedMGETMeanValue_linux $expectedMSetMeanValue = $object.expectedMSETMeanValue_linux - $expectedIncrMeanValue = $object.expectedIncrementMeanValue_linux - - $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_linux - $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_linux - $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_linux - $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_linux - $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_linux - $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_linux - $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_linux - $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_linux - + $expectedIncrMeanValue = $object.expectedIncrMeanValue_linux + $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_linux + $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_linux + $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_linux + $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_linux $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_linux $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_linux $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_linux - $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_linux - $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_linux - $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_linux - $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_linux + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux } else { # Windows expected values @@ -178,44 +178,26 @@ else { $expectedSetMeanValue = $object.expectedSETMeanValue_win $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_win $expectedGetMeanValue = $object.expectedGETMeanValue_win - $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_win $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_win $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_win $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_win $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_win - $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_win - $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_win - $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_win - $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_win - $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_win - $expectedMGetMeanValue = $object.expectedMGETMeanValue_win $expectedMSetMeanValue = $object.expectedMSETMeanValue_win - $expectedIncrMeanValue = $object.expectedIncrementMeanValue_win - - $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_win - $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_win - $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_win - $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_win - $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_win - $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_win - $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_win - $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_win - + $expectedIncrMeanValue = $object.expectedIncrMeanValue_win + $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_win + $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_win + $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_win + $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_win $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_win $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_win $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_win - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_win - $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_win - $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_win - $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_win - $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_win + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_win } # percent allowed variance when comparing expected vs actual found value - same for linux and windows. $acceptableMeanRange = $object.acceptableMeanRange -$acceptableAllocatedRange = $object.acceptableAllocatedRange # Set up the results dir and errorlog dir $resultsDir = "$basePath/test/BDNPerfTests/results" @@ -343,46 +325,6 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } - "*| ZAddRem*" { - Write-Host "** ZAddRem Allocated Value test" - $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRemAllocatedValue $expectedZAddRemAllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| LPushPop*" { - Write-Host "** LPushPop Allocated Value test" - $foundLPushPopAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPopAllocatedValue $expectedLPushPopAllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| SAddRem*" { - Write-Host "** SAddRem Allocated Value test" - $foundSAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRemAllocatedValue $expectedSAddRemAllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| HSetDel*" { - Write-Host "** HSetDel Allocated Value test" - $foundHSetDelAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDelAllocatedValue $expectedHSetDelAllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MyDictSetGet*" { - Write-Host "** MyDictSetGet Allocated Value test" - $foundMyDictSetGetAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGetAllocatedValue $expectedMyDictSetGetAllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } "*| Incr*" { Write-Host "** Incr Mean Value test" $foundIncrMeanValue = ParseValueFromResults $line $meanColumn @@ -407,42 +349,41 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } - "*| BasicLuaStress1*" { - Write-Host "** BasicLuaStress1 Mean Value test" - $foundBasicLuaStress1MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress1MeanValue $expectedBasicLuaStress1MeanValue $acceptableMeanRange $true + "*| BasicLua1*" { + Write-Host "** BasicLua1 Mean Value test" + $foundBasicLua1MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua1MeanValue $expectedBasicLua1MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLuaStress2*" { - Write-Host "** BasicLuaStress2 Mean Value test" - $foundBasicLuaStress2MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress2MeanValue $expectedBasicLuaStress2MeanValue $acceptableMeanRange $true + "*| BasicLua2*" { + Write-Host "** BasicLua2 Mean Value test" + $foundBasicLua2MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua2MeanValue $expectedBasicLua2MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } <# # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. - "*| BasicLuaStress3*" { - Write-Host "** BasicLuaStress3 Mean Value test" - $foundBasicLuaStress3MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress3MeanValue $expectedBasicLuaStress3MeanValue $acceptableMeanRange $true + "*| BasicLua3*" { + Write-Host "** BasicLua3 Mean Value test" + $foundBasicLua3MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua3MeanValue $expectedBasicLua3MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLuaStress4*" { - Write-Host "** BasicLuaStress4 Mean Value test" - $foundBasicLuaStress4MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress4MeanValue $expectedBasicLuaStress4MeanValue $acceptableMeanRange $true + "*| BasicLua4*" { + Write-Host "** BasicLua4 Mean Value test" + $foundBasicLua4MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLua4MeanValue $expectedBasicLua4MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } #> - "*| BasicLuaRunner1*" { Write-Host "** BasicLuaRunner1 Mean Value test" $foundBasicLuaRunner1MeanValue = ParseValueFromResults $line $meanColumn @@ -479,72 +420,6 @@ Get-Content $resultsFile | ForEach-Object { } } #> - - "*| BasicLuaStress1*" { - Write-Host "** BasicLuaStress1 Allocated Value test" - $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress1AllocatedValue $expectedBasicLuaStress1AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress2*" { - Write-Host "** BasicLuaStress2 Allocated Value test" - $foundBasicLuaStress2AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress2AllocatedValue $expectedBasicLuaStress2AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress3*" { - Write-Host "** BasicLuaStress3 Allocated Value test" - $foundBasicLuaStress3AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress3AllocatedValue $expectedBasicLuaStress3AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress4*" { - Write-Host "** BasicLuaStress4 Allocated Value test" - $foundBasicLuaStress4AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress4AllocatedValue $expectedBasicLuaStress4AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaRunner1*" { - Write-Host "** BasicLuaRunner1 Allocated Value test" - $foundBasicLuaRunner1AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner1AllocatedValue $expectedBasicLuaRunner1AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaRunner2*" { - Write-Host "** BasicLuaRunner2 Allocated Value test" - $foundBasicLuaRunner2AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner2AllocatedValue $expectedBasicLuaRunner2AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - - "*| BasicLuaRunner3*" { - Write-Host "** BasicLuaRunner3 Allocated Value test" - $foundBasicLuaRunner3AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner3AllocatedValue $expectedBasicLuaRunner3AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaRunner4*" { - Write-Host "** BasicLuaRunner4 Allocated Value test" - $foundBasicLuaRunner4AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner4AllocatedValue $expectedBasicLuaRunner4AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } } } @@ -559,3 +434,6 @@ if ($testSuiteResult) { } Write-Output "** " Write-Output "************************" + + +#./test/BDNPerfTests/results/${{ matrix.test }}_${{ matrix.os }}.json \ No newline at end of file From 400872685cf465e258c8501fdccd937fb805fda5 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 13:10:45 -0700 Subject: [PATCH 21/44] Versioning issue with BDN test files, fixed now --- .github/workflows/ci-bdnbenchmark.yml | 65 +++-- ..._BDN_Config_Cluster.RespClusterBench.json} | 8 +- ...nfig_Cluster.RespClusterMigrateBench.json} | 6 +- .../ConfigFiles/CI_BDN_Config_Lua.json | 27 -- ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 30 +++ .../CI_BDN_Config_Resp.RespLuaStress.json | 30 +++ .../CI_BDN_Config_Resp.RespParseStress.json | 43 +++ ...I_BDN_Config_Resp.RespTsavoriteStress.json | 16 ++ .../CI_BDN_Config_RespParseStress.json | 29 -- .../CI_BDN_Config_RespTsavoriteStress.json | 18 -- test/BDNPerfTests/run_bdnperftest.ps1 | 254 +++++++++++++----- 11 files changed, 342 insertions(+), 184 deletions(-) rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_RespClusterBench.json => CI_BDN_Config_Cluster.RespClusterBench.json} (58%) rename test/BDNPerfTests/ConfigFiles/{CI_BDN_Config_RespClusterMigrateBench.json => CI_BDN_Config_Cluster.RespClusterMigrateBench.json} (64%) delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index acccfe4bc2..cbee4641d4 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -6,13 +6,15 @@ env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_NOLOGO: true +permissions: + actions: write + deployments: write #permission to deploy GitHub pages website + contents: write # permission to update benchmark contents in gh-pages branch + jobs: changes: name: Check for changes runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code - permissions: - pull-requests: read - contents: read steps: - name: Check out code uses: actions/checkout@v4 @@ -34,12 +36,10 @@ jobs: strategy: fail-fast: false matrix: - #ORIGINALos: [ ubuntu-latest, windows-latest ] - os: [ windows-latest ] + os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] - #ORIGINAL test: [ 'CI_BDN_Config_RespParseStress', 'CI_BDN_Config_RespTsavoriteStress', 'CI_BDN_Config_RespClusterBench', 'CI_BDN_Config_RespClusterMigrateBench', 'CI_BDN_Config_Lua' ] - test: [ 'CI_BDN_Config_RespParseStress' ] + test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] steps: - name: Check out code uses: actions/checkout@v4 @@ -53,6 +53,7 @@ jobs: - name: Run BDN.benchmark Perf Test run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }} shell: pwsh + continue-on-error: true - name: Upload test results to artifacts uses: actions/upload-artifact@v4 @@ -61,6 +62,7 @@ jobs: path: | ./test/BDNPerfTests/results if: ${{ always() }} + - name: Upload Error Log to artifacts uses: actions/upload-artifact@v4 with: @@ -69,36 +71,29 @@ jobs: ./test/BDNPerfTests/errorlog if: ${{ always() }} - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - # Run `github-action-benchmark` action - - name: Store benchmark result + # Run `github-action-benchmark` action to get Commit Comment + - name: Store benchmark result for commit comment uses: benchmark-action/github-action-benchmark@v1 with: - # What benchmark tool created output file tool: 'benchmarkdotnet' - # Where the output from the benchmark tool is stored - # This can't find the file - #output-file-path: ./test/BDNPerfTests/results/Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}.results - - # This gives error: Error: Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: Unexpected token '/', "// Validat"... is not valid JSON - #output-file-path: ./test/BDNPerfTests/results/CI_BDN_Config_RespParseStress_Windows.results - - # Error: Command 'git' failed with args '-c user.name=github-action-benchmark -c user.email=github@users.noreply.github.com -c http.https://github.com/.extraheader= switch gh-pages': fatal: invalid reference: gh-pages - output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.Resp.RespParseStress-report-full-compressed.json - - # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json - # Workflow will fail when an alert happens + output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: '130%' + comment-on-alert: true fail-on-alert: true + alert-comment-cc-users: '@darrenge' + summary-always: false + auto-push: true + + # Run `github-action-benchmark` action for the Continuous Benchmarking Charts + - name: Store benchmark result for charts + uses: benchmark-action/github-action-benchmark@v1 + with: + name: ${{matrix.test}} (${{matrix.os}} ${{matrix.framework}} ${{matrix.configuration}}) + tool: 'benchmarkdotnet' + output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + summary-always: true + auto-push: true - # Take these out as not part of minimal - # github-token: ${{ secrets.GITHUB_TOKEN }} - # auto-push: false - # skip-fetch-gh-pages: true - # summary-always: true + diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json similarity index 58% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json index 4b38979781..19985d1bbf 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterBench*", @@ -12,8 +9,9 @@ "expectedGETMeanValue_win": 24, "expectedSETMeanValue_win": 24, "expectedMGETMeanValue_win": 19, - "expectedMSETMeanValue_win": 17, - "acceptableMeanRange": 10, + "expectedMSETMeanValue_win": 19, + + "acceptableMeanRange": 15, "acceptableError": ".1", "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json similarity index 64% rename from test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json rename to test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json index 5f4387832b..24be386912 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespClusterMigrateBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json @@ -1,7 +1,4 @@ { - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterMigrateBench*", @@ -13,7 +10,8 @@ "expectedSETMeanValue_win": 47, "expectedMGETMeanValue_win": 40, "expectedMSETMeanValue_win": 37, - "acceptableMeanRange": 10, + + "acceptableMeanRange": 15, "acceptableError": ".1", "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json deleted file mode 100644 index d40425bf3a..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Lua.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, - "configuration": "Release", - "framework": "net8.0", - "filter": "*Lua*", - "expectedBasicLuaRunner1MeanValue_linux": 186, - "expectedBasicLuaRunner2MeanValue_linux": 305, - "expectedBasicLuaRunner3MeanValue_linux": 1473, - "expectedBasicLuaRunner4MeanValue_linux": 3754, - "expectedBasicLuaRunner1MeanValue_win": 111, - "expectedBasicLuaRunner2MeanValue_win": 173, - "expectedBasicLuaRunner3MeanValue_win": 686, - "expectedBasicLua4RunnerMeanValue_win": 2711, - "expectedBasicLua1MeanValue_linux": 169, - "expectedBasicLua2MeanValue_linux": 284, - "expectedBasicLua3MeanValue_linux": 1407, - "expectedBasicLua4MeanValue_linux": 2733, - "expectedBasicLua1MeanValue_win": 107, - "expectedBasicLua2MeanValue_win": 180, - "expectedBasicLua3MeanValue_win": 660, - "expectedBasicLua4MeanValue_win": 1805, - "acceptableMeanRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json new file mode 100644 index 0000000000..75747dc80d --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -0,0 +1,30 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "*LuaRunner*", + "expectedBasicLuaRunner1MeanValue_linux": 182, + "expectedBasicLuaRunner2MeanValue_linux": 325, + "expectedBasicLuaRunner3MeanValue_linux": 1473, + "expectedBasicLuaRunner4MeanValue_linux": 3754, + + "expectedBasicLuaRunner1MeanValue_win": 115, + "expectedBasicLuaRunner2MeanValue_win": 180, + "expectedBasicLuaRunner3MeanValue_win": 686, + "expectedBasicLuaRunner4MeanValue_win": 2711, + + + "expectedBasicLuaRunner1AllocatedValue_linux": 24, + "expectedBasicLuaRunner2AllocatedValue_linux": 144, + "expectedBasicLuaRunner3AllocatedValue_linux": 240, + "expectedBasicLuaRunner4AllocatedValue_linux": 776, + + "expectedBasicLuaRunner1AllocatedValue_win": 24, + "expectedBasicLuaRunner2AllocatedValue_win": 144, + "expectedBasicLuaRunner3AllocatedValue_win": 240, + "expectedBasicLuaRunner4AllocatedValue_win": 776, + + "acceptableMeanRange": 15, + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json new file mode 100644 index 0000000000..92b6024bef --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -0,0 +1,30 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "*LuaStress*", + + "expectedBasicLuaStress1MeanValue_linux": 175, + "expectedBasicLuaStress2MeanValue_linux": 284, + "expectedBasicLuaStress3MeanValue_linux": 1407, + "expectedBasicLuaStress4MeanValue_linux": 2733, + + "expectedBasicLuaStress1AllocatedValue_linux": 24, + "expectedBasicLuaStress2AllocatedValue_linux": 144, + "expectedBasicLuaStress3AllocatedValue_linux": 328, + "expectedBasicLuaStress4AllocatedValue_linux": 832, + + "expectedBasicLuaStress1MeanValue_win": 107, + "expectedBasicLuaStress2MeanValue_win": 180, + "expectedBasicLuaStress3MeanValue_win": 660, + "expectedBasicLuaStress4MeanValue_win": 1805, + + "expectedBasicLuaStress1AllocatedValue_win": 24, + "expectedBasicLuaStress2AllocatedValue_win": 144, + "expectedBasicLuaStress3AllocatedValue_win": 328, + "expectedBasicLuaStress4AllocatedValue_win": 832, + + "acceptableMeanRange": 15, + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json new file mode 100644 index 0000000000..636e9a667c --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json @@ -0,0 +1,43 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespParseStress*", + "expectedInLinePingMeanValue_linux": 2.2, + "expectedSETMeanValue_linux": 19, + "expectedSETEXMeanValue_linux": 26, + "expectedGETMeanValue_linux": 13, + "expectedIncrementMeanValue_linux": 27, + "expectedZAddRemMeanValue_linux": 186, + "expectedLPushPopMeanValue_linux": 166, + "expectedSAddRemMeanValue_linux": 164, + "expectedHSetDelMeanValue_linux": 196, + "expectedMyDictSetGetMeanValue_linux": 248, + + "expectedZAddRemAllocatedValue_linux": 23552, + "expectedLPushPopAllocatedValue_linux": 18432, + "expectedSAddRemAllocatedValue_linux": 16384, + "expectedHSetDelAllocatedValue_linux": 55297, + "expectedMyDictSetGetAllocatedValue_linux": 30720, + + "expectedInLinePingMeanValue_win": 2.2, + "expectedSETMeanValue_win": 18, + "expectedSETEXMeanValue_win": 25, + "expectedGETMeanValue_win": 14, + "expectedIncrementMeanValue_win": 24, + "expectedZAddRemMeanValue_win": 144, + "expectedLPushPopMeanValue_win": 128, + "expectedSAddRemMeanValue_win": 124, + "expectedHSetDelMeanValue_win": 169, + "expectedMyDictSetGetMeanValue_win": 250, + + "expectedZAddRemAllocatedValue_win": 23552, + "expectedLPushPopAllocatedValue_win": 18432, + "expectedSAddRemAllocatedValue_win": 16384, + "expectedHSetDelAllocatedValue_win": 55297, + "expectedMyDictSetGetAllocatedValue_win": 30720, + + "acceptableMeanRange": 15, + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json new file mode 100644 index 0000000000..c6e4acb960 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json @@ -0,0 +1,16 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "*RespTsavoriteStress*", + "expectedGETMeanValue_linux": 21, + "expectedSETMeanValue_linux": 19, + "expectedIncrementMeanValue_linux": 27, + "expectedGETMeanValue_win": 20, + "expectedSETMeanValue_win": 17, + "expectedIncrementMeanValue_win": 26, + + "acceptableMeanRange": 15, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} + diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json deleted file mode 100644 index 0995827e35..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespParseStress.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespParseStress*", - "expectedInLinePingMeanValue_linux": 2.4, - "expectedSETMeanValue_linux": 19, - "expectedSETEXMeanValue_linux": 26, - "expectedGETMeanValue_linux": 14, - "expectedZAddRemMeanValue_linux": 186, - "expectedLPushPopMeanValue_linux": 200, - "expectedSAddRemMeanValue_linux": 172, - "expectedHSetDelMeanValue_linux": 200, - "expectedMyDictSetGetMeanValue_linux": 235, - "expectedInLinePingMeanValue_win": 2.4, - "expectedSETMeanValue_win": 19, - "expectedSETEXMeanValue_win": 26, - "expectedGETMeanValue_win": 14, - "expectedZAddRemMeanValue_win": 141, - "expectedLPushPopMeanValue_win": 145, - "expectedSAddRemMeanValue_win": 130, - "expectedHSetDelMeanValue_win": 160, - "expectedMyDictSetGetMeanValue_win": 242, - "acceptableMeanRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json deleted file mode 100644 index ff0c0333c1..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_RespTsavoriteStress.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "_comment": "To get proper expected results, the ExpectedCoresToTestOn needs to match the number of cores on test machine. NOT SURE IF CHECKING CORES", - "ExpectedCoresToTestOn_win": 2, - "ExpectedCoresToTestOn_linux": 2, - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespTsavoriteStress*", - "expectedGETMeanValue_linux": 21, - "expectedSETMeanValue_linux": 19, - "expectedIncrMeanValue_linux": 29, - "expectedGETMeanValue_win": 20, - "expectedSETMeanValue_win": 17, - "expectedIncrMeanValue_win": 26, - "acceptableMeanRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} - diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index b4ca30f493..2c8a95f45a 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -6,20 +6,20 @@ Script to test for performance regressions using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. - CI_BDN_Config_RespParseStress.json + CI_BDN_Config_Resp.RespParseStress.json NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% .EXAMPLE ./run_bdnperftest.ps1 - ./run_bdnperftest.ps1 CI_BDN_Config_RespParseStress.json + ./run_bdnperftest.ps1 CI_BDN_Config_Resp.RespParseStress.json #> # Send the config file for the benchmark. Defaults to a simple one param ( - [string]$configFile = "CI_BDN_Config_RespParseStress.json" + [string]$configFile = "CI_BDN_Config_Resp.RespParseStress.json" ) $OFS = "`r`n" @@ -39,20 +39,20 @@ function AnalyzeResult { [double] $LowerBound = $expectedResultValue * (1 - $Tolerance) [double] $UpperBound = $expectedResultValue * (1 + $Tolerance) [double] $dblfoundResultValue = $foundResultValue - + # Check if the actual value is within the bounds if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Mean Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Mean Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Mean Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } return $false # the values are too different @@ -74,15 +74,17 @@ param ($ResultsLine, $columnNum) # Remove the leading and trailing pipes and split the string by '|' $columns = $ResultsLine.Trim('|').Split('|') $column = $columns | ForEach-Object { $_.Trim() } - $foundValue = $column[$columnNum].Trim(' us') + $foundValue = $column[$columnNum] + + $foundValue = $foundValue.Trim(' us') $foundValue = $foundValue.Trim(' ns') + $foundValue = $foundValue.Trim(' B') + $foundValue = $foundValue.Trim(' m') return $foundValue } - - # ******** BEGIN MAIN ********* # Get base path since paths can differ from machine to machine $pathstring = $pwd.Path @@ -99,6 +101,11 @@ if ($configFile -notlike "*.json") { $configFile += ".json" } +# This is special case that allows passing test without specifying CI_BDN_Confi_ at the beginning - need for perf test +if ($configFile -notlike "CI_BDN_Config_*") { + $configFile = "CI_BDN_Config_" + $configFile +} + # Read the test config file and convert the JSON to a PowerShell object $fullConfiFileAndPath = "ConfigFiles/$configFile" if (-not (Test-Path -Path $fullConfiFileAndPath)) { @@ -114,27 +121,6 @@ if ($IsLinux) { $CurrentOS = "Linux" } - # Calculate number of cores on the test machine - used to verify the config file settings as the config settings will vary based on machine config -if ($IsLinux) { - $sockets = [int]$(lscpu | grep -E '^Socket' | awk '{print $2}') - $coresPerSocket = [int]$(lscpu | grep -E '^Core' | awk '{print $4}') - $NumberOfCores = ($sockets * $coresPerSocket) - - $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_linux -} -else { - $NumberOfCores = (Get-CimInstance -ClassName Win32_Processor).NumberOfCores | Measure-Object -Sum | Select-Object -ExpandProperty Sum - - $ExpectedCoresToTestOn = $object.ExpectedCoresToTestOn_win -} - -# To get accurate comparison of found vs expected values, double check to make sure config settings for Number of Cores of the test machine are what is specified in the test config file -if ($ExpectedCoresToTestOn -ne $NumberOfCores) { - Write-Output -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." - ##&#&# DEBUG #*#*#* Write-Error -Message "The Number of Cores on this machine ($NumberOfCores) are not the same as the Expected Cores ($ExpectedCoresToTestOn) found in the test config file: $fullConfiFileAndPath." -# exit -} - Write-Host "************** Start BDN.benchmark ********************" Write-Host " " @@ -143,10 +129,7 @@ $configuration = $object.configuration $framework = $object.framework $filter = $object.filter $meanColumn = "1" - -#*#*#* DEBUG -# TO DO: Add "Allocated" to values that are checked -#*#*#* DEBUG +$allocatedColumn = "-1" # Number of columns can differ a bit, but know the allocated one is always the last one # Set the expected values based on the OS if ($IsLinux) { @@ -155,22 +138,39 @@ if ($IsLinux) { $expectedSetMeanValue = $object.expectedSETMeanValue_linux $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_linux $expectedGetMeanValue = $object.expectedGETMeanValue_linux + $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_linux $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_linux $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_linux $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_linux $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_linux + $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_linux + $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_linux + $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_linux + $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_linux + $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_linux + $expectedMGetMeanValue = $object.expectedMGETMeanValue_linux $expectedMSetMeanValue = $object.expectedMSETMeanValue_linux - $expectedIncrMeanValue = $object.expectedIncrMeanValue_linux - $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_linux - $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_linux - $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_linux - $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_linux + $expectedIncrMeanValue = $object.expectedIncrementMeanValue_linux + + $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_linux + $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_linux + $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_linux + $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_linux + $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_linux + $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_linux + $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_linux + $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_linux + $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_linux $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_linux $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_linux + $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_linux + $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_linux + $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_linux + $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_linux } else { # Windows expected values @@ -178,26 +178,44 @@ else { $expectedSetMeanValue = $object.expectedSETMeanValue_win $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_win $expectedGetMeanValue = $object.expectedGETMeanValue_win + $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_win $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_win $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_win $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_win $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_win + $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_win + $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_win + $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_win + $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_win + $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_win + $expectedMGetMeanValue = $object.expectedMGETMeanValue_win $expectedMSetMeanValue = $object.expectedMSETMeanValue_win - $expectedIncrMeanValue = $object.expectedIncrMeanValue_win - $expectedBasicLua1MeanValue = $object.expectedBasicLua1MeanValue_win - $expectedBasicLua2MeanValue = $object.expectedBasicLua2MeanValue_win - $expectedBasicLua3MeanValue = $object.expectedBasicLua3MeanValue_win - $expectedBasicLua4MeanValue = $object.expectedBasicLua3MeanValue_win + $expectedIncrMeanValue = $object.expectedIncrementMeanValue_win + + $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_win + $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_win + $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_win + $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_win + $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_win + $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_win + $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_win + $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_win + $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_win $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_win $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_win - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner3MeanValue_win + $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_win + $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_win + $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_win + $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_win + $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_win } # percent allowed variance when comparing expected vs actual found value - same for linux and windows. $acceptableMeanRange = $object.acceptableMeanRange +$acceptableAllocatedRange = $object.acceptableAllocatedRange # Set up the results dir and errorlog dir $resultsDir = "$basePath/test/BDNPerfTests/results" @@ -325,6 +343,46 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } + "*| ZAddRem*" { + Write-Host "** ZAddRem Allocated Value test" + $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRemAllocatedValue $expectedZAddRemAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| LPushPop*" { + Write-Host "** LPushPop Allocated Value test" + $foundLPushPopAllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPopAllocatedValue $expectedLPushPopAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| SAddRem*" { + Write-Host "** SAddRem Allocated Value test" + $foundSAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRemAllocatedValue $expectedSAddRemAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| HSetDel*" { + Write-Host "** HSetDel Allocated Value test" + $foundHSetDelAllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDelAllocatedValue $expectedHSetDelAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MyDictSetGet*" { + Write-Host "** MyDictSetGet Allocated Value test" + $foundMyDictSetGetAllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGetAllocatedValue $expectedMyDictSetGetAllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } "*| Incr*" { Write-Host "** Incr Mean Value test" $foundIncrMeanValue = ParseValueFromResults $line $meanColumn @@ -349,41 +407,42 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } - "*| BasicLua1*" { - Write-Host "** BasicLua1 Mean Value test" - $foundBasicLua1MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua1MeanValue $expectedBasicLua1MeanValue $acceptableMeanRange $true + "*| BasicLuaStress1*" { + Write-Host "** BasicLuaStress1 Mean Value test" + $foundBasicLuaStress1MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress1MeanValue $expectedBasicLuaStress1MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLua2*" { - Write-Host "** BasicLua2 Mean Value test" - $foundBasicLua2MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua2MeanValue $expectedBasicLua2MeanValue $acceptableMeanRange $true + "*| BasicLuaStress2*" { + Write-Host "** BasicLuaStress2 Mean Value test" + $foundBasicLuaStress2MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress2MeanValue $expectedBasicLuaStress2MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } <# # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. - "*| BasicLua3*" { - Write-Host "** BasicLua3 Mean Value test" - $foundBasicLua3MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua3MeanValue $expectedBasicLua3MeanValue $acceptableMeanRange $true + "*| BasicLuaStress3*" { + Write-Host "** BasicLuaStress3 Mean Value test" + $foundBasicLuaStress3MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress3MeanValue $expectedBasicLuaStress3MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLua4*" { - Write-Host "** BasicLua4 Mean Value test" - $foundBasicLua4MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLua4MeanValue $expectedBasicLua4MeanValue $acceptableMeanRange $true + "*| BasicLuaStress4*" { + Write-Host "** BasicLuaStress4 Mean Value test" + $foundBasicLuaStress4MeanValue = ParseValueFromResults $line $meanColumn + $currentResults = AnalyzeResult $foundBasicLuaStress4MeanValue $expectedBasicLuaStress4MeanValue $acceptableMeanRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } #> + "*| BasicLuaRunner1*" { Write-Host "** BasicLuaRunner1 Mean Value test" $foundBasicLuaRunner1MeanValue = ParseValueFromResults $line $meanColumn @@ -420,6 +479,72 @@ Get-Content $resultsFile | ForEach-Object { } } #> + + "*| BasicLuaStress1*" { + Write-Host "** BasicLuaStress1 Allocated Value test" + $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaStress1AllocatedValue $expectedBasicLuaStress1AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaStress2*" { + Write-Host "** BasicLuaStress2 Allocated Value test" + $foundBasicLuaStress2AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaStress2AllocatedValue $expectedBasicLuaStress2AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaStress3*" { + Write-Host "** BasicLuaStress3 Allocated Value test" + $foundBasicLuaStress3AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaStress3AllocatedValue $expectedBasicLuaStress3AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaStress4*" { + Write-Host "** BasicLuaStress4 Allocated Value test" + $foundBasicLuaStress4AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaStress4AllocatedValue $expectedBasicLuaStress4AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner1*" { + Write-Host "** BasicLuaRunner1 Allocated Value test" + $foundBasicLuaRunner1AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner1AllocatedValue $expectedBasicLuaRunner1AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner2*" { + Write-Host "** BasicLuaRunner2 Allocated Value test" + $foundBasicLuaRunner2AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner2AllocatedValue $expectedBasicLuaRunner2AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + "*| BasicLuaRunner3*" { + Write-Host "** BasicLuaRunner3 Allocated Value test" + $foundBasicLuaRunner3AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner3AllocatedValue $expectedBasicLuaRunner3AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| BasicLuaRunner4*" { + Write-Host "** BasicLuaRunner4 Allocated Value test" + $foundBasicLuaRunner4AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundBasicLuaRunner4AllocatedValue $expectedBasicLuaRunner4AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } } } @@ -434,6 +559,3 @@ if ($testSuiteResult) { } Write-Output "** " Write-Output "************************" - - -#./test/BDNPerfTests/results/${{ matrix.test }}_${{ matrix.os }}.json \ No newline at end of file From cdf2d25356fad2ca234a01969809b3e4d3aeb5fb Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 13:52:20 -0700 Subject: [PATCH 22/44] Bump threshold to 35% (135%) to trigger perf failure --- .github/workflows/ci-bdnbenchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index cbee4641d4..6019b83c27 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -78,7 +78,7 @@ jobs: tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '130%' + alert-threshold: '135%' comment-on-alert: true fail-on-alert: true alert-comment-cc-users: '@darrenge' From 00b36746b37dcf422b76be0eb28d509bcbf267f7 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 14:06:52 -0700 Subject: [PATCH 23/44] Set to Linux only to reduce number of consectutive tests which caused push collisions with GitHub --- .github/workflows/ci-bdnbenchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 6019b83c27..acd0345f4c 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -36,7 +36,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-latest, windows-latest ] + #os: [ ubuntu-latest, windows-latest ] + os: [ ubuntu-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] From 414168e7f85e902b352690ef66cc10091c0d3d92 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 15:03:53 -0700 Subject: [PATCH 24/44] Just Windows only test run --- .github/workflows/ci-bdnbenchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index acd0345f4c..d5accdc94b 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -37,7 +37,7 @@ jobs: fail-fast: false matrix: #os: [ ubuntu-latest, windows-latest ] - os: [ ubuntu-latest ] + os: [ windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] From 28ccbb4548fe59778ece8ff7c6b834b992cd5ddd Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 15:15:26 -0700 Subject: [PATCH 25/44] Putting it back to both OS for the tests --- .github/workflows/ci-bdnbenchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index d5accdc94b..6019b83c27 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -36,8 +36,7 @@ jobs: strategy: fail-fast: false matrix: - #os: [ ubuntu-latest, windows-latest ] - os: [ windows-latest ] + os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] From 23296d620d07bd76f2649bf1b9fc10ebaac777a1 Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 16 Oct 2024 15:32:23 -0700 Subject: [PATCH 26/44] Set threshhold to 50% (150%) as seeing BDN ran twice on same code sometimes is in 40%+ range of difference. --- .github/workflows/ci-bdnbenchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 6019b83c27..a80cd3da1e 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -78,7 +78,7 @@ jobs: tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '135%' + alert-threshold: '150%' comment-on-alert: true fail-on-alert: true alert-comment-cc-users: '@darrenge' From 1f367684e38a298d25c8462082d303fcb4bbfc56 Mon Sep 17 00:00:00 2001 From: darrenge Date: Fri, 18 Oct 2024 09:40:20 -0700 Subject: [PATCH 27/44] Big change where using continuousbenchmark branch for all data for continuous benchmarking and also changed to where script is only checking allocated values and let github-action-benchmark compare mean values --- .github/workflows/ci-bdnbenchmark.yml | 10 +- ...I_BDN_Config_Cluster.RespClusterBench.json | 10 +- ...onfig_Cluster.RespClusterMigrateBench.json | 10 +- ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 13 +- .../CI_BDN_Config_Resp.RespLuaStress.json | 11 - .../CI_BDN_Config_Resp.RespParseStress.json | 22 -- ...I_BDN_Config_Resp.RespTsavoriteStress.json | 8 +- test/BDNPerfTests/run_bdnperftest.ps1 | 228 +----------------- 8 files changed, 16 insertions(+), 296 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index a80cd3da1e..5c0203edb2 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -51,9 +51,9 @@ jobs: run: dotnet restore - name: Run BDN.benchmark Perf Test - run: .\test\BDNPerfTests\run_bdnperftest.ps1 ${{ matrix.test }} + run: ./test/BDNPerfTests/run_bdnperftest.ps1 ${{ matrix.test }} shell: pwsh - continue-on-error: true + continue-on-error: false - name: Upload test results to artifacts uses: actions/upload-artifact@v4 @@ -78,11 +78,13 @@ jobs: tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '150%' + alert-threshold: '140%' comment-on-alert: true fail-on-alert: true alert-comment-cc-users: '@darrenge' summary-always: false + gh-pages-branch: 'continuousbenchmark' + benchmark-data-dir-path: 'website/static/charts' auto-push: true # Run `github-action-benchmark` action for the Continuous Benchmarking Charts @@ -94,6 +96,8 @@ jobs: output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} summary-always: true + gh-pages-branch: 'continuousbenchmark' + benchmark-data-dir-path: 'website/static/charts' auto-push: true diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json index 19985d1bbf..b6c38afb3b 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json @@ -2,16 +2,8 @@ "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterBench*", - "expectedGETMeanValue_linux": 27, - "expectedSETMeanValue_linux": 25, - "expectedMGETMeanValue_linux": 20, - "expectedMSETMeanValue_linux": 20, - "expectedGETMeanValue_win": 24, - "expectedSETMeanValue_win": 24, - "expectedMGETMeanValue_win": 19, - "expectedMSETMeanValue_win": 19, - "acceptableMeanRange": 15, + "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json index 24be386912..90c9408bd1 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json @@ -2,16 +2,8 @@ "configuration": "Release", "framework": "net8.0", "filter": "*RespClusterMigrateBench*", - "expectedGETMeanValue_linux": 46, - "expectedSETMeanValue_linux": 49, - "expectedMGETMeanValue_linux": 40, - "expectedMSETMeanValue_linux": 39, - "expectedGETMeanValue_win": 46, - "expectedSETMeanValue_win": 47, - "expectedMGETMeanValue_win": 40, - "expectedMSETMeanValue_win": 37, - "acceptableMeanRange": 15, + "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json index 75747dc80d..a51538c4c4 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json @@ -2,17 +2,7 @@ "configuration": "Release", "framework": "net8.0", "filter": "*LuaRunner*", - "expectedBasicLuaRunner1MeanValue_linux": 182, - "expectedBasicLuaRunner2MeanValue_linux": 325, - "expectedBasicLuaRunner3MeanValue_linux": 1473, - "expectedBasicLuaRunner4MeanValue_linux": 3754, - - "expectedBasicLuaRunner1MeanValue_win": 115, - "expectedBasicLuaRunner2MeanValue_win": 180, - "expectedBasicLuaRunner3MeanValue_win": 686, - "expectedBasicLuaRunner4MeanValue_win": 2711, - - + "expectedBasicLuaRunner1AllocatedValue_linux": 24, "expectedBasicLuaRunner2AllocatedValue_linux": 144, "expectedBasicLuaRunner3AllocatedValue_linux": 240, @@ -23,7 +13,6 @@ "expectedBasicLuaRunner3AllocatedValue_win": 240, "expectedBasicLuaRunner4AllocatedValue_win": 776, - "acceptableMeanRange": 15, "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json index 92b6024bef..ca67c58cca 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json @@ -3,27 +3,16 @@ "framework": "net8.0", "filter": "*LuaStress*", - "expectedBasicLuaStress1MeanValue_linux": 175, - "expectedBasicLuaStress2MeanValue_linux": 284, - "expectedBasicLuaStress3MeanValue_linux": 1407, - "expectedBasicLuaStress4MeanValue_linux": 2733, - "expectedBasicLuaStress1AllocatedValue_linux": 24, "expectedBasicLuaStress2AllocatedValue_linux": 144, "expectedBasicLuaStress3AllocatedValue_linux": 328, "expectedBasicLuaStress4AllocatedValue_linux": 832, - "expectedBasicLuaStress1MeanValue_win": 107, - "expectedBasicLuaStress2MeanValue_win": 180, - "expectedBasicLuaStress3MeanValue_win": 660, - "expectedBasicLuaStress4MeanValue_win": 1805, - "expectedBasicLuaStress1AllocatedValue_win": 24, "expectedBasicLuaStress2AllocatedValue_win": 144, "expectedBasicLuaStress3AllocatedValue_win": 328, "expectedBasicLuaStress4AllocatedValue_win": 832, - "acceptableMeanRange": 15, "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json index 636e9a667c..c32c8979c0 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json @@ -2,16 +2,6 @@ "configuration": "Release", "framework": "net8.0", "filter": "*RespParseStress*", - "expectedInLinePingMeanValue_linux": 2.2, - "expectedSETMeanValue_linux": 19, - "expectedSETEXMeanValue_linux": 26, - "expectedGETMeanValue_linux": 13, - "expectedIncrementMeanValue_linux": 27, - "expectedZAddRemMeanValue_linux": 186, - "expectedLPushPopMeanValue_linux": 166, - "expectedSAddRemMeanValue_linux": 164, - "expectedHSetDelMeanValue_linux": 196, - "expectedMyDictSetGetMeanValue_linux": 248, "expectedZAddRemAllocatedValue_linux": 23552, "expectedLPushPopAllocatedValue_linux": 18432, @@ -19,24 +9,12 @@ "expectedHSetDelAllocatedValue_linux": 55297, "expectedMyDictSetGetAllocatedValue_linux": 30720, - "expectedInLinePingMeanValue_win": 2.2, - "expectedSETMeanValue_win": 18, - "expectedSETEXMeanValue_win": 25, - "expectedGETMeanValue_win": 14, - "expectedIncrementMeanValue_win": 24, - "expectedZAddRemMeanValue_win": 144, - "expectedLPushPopMeanValue_win": 128, - "expectedSAddRemMeanValue_win": 124, - "expectedHSetDelMeanValue_win": 169, - "expectedMyDictSetGetMeanValue_win": 250, - "expectedZAddRemAllocatedValue_win": 23552, "expectedLPushPopAllocatedValue_win": 18432, "expectedSAddRemAllocatedValue_win": 16384, "expectedHSetDelAllocatedValue_win": 55297, "expectedMyDictSetGetAllocatedValue_win": 30720, - "acceptableMeanRange": 15, "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json index c6e4acb960..954ecd6e61 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json +++ b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json @@ -2,14 +2,8 @@ "configuration": "Release", "framework": "net8.0", "filter": "*RespTsavoriteStress*", - "expectedGETMeanValue_linux": 21, - "expectedSETMeanValue_linux": 19, - "expectedIncrementMeanValue_linux": 27, - "expectedGETMeanValue_win": 20, - "expectedSETMeanValue_win": 17, - "expectedIncrementMeanValue_win": 26, - "acceptableMeanRange": 15, + "acceptableAllocatedRange": 10, "acceptableError": ".1", "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 2c8a95f45a..875fbd1cd8 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -4,7 +4,7 @@ .DESCRIPTION - Script to test for performance regressions using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. + Script to test for performance regressions in Allocated Memory using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. CI_BDN_Config_Resp.RespParseStress.json @@ -42,17 +42,17 @@ function AnalyzeResult { # Check if the actual value is within the bounds if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Test Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + Write-Host "** ** PASS! ** Acceptable Allocated Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Allocated Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" Write-Host "** " } return $false # the values are too different @@ -76,8 +76,6 @@ param ($ResultsLine, $columnNum) $column = $columns | ForEach-Object { $_.Trim() } $foundValue = $column[$columnNum] - $foundValue = $foundValue.Trim(' us') - $foundValue = $foundValue.Trim(' ns') $foundValue = $foundValue.Trim(' B') $foundValue = $foundValue.Trim(' m') @@ -121,52 +119,29 @@ if ($IsLinux) { $CurrentOS = "Linux" } -Write-Host "************** Start BDN.benchmark ********************" +Write-Host "************** Start BDN.benchmark Test Run ********************" Write-Host " " # Set all the config options (args to benchmark app) based on values from config json file $configuration = $object.configuration $framework = $object.framework $filter = $object.filter -$meanColumn = "1" $allocatedColumn = "-1" # Number of columns can differ a bit, but know the allocated one is always the last one # Set the expected values based on the OS if ($IsLinux) { # Linux expected values - $expectedInLinePingMeanValue = $object.expectedInLinePingMeanValue_linux - $expectedSetMeanValue = $object.expectedSETMeanValue_linux - $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_linux - $expectedGetMeanValue = $object.expectedGETMeanValue_linux - - $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_linux - $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_linux - $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_linux - $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_linux - $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_linux $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_linux $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_linux $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_linux $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_linux $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_linux - $expectedMGetMeanValue = $object.expectedMGETMeanValue_linux - $expectedMSetMeanValue = $object.expectedMSETMeanValue_linux - $expectedIncrMeanValue = $object.expectedIncrementMeanValue_linux - - $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_linux - $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_linux - $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_linux - $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_linux $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_linux $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_linux $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_linux $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_linux - $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_linux - $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_linux - $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_linux - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_linux $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_linux $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_linux $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_linux @@ -174,39 +149,17 @@ if ($IsLinux) { } else { # Windows expected values - $expectedInLinePingMeanValue = $object.expectedInLinePingMeanValue_win - $expectedSetMeanValue = $object.expectedSETMeanValue_win - $expectedSetEXMeanValue = $object.expectedSETEXMeanValue_win - $expectedGetMeanValue = $object.expectedGETMeanValue_win - - $expectedZAddRemMeanValue = $object.expectedZAddRemMeanValue_win - $expectedLPushPopMeanValue = $object.expectedLPushPopMeanValue_win - $expectedSAddRemMeanValue = $object.expectedSAddRemMeanValue_win - $expectedHSetDelMeanValue = $object.expectedHSetDelMeanValue_win - $expectedMyDictSetGetMeanValue = $object.expectedMyDictSetGetMeanValue_win $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_win $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_win $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_win $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_win $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_win - $expectedMGetMeanValue = $object.expectedMGETMeanValue_win - $expectedMSetMeanValue = $object.expectedMSETMeanValue_win - $expectedIncrMeanValue = $object.expectedIncrementMeanValue_win - - $expectedBasicLuaStress1MeanValue = $object.expectedBasicLuaStress1MeanValue_win - $expectedBasicLuaStress2MeanValue = $object.expectedBasicLuaStress2MeanValue_win - $expectedBasicLuaStress3MeanValue = $object.expectedBasicLuaStress3MeanValue_win - $expectedBasicLuaStress4MeanValue = $object.expectedBasicLuaStress4MeanValue_win $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_win $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_win $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_win $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_win - $expectedBasicLuaRunner1MeanValue = $object.expectedBasicLuaRunner1MeanValue_win - $expectedBasicLuaRunner2MeanValue = $object.expectedBasicLuaRunner2MeanValue_win - $expectedBasicLuaRunner3MeanValue = $object.expectedBasicLuaRunner3MeanValue_win - $expectedBasicLuaRunner4MeanValue = $object.expectedBasicLuaRunner4MeanValue_win $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_win $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_win $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_win @@ -214,7 +167,6 @@ else { } # percent allowed variance when comparing expected vs actual found value - same for linux and windows. -$acceptableMeanRange = $object.acceptableMeanRange $acceptableAllocatedRange = $object.acceptableAllocatedRange # Set up the results dir and errorlog dir @@ -270,79 +222,6 @@ $testSuiteResult = $true Get-Content $resultsFile | ForEach-Object { $line = $_ switch -Wildcard ($line) { - "*| InlinePing*" { - Write-Output "** InlinePing Mean Value test" - $foundInLinePingMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundInLinePingMeanValue $expectedInLinePingMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - # This one a bit different as need extra space in the check so doesn't pick up other Set* calls - "*| Set *" { - Write-Host "** Set Mean Value test" - $foundSetMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundSetMeanValue $expectedSetMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| SetEx*" { - Write-Host "** SetEx Mean Value test" - $foundSetExMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundSetExMeanValue $expectedSETEXMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| Get*" { - Write-Host "** Get Mean Value test" - $foundGetMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundGetMeanValue $expectedGetMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| ZAddRem*" { - Write-Host "** ZAddRem Mean Value test" - $foundZAddRemMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundZAddRemMeanValue $expectedZAddRemMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| LPushPop*" { - Write-Host "** LPushPop Mean Value test" - $foundLPushPopMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundLPushPopMeanValue $expectedLPushPopMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| SAddRem*" { - Write-Host "** SAddRem Mean Value test" - $foundSAddRemMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundSAddRemMeanValue $expectedSAddRemMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| HSetDel*" { - Write-Host "** HSetDel Mean Value test" - $foundHSetDelMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundHSetDelMeanValue $expectedHSetDelMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MyDictSetGet*" { - Write-Host "** MyDictSetGet Mean Value test" - $foundMyDictSetGetMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundMyDictSetGetMeanValue $expectedMyDictSetGetMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } "*| ZAddRem*" { Write-Host "** ZAddRem Allocated Value test" $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn @@ -383,103 +262,6 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } - "*| Incr*" { - Write-Host "** Incr Mean Value test" - $foundIncrMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundIncrMeanValue $expectedIncrMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MGet*" { - Write-Host "** MGet Mean Value test" - $foundMGetMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundMGetMeanValue $expectedMGetMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MSet*" { - Write-Host "** MSet Mean Value test" - $foundMSetMeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundMSetMeanValue $expectedMSetMeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress1*" { - Write-Host "** BasicLuaStress1 Mean Value test" - $foundBasicLuaStress1MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress1MeanValue $expectedBasicLuaStress1MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress2*" { - Write-Host "** BasicLuaStress2 Mean Value test" - $foundBasicLuaStress2MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress2MeanValue $expectedBasicLuaStress2MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - <# - # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. - "*| BasicLuaStress3*" { - Write-Host "** BasicLuaStress3 Mean Value test" - $foundBasicLuaStress3MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress3MeanValue $expectedBasicLuaStress3MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress4*" { - Write-Host "** BasicLuaStress4 Mean Value test" - $foundBasicLuaStress4MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaStress4MeanValue $expectedBasicLuaStress4MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - #> - - "*| BasicLuaRunner1*" { - Write-Host "** BasicLuaRunner1 Mean Value test" - $foundBasicLuaRunner1MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner1MeanValue $expectedBasicLuaRunner1MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaRunner2*" { - Write-Host "** BasicLuaRunner2 Mean Value test" - $foundBasicLuaRunner2MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner2MeanValue $expectedBasicLuaRunner2MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - - <# - # Have this disabled for now for the CI runs. These are too volatile to have a CI gated on them. - "*| BasicLuaRunner3*" { - Write-Host "** BasicLuaRunner3 Mean Value test" - $foundBasicLuaRunner3MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner3MeanValue $expectedBasicLuaRunner3MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaRunner4*" { - Write-Host "** BasicLuaRunner4 Mean Value test" - $foundBasicLuaRunner4MeanValue = ParseValueFromResults $line $meanColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner4MeanValue $expectedBasicLuaRunner4MeanValue $acceptableMeanRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - #> - "*| BasicLuaStress1*" { Write-Host "** BasicLuaStress1 Allocated Value test" $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedColumn From ef0cf7bada23f9a0ce33e6af1e1fea3592e0a382 Mon Sep 17 00:00:00 2001 From: darrenge Date: Fri, 18 Oct 2024 09:45:35 -0700 Subject: [PATCH 28/44] Updated ClusterBench to get main changes --- .../BDN.benchmark/Cluster/RespClusterBench.cs | 160 ++---------------- 1 file changed, 14 insertions(+), 146 deletions(-) diff --git a/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs b/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs index e9af4ba3f2..f13a6f0df9 100644 --- a/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs +++ b/benchmark/BDN.benchmark/Cluster/RespClusterBench.cs @@ -1,170 +1,32 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Text; using BenchmarkDotNet.Attributes; -using Embedded.perftest; -using Garnet.common; -using Garnet.server; namespace BDN.benchmark.Cluster { - public unsafe struct Request - { - public byte[] buffer; - public byte* ptr; - - public Request(int size) - { - buffer = GC.AllocateArray(size, pinned: true); - ptr = (byte*)Unsafe.AsPointer(ref buffer[0]); - } - } - - public unsafe class ClusterContext - { - EmbeddedRespServer server; - RespServerSession session; - BenchUtils benchUtils = new(); - - private readonly int Port = 7000; - - public ReadOnlySpan keyTag => "{0}"u8; - public Request[] singleGetSet; - public Request[] singleMGetMSet; - - public void Dispose() - { - session.Dispose(); - server.Dispose(); - } - - public void SetupSingleInstance() - { - var opt = new GarnetServerOptions - { - QuietMode = true, - EnableCluster = true, - Port = Port, - CleanClusterConfig = true, - CheckpointDir = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "/tmp" : null - }; - server = new EmbeddedRespServer(opt); - session = server.GetRespSession(); - } - - public void AddSlotRange(List<(int, int)> slotRanges) - { - foreach (var slotRange in slotRanges) - { - var clusterAddSlotsRange = Encoding.ASCII.GetBytes($"*4\r\n$7\r\nCLUSTER\r\n$13\r\nADDSLOTSRANGE\r\n" + - $"${NumUtils.NumDigits(slotRange.Item1)}\r\n{slotRange.Item1}\r\n" + - $"${NumUtils.NumDigits(slotRange.Item2)}\r\n{slotRange.Item2}\r\n"); - fixed (byte* req = clusterAddSlotsRange) - _ = session.TryConsumeMessages(req, clusterAddSlotsRange.Length); - } - } - - public void CreateGetSet(int keySize = 8, int valueSize = 32, int batchSize = 128) - { - var pairs = new (byte[], byte[])[batchSize]; - for (var i = 0; i < batchSize; i++) - { - pairs[i] = (new byte[keySize], new byte[valueSize]); - - keyTag.CopyTo(pairs[i].Item1.AsSpan()); - benchUtils.RandomBytes(ref pairs[i].Item1, startOffset: keyTag.Length); - benchUtils.RandomBytes(ref pairs[i].Item2); - } - - var setByteCount = batchSize * ("*2\r\n$3\r\nSET\r\n"u8.Length + (1 + NumUtils.NumDigits(keySize) + 2 + keySize + 2) + (1 + NumUtils.NumDigits(valueSize) + 2 + valueSize + 2)); - var setReq = new Request(setByteCount); - var curr = setReq.ptr; - var end = curr + setReq.buffer.Length; - for (var i = 0; i < batchSize; i++) - { - _ = RespWriteUtils.WriteArrayLength(3, ref curr, end); - _ = RespWriteUtils.WriteBulkString("SET"u8, ref curr, end); - _ = RespWriteUtils.WriteBulkString(pairs[i].Item1, ref curr, end); - _ = RespWriteUtils.WriteBulkString(pairs[i].Item2, ref curr, end); - } - - var getByteCount = batchSize * ("*2\r\n$3\r\nGET\r\n"u8.Length + 1 + NumUtils.NumDigits(keySize) + 2 + keySize + 2); - var getReq = new Request(getByteCount); - curr = getReq.ptr; - end = curr + getReq.buffer.Length; - for (var i = 0; i < batchSize; i++) - { - _ = RespWriteUtils.WriteArrayLength(2, ref curr, end); - _ = RespWriteUtils.WriteBulkString("GET"u8, ref curr, end); - _ = RespWriteUtils.WriteBulkString(pairs[i].Item1, ref curr, end); - } - singleGetSet = [getReq, setReq]; - } - - public void CreateMGetMSet(int keySize = 8, int valueSize = 32, int batchSize = 128) - { - var pairs = new (byte[], byte[])[batchSize]; - for (var i = 0; i < batchSize; i++) - { - pairs[i] = (new byte[keySize], new byte[valueSize]); - - keyTag.CopyTo(pairs[i].Item1.AsSpan()); - benchUtils.RandomBytes(ref pairs[i].Item1, startOffset: keyTag.Length); - benchUtils.RandomBytes(ref pairs[i].Item2); - } - - var mGetHeaderSize = 1 + NumUtils.NumDigits(1 + batchSize) + 2 + "$4\r\nMGET\r\n"u8.Length; - var getRespSize = 1 + NumUtils.NumDigits(keySize) + 2 + keySize + 2; - var mGetByteCount = mGetHeaderSize + (batchSize * getRespSize); - var mGetReq = new Request(mGetByteCount); - - var curr = mGetReq.ptr; - var end = curr + mGetReq.buffer.Length; - _ = RespWriteUtils.WriteArrayLength(1 + batchSize, ref curr, end); - _ = RespWriteUtils.WriteBulkString("MGET"u8, ref curr, end); - for (var i = 0; i < batchSize; i++) - _ = RespWriteUtils.WriteBulkString(pairs[i].Item1, ref curr, end); - - var mSetHeaderSize = 1 + NumUtils.NumDigits(1 + batchSize * 2) + 2 + "$4\r\nMSET\r\n"u8.Length; - var setRespSize = 1 + NumUtils.NumDigits(keySize) + 2 + keySize + 2 + 1 + NumUtils.NumDigits(valueSize) + 2 + valueSize + 2; - var mSetByteCount = mSetHeaderSize + (batchSize * setRespSize); - var mSetReq = new Request(mSetByteCount); - - curr = mSetReq.ptr; - end = curr + mSetReq.buffer.Length; - _ = RespWriteUtils.WriteArrayLength(1 + batchSize * 2, ref curr, end); - _ = RespWriteUtils.WriteBulkString("MSET"u8, ref curr, end); - for (var i = 0; i < batchSize; i++) - { - _ = RespWriteUtils.WriteBulkString(pairs[i].Item1, ref curr, end); - _ = RespWriteUtils.WriteBulkString(pairs[i].Item2, ref curr, end); - } - singleMGetMSet = [mGetReq, mSetReq]; - } - - public void Consume(byte* ptr, int length) - => session.TryConsumeMessages(ptr, length); - } - [MemoryDiagnoser] public unsafe class RespClusterBench { + protected bool enableCluster = true; ClusterContext cc; [GlobalSetup] - public void GlobalSetup() + public virtual void GlobalSetup() { cc = new ClusterContext(); - cc.SetupSingleInstance(); + cc.SetupSingleInstance(enableCluster); cc.AddSlotRange([(0, 16383)]); cc.CreateGetSet(); cc.CreateMGetMSet(); + cc.CreateCPBSET(); + // Warmup/Prepopulate stage cc.Consume(cc.singleGetSet[1].ptr, cc.singleGetSet[1].buffer.Length); + // Warmup/Prepopulate stage cc.Consume(cc.singleMGetMSet[1].ptr, cc.singleMGetMSet[1].buffer.Length); + // Warmup/Prepopulate stage + cc.Consume(cc.singleCPBSET.ptr, cc.singleCPBSET.buffer.Length); } [GlobalCleanup] @@ -196,5 +58,11 @@ public void MSet() { cc.Consume(cc.singleMGetMSet[1].ptr, cc.singleMGetMSet[1].buffer.Length); } + + [Benchmark] + public void CPBSET() + { + cc.Consume(cc.singleCPBSET.ptr, cc.singleCPBSET.buffer.Length); + } } } \ No newline at end of file From d5953a26de9b92b7137652625b32fc245e3ed9ce Mon Sep 17 00:00:00 2001 From: darrenge Date: Fri, 18 Oct 2024 10:53:51 -0700 Subject: [PATCH 29/44] Fix issue with Cluster BDN is not able to access folder so setting to tmp which does have access --- benchmark/BDN.benchmark/Cluster/ClusterContext.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs index 243264acfe..4f56c284f1 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; using Embedded.perftest; using Garnet.common; @@ -49,6 +50,8 @@ public void SetupSingleInstance(bool enableCluster = true) Port = Port, CleanClusterConfig = true, }; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + opt.CheckpointDir = "/tmp"; server = new EmbeddedRespServer(opt); session = server.GetRespSession(); server.Register.NewTransactionProc(CustomProcSetBench.CommandName, () => new CustomProcSetBench(), new RespCommandsInfo { Arity = CustomProcSetBench.Arity }); From d58bf0d17e760cd279f1e5415d24d0aabb830aa9 Mon Sep 17 00:00:00 2001 From: darrenge Date: Fri, 18 Oct 2024 11:03:04 -0700 Subject: [PATCH 30/44] Bumping threshold to 50% (150%) as mean value in RespParseStress can vary into the 40% from run to run. --- .github/workflows/ci-bdnbenchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 5c0203edb2..51dcbb0d2e 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -78,7 +78,7 @@ jobs: tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '140%' + alert-threshold: '150%' comment-on-alert: true fail-on-alert: true alert-comment-cc-users: '@darrenge' From 75e78c3e005201592e7d417ebf0d65db3c1e018f Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 22 Oct 2024 15:12:17 -0700 Subject: [PATCH 31/44] First update --- ...I_BDN_Config_Cluster.RespClusterBench.json | 9 -- ...onfig_Cluster.RespClusterMigrateBench.json | 9 -- ...I_BDN_Config_Resp.RespLuaRunnerStress.json | 19 --- .../CI_BDN_Config_Resp.RespLuaStress.json | 19 --- .../CI_BDN_Config_Resp.RespParseStress.json | 21 ---- ...I_BDN_Config_Resp.RespTsavoriteStress.json | 10 -- ..._CONFIG_BDN_Benchmark_BasicOperations.json | 17 +++ ...I_CONFIG_BDN_Benchmark_ClusterMigrate.json | 19 +++ ...ONFIG_BDN_Benchmark_ClusterOperations.json | 31 +++++ .../CI_CONFIG_BDN_Benchmark_Lua.json | 19 +++ ...CONFIG_BDN_Benchmark_ObjectOperations.json | 53 +++++++++ test/BDNPerfTests/run_bdnperftest.ps1 | 108 ++++++------------ 12 files changed, 171 insertions(+), 163 deletions(-) delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_BasicOperations.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterOperations.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json deleted file mode 100644 index b6c38afb3b..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterBench.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespClusterBench*", - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json deleted file mode 100644 index 90c9408bd1..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Cluster.RespClusterMigrateBench.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespClusterMigrateBench*", - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json deleted file mode 100644 index a51538c4c4..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaRunnerStress.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*LuaRunner*", - - "expectedBasicLuaRunner1AllocatedValue_linux": 24, - "expectedBasicLuaRunner2AllocatedValue_linux": 144, - "expectedBasicLuaRunner3AllocatedValue_linux": 240, - "expectedBasicLuaRunner4AllocatedValue_linux": 776, - - "expectedBasicLuaRunner1AllocatedValue_win": 24, - "expectedBasicLuaRunner2AllocatedValue_win": 144, - "expectedBasicLuaRunner3AllocatedValue_win": 240, - "expectedBasicLuaRunner4AllocatedValue_win": 776, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json deleted file mode 100644 index ca67c58cca..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespLuaStress.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*LuaStress*", - - "expectedBasicLuaStress1AllocatedValue_linux": 24, - "expectedBasicLuaStress2AllocatedValue_linux": 144, - "expectedBasicLuaStress3AllocatedValue_linux": 328, - "expectedBasicLuaStress4AllocatedValue_linux": 832, - - "expectedBasicLuaStress1AllocatedValue_win": 24, - "expectedBasicLuaStress2AllocatedValue_win": 144, - "expectedBasicLuaStress3AllocatedValue_win": 328, - "expectedBasicLuaStress4AllocatedValue_win": 832, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json deleted file mode 100644 index c32c8979c0..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespParseStress.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespParseStress*", - - "expectedZAddRemAllocatedValue_linux": 23552, - "expectedLPushPopAllocatedValue_linux": 18432, - "expectedSAddRemAllocatedValue_linux": 16384, - "expectedHSetDelAllocatedValue_linux": 55297, - "expectedMyDictSetGetAllocatedValue_linux": 30720, - - "expectedZAddRemAllocatedValue_win": 23552, - "expectedLPushPopAllocatedValue_win": 18432, - "expectedSAddRemAllocatedValue_win": 16384, - "expectedHSetDelAllocatedValue_win": 55297, - "expectedMyDictSetGetAllocatedValue_win": 30720, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json b/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json deleted file mode 100644 index 954ecd6e61..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_BDN_Config_Resp.RespTsavoriteStress.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "*RespTsavoriteStress*", - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} - diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_BasicOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_BasicOperations.json new file mode 100644 index 0000000000..9dd402b8d9 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_BasicOperations.json @@ -0,0 +1,17 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Operations.BasicOperations.*", + + "expectedInlinePing_ACL_AllocatedValue_linux": 0, + "expectedInlinePing_AOF_AllocatedValue_linux": 0, + "expectedInlinePing_None_AllocatedValue_linux": 0, + + "expectedInlinePing_ACL_AllocatedValue_win": 0, + "expectedInlinePing_AOF_AllocatedValue_win": 0, + "expectedInlinePing_None_AllocatedValue_win": 0, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json new file mode 100644 index 0000000000..0a45091e1c --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json @@ -0,0 +1,19 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Cluster.ClusterMigrate.*", + + "expectedGet_AllocatedValue_linux": 0, + "expectedSet_AllocatedValue_linux": 0, + "expectedMGet_AllocatedValue_linux": 0, + "expectedMSet_AllocatedValue_linux": 0, + + "expectedGet_AllocatedValue_win": 0, + "expectedSet_AllocatedValue_win": 0, + "expectedMGet_AllocatedValue_win": 0, + "expectedMSet_AllocatedValue_win": 0, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterOperations.json new file mode 100644 index 0000000000..66006975c3 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterOperations.json @@ -0,0 +1,31 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Cluster.ClusterOperations.*", + + "expectedGet_DSV_AllocatedValue_linux": 0, + "expectedSet_DSV_AllocatedValue_linux": 0, + "expectedMGet_DSV_AllocatedValue_linux": 0, + "expectedMSet_DSV_AllocatedValue_linux": 0, + "expectedCPBSet_DSV_AllocatedValue_linux": 0, + "expectedGet_None_AllocatedValue_linux": 0, + "expectedSet_None_AllocatedValue_linux": 0, + "expectedMGet_None_AllocatedValue_linux": 0, + "expectedMSet_None_AllocatedValue_linux": 0, + "expectedCPBSet_None_AllocatedValue_linux": 0, + + "expectedGet_DSV_AllocatedValue_win": 0, + "expectedSet_DSV_AllocatedValue_win": 0, + "expectedMGet_DSV_AllocatedValue_win": 0, + "expectedMSet_DSV_AllocatedValue_win": 0, + "expectedCPBSet_DSV_AllocatedValue_win": 0, + "expectedGet_None_AllocatedValue_win": 0, + "expectedSet_None_AllocatedValue_win": 0, + "expectedMGet_None_AllocatedValue_win": 0, + "expectedMSet_None_AllocatedValue_win": 0, + "expectedCPBSet_None_AllocatedValue_win": 0, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json new file mode 100644 index 0000000000..3b5c94a5f5 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json @@ -0,0 +1,19 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Lua.*", + + "expectedLua1AllocatedValue_linux": 24, + "expectedLua2AllocatedValue_linux": 144, + "expectedLua3AllocatedValue_linux": 240, + "expectedLua4AllocatedValue_linux": 776, + + "expectedLua1AllocatedValue_win": 24, + "expectedLua2AllocatedValue_win": 144, + "expectedLua3AllocatedValue_win": 240, + "expectedLua4AllocatedValue_win": 776, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json new file mode 100644 index 0000000000..5782b0489c --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json @@ -0,0 +1,53 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Operations.ObjectOperations.*", + + "expectedZAddRem_ACL_AllocatedValue_linux": 0, + "expectedLPushPop_ACL_AllocatedValue_linux": 0, + "expectedSAddRem_ACL_AllocatedValue_linux": 0, + "expectedHSetDel_ACL_AllocatedValue_linux": 0, + "expectedMyDictSetGet_ACL_AllocatedValue_linux": 0, + "expectedCustomProcSet_ACL_AllocatedValue_linux": 0, + + "expectedZAddRem_AOF_AllocatedValue_linux": 18400, + "expectedLPushPop_AOF_AllocatedValue_linux": 14400, + "expectedSAddRem_AOF_AllocatedValue_linux": 12800, + "expectedHSetDel_AOF_AllocatedValue_linux": 43201, + "expectedMyDictSetGet_AOF_AllocatedValue_linux": 24000, + "expectedCustomProcSet_AOF_AllocatedValue_linux": 0, + + "expectedZAddRem_None_AllocatedValue_linux": 18400, + "expectedLPushPop_None_AllocatedValue_linux": 14400, + "expectedSAddRem_None_AllocatedValue_linux": 12800, + "expectedHSetDel_None_AllocatedValue_linux": 43201, + "expectedMyDictSetGet_None_AllocatedValue_linux": 24000, + "expectedCustomProcSet_None_AllocatedValue_linux": 0, + + + "expectedZAddRem_ACL_AllocatedValue_win": 0, + "expectedLPushPop_ACL_AllocatedValue_win": 0, + "expectedSAddRem_ACL_AllocatedValue_win": 0, + "expectedHSetDel_ACL_AllocatedValue_win": 0, + "expectedMyDictSetGet_ACL_AllocatedValue_win": 0, + "expectedCustomProcSet_ACL_AllocatedValue_win": 0, + + "expectedZAddRem_AOF_AllocatedValue_win": 18400, + "expectedLPushPop_AOF_AllocatedValue_win": 14400, + "expectedSAddRem_AOF_AllocatedValue_win": 12800, + "expectedHSetDel_AOF_AllocatedValue_win": 43201, + "expectedMyDictSetGet_AOF_AllocatedValue_win": 24000, + "expectedCustomProcSet_AOF_AllocatedValue_win": 0, + + "expectedZAddRem_None_AllocatedValue_win": 18400, + "expectedLPushPop_None_AllocatedValue_win": 14400, + "expectedSAddRem_None_AllocatedValue_win": 12800, + "expectedHSetDel_None_AllocatedValue_win": 43201, + "expectedMyDictSetGet_None_AllocatedValue_win": 24000, + "expectedCustomProcSet_None_AllocatedValue_win": 0, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} + diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 875fbd1cd8..b22dcc08bf 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -34,25 +34,24 @@ $OFS = "`r`n" function AnalyzeResult { param ($foundResultValue, $expectedResultValue, $acceptablePercentRange, $warnonly) - # Calculate the lower and upper bounds of the expected value + # Calculate the upper bounds of the expected value [double] $Tolerance = $acceptablePercentRange / 100 - [double] $LowerBound = $expectedResultValue * (1 - $Tolerance) [double] $UpperBound = $expectedResultValue * (1 + $Tolerance) [double] $dblfoundResultValue = $foundResultValue # Check if the actual value is within the bounds - if ($dblfoundResultValue -ge $LowerBound -and $dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** Acceptable Allocated Value result ($dblfoundResultValue) is in the acceptable range +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue " + if ($dblfoundResultValue -le $UpperBound) { + Write-Host "** ** PASS! ** The Allocated Value result ($dblfoundResultValue) is under the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%.)" Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is OUT OF RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%.)" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Allocated Value ($dblfoundResultValue) is OUT OF ACCEPTABLE RANGE +/-$acceptablePercentRange% ($LowerBound -> $UpperBound) of expected value: $expectedResultValue" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%.)" Write-Host "** " } return $false # the values are too different @@ -99,9 +98,9 @@ if ($configFile -notlike "*.json") { $configFile += ".json" } -# This is special case that allows passing test without specifying CI_BDN_Confi_ at the beginning - need for perf test -if ($configFile -notlike "CI_BDN_Config_*") { - $configFile = "CI_BDN_Config_" + $configFile +# This is special case that allows passing test without specifying CI_CONFIG_BDN_ at the beginning - need for perf test +if ($configFile -notlike "CI_CONFIG_BDN_*") { + $configFile = "CI_CONFIG_BDN_" + $configFile } # Read the test config file and convert the JSON to a PowerShell object @@ -137,15 +136,10 @@ if ($IsLinux) { $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_linux $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_linux - $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_linux - $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_linux - $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_linux - $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_linux - - $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_linux - $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_linux - $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_linux - $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_linux + $expectedLua1AllocatedValue = $object.expectedLua1AllocatedValue_linux + $expectedLua2AllocatedValue = $object.expectedLua2AllocatedValue_linux + $expectedLua3AllocatedValue = $object.expectedLua3AllocatedValue_linux + $expectedLua4AllocatedValue = $object.expectedLua4AllocatedValue_linux } else { # Windows expected values @@ -155,15 +149,10 @@ else { $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_win $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_win - $expectedBasicLuaStress1AllocatedValue = $object.expectedBasicLuaStress1AllocatedValue_win - $expectedBasicLuaStress2AllocatedValue = $object.expectedBasicLuaStress2AllocatedValue_win - $expectedBasicLuaStress3AllocatedValue = $object.expectedBasicLuaStress3AllocatedValue_win - $expectedBasicLuaStress4AllocatedValue = $object.expectedBasicLuaStress4AllocatedValue_win - - $expectedBasicLuaRunner1AllocatedValue = $object.expectedBasicLuaRunner1AllocatedValue_win - $expectedBasicLuaRunner2AllocatedValue = $object.expectedBasicLuaRunner2AllocatedValue_win - $expectedBasicLuaRunner3AllocatedValue = $object.expectedBasicLuaRunner3AllocatedValue_win - $expectedBasicLuaRunner4AllocatedValue = $object.expectedBasicLuaRunner4AllocatedValue_win + $expectedLua1AllocatedValue = $object.expectedLua1AllocatedValue_win + $expectedLua2AllocatedValue = $object.expectedLua2AllocatedValue_win + $expectedLua3AllocatedValue = $object.expectedLua3AllocatedValue_win + $expectedLua4AllocatedValue = $object.expectedLua4AllocatedValue_win } # percent allowed variance when comparing expected vs actual found value - same for linux and windows. @@ -262,67 +251,34 @@ Get-Content $resultsFile | ForEach-Object { $testSuiteResult = $false } } - "*| BasicLuaStress1*" { - Write-Host "** BasicLuaStress1 Allocated Value test" - $foundBasicLuaStress1AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress1AllocatedValue $expectedBasicLuaStress1AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress2*" { - Write-Host "** BasicLuaStress2 Allocated Value test" - $foundBasicLuaStress2AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress2AllocatedValue $expectedBasicLuaStress2AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress3*" { - Write-Host "** BasicLuaStress3 Allocated Value test" - $foundBasicLuaStress3AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress3AllocatedValue $expectedBasicLuaStress3AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaStress4*" { - Write-Host "** BasicLuaStress4 Allocated Value test" - $foundBasicLuaStress4AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaStress4AllocatedValue $expectedBasicLuaStress4AllocatedValue $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| BasicLuaRunner1*" { - Write-Host "** BasicLuaRunner1 Allocated Value test" - $foundBasicLuaRunner1AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner1AllocatedValue $expectedBasicLuaRunner1AllocatedValue $acceptableAllocatedRange $true + "*| Lua1*" { + Write-Host "** Lua1 Allocated Value test" + $foundLua1AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLua1AllocatedValue $expectedLua1AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLuaRunner2*" { - Write-Host "** BasicLuaRunner2 Allocated Value test" - $foundBasicLuaRunner2AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner2AllocatedValue $expectedBasicLuaRunner2AllocatedValue $acceptableAllocatedRange $true + "*| Lua2*" { + Write-Host "** Lua2 Allocated Value test" + $foundLua2AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLua2AllocatedValue $expectedLua2AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - - "*| BasicLuaRunner3*" { - Write-Host "** BasicLuaRunner3 Allocated Value test" - $foundBasicLuaRunner3AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner3AllocatedValue $expectedBasicLuaRunner3AllocatedValue $acceptableAllocatedRange $true + "*| Lua3*" { + Write-Host "** Lua3 Allocated Value test" + $foundLua3AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLua3AllocatedValue $expectedLua3AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| BasicLuaRunner4*" { - Write-Host "** BasicLuaRunner4 Allocated Value test" - $foundBasicLuaRunner4AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundBasicLuaRunner4AllocatedValue $expectedBasicLuaRunner4AllocatedValue $acceptableAllocatedRange $true + "*| Lua4*" { + Write-Host "** Lua4 Allocated Value test" + $foundLua4AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLua4AllocatedValue $expectedLua4AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } From e22218c8c49b521fa8cd46187ac44ef5b862dc4a Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 22 Oct 2024 15:16:31 -0700 Subject: [PATCH 32/44] Hiding the Gen0 column for Lua benchmark --- benchmark/BDN.benchmark/Lua/LuaScripts.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/BDN.benchmark/Lua/LuaScripts.cs b/benchmark/BDN.benchmark/Lua/LuaScripts.cs index 738d9a0793..5c6060d2e6 100644 --- a/benchmark/BDN.benchmark/Lua/LuaScripts.cs +++ b/benchmark/BDN.benchmark/Lua/LuaScripts.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; using Garnet.server; namespace BDN.benchmark.Lua @@ -10,6 +11,7 @@ namespace BDN.benchmark.Lua /// Benchmark for Lua /// [MemoryDiagnoser] + [HideColumns(Column.Gen0)] public unsafe class LuaScripts { /// From f9a963b22f8b4f864967375055b603ba01701229 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 28 Oct 2024 14:07:18 -0700 Subject: [PATCH 33/44] Updated BDN test infrastructure to match the new BDN org. --- .github/workflows/ci-bdnbenchmark.yml | 2 +- ..._BDN_Benchmark_Cluster.ClusterMigrate.json | 19 + ..._Benchmark_Cluster.ClusterOperations.json} | 0 ...I_CONFIG_BDN_Benchmark_ClusterMigrate.json | 19 - ...I_CONFIG_BDN_Benchmark_Lua.LuaScripts.json | 19 + .../CI_CONFIG_BDN_Benchmark_Lua.json | 19 - ...Benchmark_Operations.BasicOperations.json} | 0 ...enchmark_Operations.ObjectOperations.json} | 20 +- test/BDNPerfTests/run_bdnperftest.ps1 | 408 +++++++++++++++--- 9 files changed, 391 insertions(+), 115 deletions(-) create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json rename test/BDNPerfTests/ConfigFiles/{CI_CONFIG_BDN_Benchmark_ClusterOperations.json => CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json} (100%) delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json create mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json rename test/BDNPerfTests/ConfigFiles/{CI_CONFIG_BDN_Benchmark_BasicOperations.json => CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json} (100%) rename test/BDNPerfTests/ConfigFiles/{CI_CONFIG_BDN_Benchmark_ObjectOperations.json => CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json} (75%) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 51dcbb0d2e..20b64a7134 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -39,7 +39,7 @@ jobs: os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] - test: [ 'Resp.RespParseStress', 'Resp.RespTsavoriteStress', 'Cluster.RespClusterMigrateBench', 'Cluster.RespClusterBench', 'Resp.RespLuaStress', 'Resp.RespLuaRunnerStress' ] + test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts' ] steps: - name: Check out code uses: actions/checkout@v4 diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json new file mode 100644 index 0000000000..2b14870bc9 --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json @@ -0,0 +1,19 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Cluster.ClusterMigrate.*", + + "expectedGet_None_AllocatedValue_linux": 0, + "expectedSet_None_AllocatedValue_linux": 0, + "expectedMGet_None_AllocatedValue_linux": 0, + "expectedMSet_None_AllocatedValue_linux": 0, + + "expectedGet_None_AllocatedValue_win": 0, + "expectedSet_None_AllocatedValue_win": 0, + "expectedMGet_None_AllocatedValue_win": 0, + "expectedMSet_None_AllocatedValue_win": 0, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json similarity index 100% rename from test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterOperations.json rename to test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json deleted file mode 100644 index 0a45091e1c..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ClusterMigrate.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "BDN.benchmark.Cluster.ClusterMigrate.*", - - "expectedGet_AllocatedValue_linux": 0, - "expectedSet_AllocatedValue_linux": 0, - "expectedMGet_AllocatedValue_linux": 0, - "expectedMSet_AllocatedValue_linux": 0, - - "expectedGet_AllocatedValue_win": 0, - "expectedSet_AllocatedValue_win": 0, - "expectedMGet_AllocatedValue_win": 0, - "expectedMSet_AllocatedValue_win": 0, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json new file mode 100644 index 0000000000..34c64f8f7b --- /dev/null +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json @@ -0,0 +1,19 @@ +{ + "configuration": "Release", + "framework": "net8.0", + "filter": "BDN.benchmark.Lua.*", + + "expectedScript1_None_AllocatedValue_linux": 24, + "expectedScript2_None_AllocatedValue_linux": 144, + "expectedScript3_None_AllocatedValue_linux": 240, + "expectedScript4_None_AllocatedValue_linux": 776, + + "expectedScript1_None_AllocatedValue_win": 24, + "expectedScript2_None_AllocatedValue_win": 144, + "expectedScript3_None_AllocatedValue_win": 240, + "expectedScript4_None_AllocatedValue_win": 776, + + "acceptableAllocatedRange": 10, + "acceptableError": ".1", + "acceptableStdDev": ".1" +} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json deleted file mode 100644 index 3b5c94a5f5..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "configuration": "Release", - "framework": "net8.0", - "filter": "BDN.benchmark.Lua.*", - - "expectedLua1AllocatedValue_linux": 24, - "expectedLua2AllocatedValue_linux": 144, - "expectedLua3AllocatedValue_linux": 240, - "expectedLua4AllocatedValue_linux": 776, - - "expectedLua1AllocatedValue_win": 24, - "expectedLua2AllocatedValue_win": 144, - "expectedLua3AllocatedValue_win": 240, - "expectedLua4AllocatedValue_win": 776, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_BasicOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json similarity index 100% rename from test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_BasicOperations.json rename to test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json similarity index 75% rename from test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json rename to test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json index 5782b0489c..c413bc3e20 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_ObjectOperations.json +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json @@ -3,11 +3,11 @@ "framework": "net8.0", "filter": "BDN.benchmark.Operations.ObjectOperations.*", - "expectedZAddRem_ACL_AllocatedValue_linux": 0, - "expectedLPushPop_ACL_AllocatedValue_linux": 0, - "expectedSAddRem_ACL_AllocatedValue_linux": 0, - "expectedHSetDel_ACL_AllocatedValue_linux": 0, - "expectedMyDictSetGet_ACL_AllocatedValue_linux": 0, + "expectedZAddRem_ACL_AllocatedValue_linux": 18400, + "expectedLPushPop_ACL_AllocatedValue_linux": 14400, + "expectedSAddRem_ACL_AllocatedValue_linux": 12800, + "expectedHSetDel_ACL_AllocatedValue_linux": 43201, + "expectedMyDictSetGet_ACL_AllocatedValue_linux": 24000, "expectedCustomProcSet_ACL_AllocatedValue_linux": 0, "expectedZAddRem_AOF_AllocatedValue_linux": 18400, @@ -25,11 +25,11 @@ "expectedCustomProcSet_None_AllocatedValue_linux": 0, - "expectedZAddRem_ACL_AllocatedValue_win": 0, - "expectedLPushPop_ACL_AllocatedValue_win": 0, - "expectedSAddRem_ACL_AllocatedValue_win": 0, - "expectedHSetDel_ACL_AllocatedValue_win": 0, - "expectedMyDictSetGet_ACL_AllocatedValue_win": 0, + "expectedZAddRem_ACL_AllocatedValue_win": 18400, + "expectedLPushPop_ACL_AllocatedValue_win": 14400, + "expectedSAddRem_ACL_AllocatedValue_win": 12800, + "expectedHSetDel_ACL_AllocatedValue_win": 43201, + "expectedMyDictSetGet_ACL_AllocatedValue_win": 24000, "expectedCustomProcSet_ACL_AllocatedValue_win": 0, "expectedZAddRem_AOF_AllocatedValue_win": 18400, diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index b22dcc08bf..4347b94cbe 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -6,20 +6,20 @@ Script to test for performance regressions in Allocated Memory using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. - CI_BDN_Config_Resp.RespParseStress.json + CI_CONFIG_BDN_Benchmark_BasicOperations.json NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% .EXAMPLE ./run_bdnperftest.ps1 - ./run_bdnperftest.ps1 CI_BDN_Config_Resp.RespParseStress.json + ./run_bdnperftest.ps1 CI_CONFIG_BDN_Benchmark_BasicOperations.json #> # Send the config file for the benchmark. Defaults to a simple one param ( - [string]$configFile = "CI_BDN_Config_Resp.RespParseStress.json" + [string]$configFile = "CI_CONFIG_BDN_Benchmark_BasicOperations.json" ) $OFS = "`r`n" @@ -41,17 +41,17 @@ function AnalyzeResult { # Check if the actual value is within the bounds if ($dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** The Allocated Value result ($dblfoundResultValue) is under the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%.)" + Write-Host "** ** PASS! ** The Allocated Value result ($dblfoundResultValue) is under the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%.)" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%.)" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " } return $false # the values are too different @@ -74,7 +74,10 @@ param ($ResultsLine, $columnNum) $columns = $ResultsLine.Trim('|').Split('|') $column = $columns | ForEach-Object { $_.Trim() } $foundValue = $column[$columnNum] - + if ($foundValue -eq "-") { + $foundValue = "0" + } + $foundValue = $foundValue.Trim(' B') $foundValue = $foundValue.Trim(' m') @@ -99,8 +102,8 @@ if ($configFile -notlike "*.json") { } # This is special case that allows passing test without specifying CI_CONFIG_BDN_ at the beginning - need for perf test -if ($configFile -notlike "CI_CONFIG_BDN_*") { - $configFile = "CI_CONFIG_BDN_" + $configFile +if ($configFile -notlike "CI_CONFIG_BDN_Benchmark_*") { + $configFile = "CI_CONFIG_BDN_Benchmark_" + $configFile } # Read the test config file and convert the JSON to a PowerShell object @@ -125,34 +128,90 @@ Write-Host " " $configuration = $object.configuration $framework = $object.framework $filter = $object.filter -$allocatedColumn = "-1" # Number of columns can differ a bit, but know the allocated one is always the last one +$allocatedColumn = "-1" # last one is allocated, just to ensure in case other column gets added +$paramNone = "None" +$paramDSV = "DSV" +$paramACL = "ACL" +$paramAOF = "AOF" # Set the expected values based on the OS if ($IsLinux) { # Linux expected values - $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_linux - $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_linux - $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_linux - $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_linux - $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_linux - - $expectedLua1AllocatedValue = $object.expectedLua1AllocatedValue_linux - $expectedLua2AllocatedValue = $object.expectedLua2AllocatedValue_linux - $expectedLua3AllocatedValue = $object.expectedLua3AllocatedValue_linux - $expectedLua4AllocatedValue = $object.expectedLua4AllocatedValue_linux + $expectedGet_None_AllocatedValue = $object.expectedGET_None_AllocatedValue_linux + $expectedSet_None_AllocatedValue = $object.expectedSET_None_AllocatedValue_linux + $expectedMGet_None_AllocatedValue = $object.expectedMGET_None_AllocatedValue_linux + $expectedMSet_None_AllocatedValue = $object.expectedMSET_None_AllocatedValue_linux + $expectedCPBSET_None_AllocatedValue = $object.expectedCPBSET_None_AllocatedValue_linux + + $expectedGet_DSV_AllocatedValue = $object.expectedGET_DSV_AllocatedValue_linux + $expectedSet_DSV_AllocatedValue = $object.expectedSET_DSV_AllocatedValue_linux + $expectedMGet_DSV_AllocatedValue = $object.expectedMGET_DSV_AllocatedValue_linux + $expectedMSet_DSV_AllocatedValue = $object.expectedMSET_DSV_AllocatedValue_linux + $expectedCPBSET_DSV_AllocatedValue = $object.expectedCPBSET_DSV_AllocatedValue_linux + + $expectedZAddRem_None_AllocatedValue = $object.expectedZAddRem_None_AllocatedValue_linux + $expectedLPushPop_None_AllocatedValue = $object.expectedLPushPop_None_AllocatedValue_linux + $expectedSAddRem_None_AllocatedValue = $object.expectedSAddRem_None_AllocatedValue_linux + $expectedHSetDel_None_AllocatedValue = $object.expectedHSetDel_None_AllocatedValue_linux + $expectedMyDictSetGet_None_AllocatedValue = $object.expectedMyDictSetGet_None_AllocatedValue_linux + $expectedZAddRem_ACL_AllocatedValue = $object.expectedZAddRem_ACL_AllocatedValue_linux + $expectedLPushPop_ACL_AllocatedValue = $object.expectedLPushPop_ACL_AllocatedValue_linux + $expectedSAddRem_ACL_AllocatedValue = $object.expectedSAddRem_ACL_AllocatedValue_linux + $expectedHSetDel_ACL_AllocatedValue = $object.expectedHSetDel_ACL_AllocatedValue_linux + $expectedMyDictSetGet_ACL_AllocatedValue = $object.expectedMyDictSetGet_ACL_AllocatedValue_linux + $expectedZAddRem_AOF_AllocatedValue = $object.expectedZAddRem_AOF_AllocatedValue_linux + $expectedLPushPop_AOF_AllocatedValue = $object.expectedLPushPop_AOF_AllocatedValue_linux + $expectedSAddRem_AOF_AllocatedValue = $object.expectedSAddRem_AOF_AllocatedValue_linux + $expectedHSetDel_AOF_AllocatedValue = $object.expectedHSetDel_AOF_AllocatedValue_linux + $expectedMyDictSetGet_AOF_AllocatedValue = $object.expectedMyDictSetGet_AOF_AllocatedValue_linux + + $expectedScript1_None_AllocatedValue = $object.expectedScript1_None_AllocatedValue_linux + $expectedScript2_None_AllocatedValue = $object.expectedScript2_None_AllocatedValue_linux + $expectedScript3_None_AllocatedValue = $object.expectedScript3_None_AllocatedValue_linux + $expectedScript4_None_AllocatedValue = $object.expectedScript4_None_AllocatedValue_linux + + $expectedInlinePing_ACL_AllocatedValue = $object.expectedInlinePing_ACL_AllocatedValue_linux + $expectedInlinePing_AOF_AllocatedValue = $object.expectedInlinePing_AOF_AllocatedValue_linux + $expectedInlinePing_None_AllocatedValue = $object.expectedInlinePing_None_AllocatedValue_linux } else { # Windows expected values - $expectedZAddRemAllocatedValue = $object.expectedZAddRemAllocatedValue_win - $expectedLPushPopAllocatedValue = $object.expectedLPushPopAllocatedValue_win - $expectedSAddRemAllocatedValue = $object.expectedSAddRemAllocatedValue_win - $expectedHSetDelAllocatedValue = $object.expectedHSetDelAllocatedValue_win - $expectedMyDictSetGetAllocatedValue = $object.expectedMyDictSetGetAllocatedValue_win - - $expectedLua1AllocatedValue = $object.expectedLua1AllocatedValue_win - $expectedLua2AllocatedValue = $object.expectedLua2AllocatedValue_win - $expectedLua3AllocatedValue = $object.expectedLua3AllocatedValue_win - $expectedLua4AllocatedValue = $object.expectedLua4AllocatedValue_win + $expectedGet_None_AllocatedValue = $object.expectedGET_None_AllocatedValue_win + $expectedSet_None_AllocatedValue = $object.expectedSET_None_AllocatedValue_win + $expectedMGet_None_AllocatedValue = $object.expectedMGET_None_AllocatedValue_win + $expectedMSet_None_AllocatedValue = $object.expectedMSET_None_AllocatedValue_win + $expectedCPBSET_None_AllocatedValue = $object.expectedCPBSET_None_AllocatedValue_win + + $expectedGet_DSV_AllocatedValue = $object.expectedGET_DSV_AllocatedValue_win + $expectedSet_DSV_AllocatedValue = $object.expectedSET_DSV_AllocatedValue_win + $expectedMGet_DSV_AllocatedValue = $object.expectedMGET_DSV_AllocatedValue_win + $expectedMSet_DSV_AllocatedValue = $object.expectedMSET_DSV_AllocatedValue_win + $expectedCPBSET_DSV_AllocatedValue = $object.expectedCPBSET_DSV_AllocatedValue_win + + $expectedZAddRem_None_AllocatedValue = $object.expectedZAddRem_None_AllocatedValue_win + $expectedLPushPop_None_AllocatedValue = $object.expectedLPushPop_None_AllocatedValue_win + $expectedSAddRem_None_AllocatedValue = $object.expectedSAddRem_None_AllocatedValue_win + $expectedHSetDel_None_AllocatedValue = $object.expectedHSetDel_None_AllocatedValue_win + $expectedMyDictSetGet_None_AllocatedValue = $object.expectedMyDictSetGet_None_AllocatedValue_win + $expectedZAddRem_ACL_AllocatedValue = $object.expectedZAddRem_ACL_AllocatedValue_win + $expectedLPushPop_ACL_AllocatedValue = $object.expectedLPushPop_ACL_AllocatedValue_win + $expectedSAddRem_ACL_AllocatedValue = $object.expectedSAddRem_ACL_AllocatedValue_win + $expectedHSetDel_ACL_AllocatedValue = $object.expectedHSetDel_ACL_AllocatedValue_win + $expectedMyDictSetGet_ACL_AllocatedValue = $object.expectedMyDictSetGet_ACL_AllocatedValue_win + $expectedZAddRem_AOF_AllocatedValue = $object.expectedZAddRem_AOF_AllocatedValue_win + $expectedLPushPop_AOF_AllocatedValue = $object.expectedLPushPop_AOF_AllocatedValue_win + $expectedSAddRem_AOF_AllocatedValue = $object.expectedSAddRem_AOF_AllocatedValue_win + $expectedHSetDel_AOF_AllocatedValue = $object.expectedHSetDel_AOF_AllocatedValue_win + $expectedMyDictSetGet_AOF_AllocatedValue = $object.expectedMyDictSetGet_AOF_AllocatedValue_win + + $expectedScript1_None_AllocatedValue = $object.expectedScript1_None_AllocatedValue_win + $expectedScript2_None_AllocatedValue = $object.expectedScript2_None_AllocatedValue_win + $expectedScript3_None_AllocatedValue = $object.expectedScript3_None_AllocatedValue_win + $expectedScript4_None_AllocatedValue = $object.expectedScript4_None_AllocatedValue_win + + $expectedInlinePing_ACL_AllocatedValue = $object.expectedInlinePing_ACL_AllocatedValue_win + $expectedInlinePing_AOF_AllocatedValue = $object.expectedInlinePing_AOF_AllocatedValue_win + $expectedInlinePing_None_AllocatedValue = $object.expectedInlinePing_None_AllocatedValue_win } # percent allowed variance when comparing expected vs actual found value - same for linux and windows. @@ -211,78 +270,295 @@ $testSuiteResult = $true Get-Content $resultsFile | ForEach-Object { $line = $_ switch -Wildcard ($line) { - "*| ZAddRem*" { - Write-Host "** ZAddRem Allocated Value test" - $foundZAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRemAllocatedValue $expectedZAddRemAllocatedValue $acceptableAllocatedRange $true + "*| Set*$paramNone*" { + Write-Host "** Set ($paramNone) Allocated Value test" + $foundSet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSet_None_AllocatedValue $expectedSet_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Get*$paramNone*" { + Write-Host "** Get ($paramNone) Allocated Value test" + $foundGet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundGet_None_AllocatedValue $expectedGet_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MSet*$paramNone*" { + Write-Host "** MSet ($paramNone) Allocated Value test" + $foundMSet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMSet_None_AllocatedValue $expectedMSet_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MGet*$paramNone*" { + Write-Host "** MGet ($paramNone) Allocated Value test" + $foundMGet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMGet_None_AllocatedValue $expectedMGet_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| CPBSET*$paramNone*" { + Write-Host "** CPBSET ($paramNone) Allocated Value test" + $foundCPBSET_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCPBSET_None_AllocatedValue $expectedCPBSET_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + "*| Set*$paramDSV*" { + Write-Host "** Set ($paramDSV) Allocated Value test" + $foundSet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSet_DSV_AllocatedValue $expectedSet_DSV_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Get*$paramDSV*" { + Write-Host "** Get ($paramDSV) Allocated Value test" + $foundGet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundGet_DSV_AllocatedValue $expectedGet_DSV_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MSet*$paramDSV*" { + Write-Host "** MSet ($paramDSV) Allocated Value test" + $foundMSet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMSet_DSV_AllocatedValue $expectedMSet_DSV_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MGet*$paramDSV*" { + Write-Host "** MGet ($paramDSV) Allocated Value test" + $foundMGet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMGet_DSV_AllocatedValue $expectedMGet_DSV_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| LPushPop*" { - Write-Host "** LPushPop Allocated Value test" - $foundLPushPopAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPopAllocatedValue $expectedLPushPopAllocatedValue $acceptableAllocatedRange $true + "*| CPBSET*$paramDSV*" { + Write-Host "** CPBSET ($paramDSV) Allocated Value test" + $foundCPBSET_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCPBSET_DSV_AllocatedValue $expectedCPBSET_DSV_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| SAddRem*" { - Write-Host "** SAddRem Allocated Value test" - $foundSAddRemAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRemAllocatedValue $expectedSAddRemAllocatedValue $acceptableAllocatedRange $true + + "*| ZAddRem*$paramNone*" { + Write-Host "** ZAddRem ($paramNone) Allocated Value test" + $foundZAddRem_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRem_None_AllocatedValue $expectedZAddRem_None_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| HSetDel*" { - Write-Host "** HSetDel Allocated Value test" - $foundHSetDelAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDelAllocatedValue $expectedHSetDelAllocatedValue $acceptableAllocatedRange $true + "*| LPushPop*$paramNone*" { + Write-Host "** LPushPop ($paramNone) Allocated Value test" + $foundLPushPop_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPop_None_AllocatedValue $expectedLPushPop_None_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| MyDictSetGet*" { - Write-Host "** MyDictSetGet Allocated Value test" - $foundMyDictSetGetAllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGetAllocatedValue $expectedMyDictSetGetAllocatedValue $acceptableAllocatedRange $true + "*| SAddRem*$paramNone*" { + Write-Host "** SAddRem ($paramNone) Allocated Value test" + $foundSAddRem_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRem_None_AllocatedValue $expectedSAddRem_None_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| Lua1*" { - Write-Host "** Lua1 Allocated Value test" - $foundLua1AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLua1AllocatedValue $expectedLua1AllocatedValue $acceptableAllocatedRange $true + "*| HSetDel*$paramNone*" { + Write-Host "** HSetDel ($paramNone) Allocated Value test" + $foundHSetDel_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDel_None_AllocatedValue $expectedHSetDel_None_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| Lua2*" { - Write-Host "** Lua2 Allocated Value test" - $foundLua2AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLua2AllocatedValue $expectedLua2AllocatedValue $acceptableAllocatedRange $true + "*| MyDictSetGet*$paramNone*" { + Write-Host "** MyDictSetGet ($paramNone) Allocated Value test" + $foundMyDictSetGet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGet_None_AllocatedValue $expectedMyDictSetGet_None_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| Lua3*" { - Write-Host "** Lua3 Allocated Value test" - $foundLua3AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLua3AllocatedValue $expectedLua3AllocatedValue $acceptableAllocatedRange $true + "*| CustomProcSet*$paramNone*" { + Write-Host "** MyDictSetGet ($paramNone) Allocated Value test" + $foundCustomProcSet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCustomProcSet_None_AllocatedValue $expectedCustomProcSet_None_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - "*| Lua4*" { - Write-Host "** Lua4 Allocated Value test" - $foundLua4AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLua4AllocatedValue $expectedLua4AllocatedValue $acceptableAllocatedRange $true + "*| ZAddRem*$paramACL*" { + Write-Host "** ZAddRem ($paramACL) Allocated Value test" + $foundZAddRem_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRem_ACL_AllocatedValue $expectedZAddRem_ACL_AllocatedValue $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } + "*| LPushPop*$paramACL*" { + Write-Host "** LPushPop ($paramACL) Allocated Value test" + $foundLPushPop_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPop_ACL_AllocatedValue $expectedLPushPop_ACL_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| SAddRem*$paramACL*" { + Write-Host "** SAddRem ($paramACL) Allocated Value test" + $foundSAddRem_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRem_ACL_AllocatedValue $expectedSAddRem_ACL_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| HSetDel*$paramACL*" { + Write-Host "** HSetDel ($paramACL) Allocated Value test" + $foundHSetDel_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDel_ACL_AllocatedValue $expectedHSetDel_ACL_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MyDictSetGet*$paramACL*" { + Write-Host "** MyDictSetGet ($paramACL) Allocated Value test" + $foundMyDictSetGet_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGet_ACL_AllocatedValue $expectedMyDictSetGet_ACL_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| CustomProcSet*$paramACL*" { + Write-Host "** MyDictSetGet ($paramACL) Allocated Value test" + $foundCustomProcSet_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCustomProcSet_ACL_AllocatedValue $expectedCustomProcSet_ACL_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + "*| ZAddRem*$paramAOF*" { + Write-Host "** ZAddRem ($paramAOF) Allocated Value test" + $foundZAddRem_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRem_AOF_AllocatedValue $expectedZAddRem_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| LPushPop*$paramAOF*" { + Write-Host "** LPushPop ($paramAOF) Allocated Value test" + $foundLPushPop_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPop_AOF_AllocatedValue $expectedLPushPop_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| SAddRem*$paramAOF*" { + Write-Host "** SAddRem ($paramAOF) Allocated Value test" + $foundSAddRem_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRem_AOF_AllocatedValue $expectedSAddRem_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| HSetDel*$paramAOF*" { + Write-Host "** HSetDel ($paramAOF) Allocated Value test" + $foundHSetDel_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDel_AOF_AllocatedValue $expectedHSetDel_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| MyDictSetGet*$paramAOF*" { + Write-Host "** MyDictSetGet ($paramAOF) Allocated Value test" + $foundMyDictSetGet_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGet_AOF_AllocatedValue $expectedMyDictSetGet_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| CustomProcSet*$paramAOF*" { + Write-Host "** MyDictSetGet ($paramAOF) Allocated Value test" + $foundMyCustomProcSet_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyCustomProcSet_AOF_AllocatedValue $expectedCustomProcSetSetGet_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + "*| Script1*$paramNone*" { + Write-Host "** Lua Script1 ($paramNone) Allocated Value test" + $foundScript1_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript1_None_AllocatedValue $expectedScript1_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Script2*$paramNone*" { + Write-Host "** Lua Script2 ($paramNone) Allocated Value test" + $foundScript2_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript2_None_AllocatedValue $expectedScript2_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Script3*$paramNone*" { + Write-Host "** Lua Script3 ($paramNone) Allocated Value test" + $foundScript3_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript3_None_AllocatedValue $expectedScript3_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| Script4*$paramNone*" { + Write-Host "** LuaScript4 ($paramNone) Allocated Value test" + $foundScript4_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript4_None_AllocatedValue $expectedScript4_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + + "*| InlinePing*$paramNone*" { + Write-Host "** InlinePing ($paramNone) Allocated Value test" + $foundInlinePing_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundInlinePing_None_AllocatedValue $expectedInlinePing_None_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + "*| InlinePing*$paramACL*" { + Write-Host "** InlinePing ($paramACL) Allocated Value test" + $foundInlinePing_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundInlinePing_ACL_AllocatedValue $expectedInlinePing_ACL_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + "*| InlinePing*$paramAOF*" { + Write-Host "** InlinePing ($paramAOF) Allocated Value test" + $foundInlinePing_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundInlinePing_AOF_AllocatedValue $expectedInlinePing_AOF_AllocatedValue $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } + } + + } } From 665bda1ae92aaff071c7d8c05bb2edd11b23c0a7 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 28 Oct 2024 14:22:42 -0700 Subject: [PATCH 34/44] Fix bug in ObjectOperations --- test/BDNPerfTests/run_bdnperftest.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 4347b94cbe..3c27ed9f84 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -79,6 +79,7 @@ param ($ResultsLine, $columnNum) } $foundValue = $foundValue.Trim(' B') + $foundValue = $foundValue.Trim(' K') $foundValue = $foundValue.Trim(' m') return $foundValue From 0d1fd6fdb128f4892ef3bcc20ea725d764d17748 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 28 Oct 2024 16:12:56 -0700 Subject: [PATCH 35/44] Set max items on chart to 50 ... can modify later if want --- .github/workflows/ci-bdnbenchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 20b64a7134..f99ab4d190 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -99,5 +99,4 @@ jobs: gh-pages-branch: 'continuousbenchmark' benchmark-data-dir-path: 'website/static/charts' auto-push: true - - + max-items-in-chart: 50 From 6725d994941b0013ea1e58da7f78e4296ef1217a Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 29 Oct 2024 13:21:29 -0700 Subject: [PATCH 36/44] Added the run on push to main --- .github/workflows/ci-bdnbenchmark.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index f99ab4d190..bb0a6e5eb8 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -1,7 +1,10 @@ name: Garnet CI BDN.benchmark on: workflow_dispatch: - + push: + branches: + - main + env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_NOLOGO: true @@ -14,7 +17,7 @@ permissions: jobs: changes: name: Check for changes - runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code + runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v4 From da6b517246291a60f980ef4c0a59eb7830457268 Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 29 Oct 2024 13:27:31 -0700 Subject: [PATCH 37/44] Put Performance Pipeline back --- .../pipelines/azure-pipelines-performance.yml | 237 +++++++++++++++++- 1 file changed, 226 insertions(+), 11 deletions(-) diff --git a/.azure/pipelines/azure-pipelines-performance.yml b/.azure/pipelines/azure-pipelines-performance.yml index 8829756c3a..6806f286e0 100644 --- a/.azure/pipelines/azure-pipelines-performance.yml +++ b/.azure/pipelines/azure-pipelines-performance.yml @@ -3,6 +3,13 @@ trigger: include: - refs/heads/main +schedules: +- cron: "0 7 * * *" # 7:00 UTC is 11:00PM Pacific + branches: + include: + - refs/heads/main + always: true + resources: repositories: - repository: self @@ -42,26 +49,124 @@ jobs: - script: dotnet restore displayName: .NET Restore + - task: DotNetCoreCLI@2 + displayName: '.NET Core build Cluster binaries and tests $(buildConfiguration)' + inputs: + command: 'build' + projects: '**/Garnet.test.cluster.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: '.NET Core build Garnet binaries and tests $(buildConfiguration)' + inputs: + command: 'build' + projects: '**/Garnet.test.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: '.NET Core build Garnet Server' + inputs: + command: 'build' + projects: '**/GarnetServer.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: '.NET Core build Resp.benchmark' + inputs: + command: 'build' + projects: '**/Resp.benchmark.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: PowerShell@2 + displayName: '1: Resp.Benchmark ONLINE; Get Set; 1 Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_GetSet_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '2: Resp.Benchmark ONLINE; Get Set; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_GetSet_MaxThr.json' + workingDirectory: test/PerfRegressionTesting + - task: PowerShell@2 - displayName: '1: BDN.benchmark ' + displayName: '3: Resp.Benchmark ONLINE; ZADD ZREM; 1 Thread' continueOnError: true enabled: true inputs: targetType: filePath - filePath: './test/BDNPerfTests/run_bdnperftest.ps1' - arguments: 'CI_BDN_Config_RespParseStress.json' - workingDirectory: test/BDNPerfTests + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '4: Resp.Benchmark ONLINE; ZADD ZREM; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_MaxThr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '5: Resp.Benchmark OFFLINE; Get; 1 Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_Get_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '6: Resp.Benchmark OFFLINE; Get; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_Get_MaxThr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '7: Resp.Benchmark OFFLINE; ZADDREM; 1 Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '8: Resp.Benchmark OFFLINE; ZADDREM; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_MaxThr.json' + workingDirectory: test/PerfRegressionTesting - task: CopyFiles@2 displayName: 'Copy Results Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/BDNPerfTests/results/**/*' + Contents: '**/test/PerfRegressionTesting/results/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: CopyFiles@2 displayName: 'Copy ErrorLog Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/BDNPerfTests/errorlog/**/*' + Contents: '**/test/PerfRegressionTesting/errorlog/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: PublishBuildArtifacts@1 @@ -96,26 +201,136 @@ jobs: - task: NuGetAuthenticate@1 displayName: 'NuGet Authenticate' + - task: DotNetCoreCLI@2 + displayName: 'Build Garnet Cluster binaries and tests $(buildConfiguration)' + inputs: + command: 'build' + projects: '**/Garnet.test.cluster.csproj' + arguments: '--configuration $(buildConfiguration)' + + - script: nuget restore + displayName: Nuget Restore + + - task: DotNetCoreCLI@2 + displayName: 'Build Garnet binaries and tests $(buildConfiguration)' + inputs: + command: 'build' + projects: '**/Garnet.test.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: 'Build Garnet Server $(buildConfiguration)' + inputs: + command: 'build' + projects: '**/GarnetServer.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: DotNetCoreCLI@2 + displayName: '.NET Core build Resp.benchmark' + inputs: + command: 'build' + projects: '**/Resp.benchmark.csproj' + arguments: '--configuration $(buildConfiguration)' + + - task: PowerShell@2 + displayName: 'Run Tsavorite Benchmark Test' + enabled: false + inputs: + targetType: filePath + filePath: './libs/storage/Tsavorite/cs/benchmark/scripts/run_benchmark.ps1' + arguments: '$(Build.SourcesDirectory)' + workingDirectory: libs/storage/Tsavorite/cs/benchmark/scripts + - task: PowerShell@2 displayName: '1: Resp.Benchmark ONLINE; Get Set; 1 Thread' continueOnError: true enabled: true inputs: targetType: filePath - filePath: './test/BDNPerfTests/run_bdnperftest.ps1' - arguments: 'CI_BDN_Config_RespParseStress.json' - workingDirectory: test/BDNPerfTests + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_GetSet_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '2: Resp.Benchmark ONLINE; Get Set; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_GetSet_MaxThr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '3: Resp.Benchmark ONLINE; ZADD ZREM; 1 Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '4: Resp.Benchmark ONLINE; ZADD ZREM; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Online_ZADDZREM_MaxThr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '5: Resp.Benchmark OFFLINE; Get; 1 Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_Get_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '6: Resp.Benchmark OFFLINE; Get; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_Get_MaxThr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '7: Resp.Benchmark OFFLINE; ZADDREM; 1 Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_1Thr.json' + workingDirectory: test/PerfRegressionTesting + + - task: PowerShell@2 + displayName: '8: Resp.Benchmark OFFLINE; ZADDREM; Max Thread' + continueOnError: true + enabled: true + inputs: + targetType: filePath + filePath: './test/PerfRegressionTesting/run_benchmark.ps1' + arguments: 'ConfigFiles\CI_Config_Offline_ZADDREM_MaxThr.json' + workingDirectory: test/PerfRegressionTesting - task: CopyFiles@2 displayName: 'Copy Results Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/BDNPerfTests/results/**/*' + Contents: '**/test/PerfRegressionTesting/results/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: CopyFiles@2 displayName: 'Copy ErrorLog Files to: $(build.artifactstagingdirectory)' inputs: - Contents: '**/test/BDNPerfTests/errorlog/**/*' + Contents: '**/test/PerfRegressionTesting/errorlog/**/*' TargetFolder: $(build.artifactstagingdirectory) - task: PublishBuildArtifacts@1 From 3871a3f846d3311622f0517098062fc9ff6f670e Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 29 Oct 2024 13:32:16 -0700 Subject: [PATCH 38/44] Fixed the default case when no parameters sent to ps1 --- test/BDNPerfTests/run_bdnperftest.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 3c27ed9f84..971a82742f 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -4,22 +4,23 @@ .DESCRIPTION - Script to test for performance regressions in Allocated Memory using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. Any of these can be sent as the parameter to the file. + Script to test for performance regressions in Allocated Memory using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. + Any of these can be sent as the parameter to the file. - CI_CONFIG_BDN_Benchmark_BasicOperations.json + CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% .EXAMPLE ./run_bdnperftest.ps1 - ./run_bdnperftest.ps1 CI_CONFIG_BDN_Benchmark_BasicOperations.json + ./run_bdnperftest.ps1 CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json #> # Send the config file for the benchmark. Defaults to a simple one param ( - [string]$configFile = "CI_CONFIG_BDN_Benchmark_BasicOperations.json" + [string]$configFile = "CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json" ) $OFS = "`r`n" From bf067270173e446efc2bf964184f4a594436b460 Mon Sep 17 00:00:00 2001 From: darrenge Date: Thu, 31 Oct 2024 15:00:57 -0700 Subject: [PATCH 39/44] Cleaned up config files: have shorter names for values not have configuration or framework (moved to ps1 file) not have windows / linux differnce in expected Moved acceptable allocated range to script modified the script: to only load values for the specific test instead of looking for all values changes to match the changes in config (shorter names, etc) --- ..._BDN_Benchmark_Cluster.ClusterMigrate.json | 20 +- ...N_Benchmark_Cluster.ClusterOperations.json | 38 +-- ...I_CONFIG_BDN_Benchmark_Lua.LuaScripts.json | 20 +- ..._Benchmark_Operations.BasicOperations.json | 17 +- ...Benchmark_Operations.ObjectOperations.json | 68 ++--- test/BDNPerfTests/run_bdnperftest.ps1 | 288 ++++++++---------- 6 files changed, 176 insertions(+), 275 deletions(-) diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json index 2b14870bc9..4e4b10edc2 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json @@ -1,19 +1,9 @@ { - "configuration": "Release", - "framework": "net8.0", "filter": "BDN.benchmark.Cluster.ClusterMigrate.*", - "expectedGet_None_AllocatedValue_linux": 0, - "expectedSet_None_AllocatedValue_linux": 0, - "expectedMGet_None_AllocatedValue_linux": 0, - "expectedMSet_None_AllocatedValue_linux": 0, - - "expectedGet_None_AllocatedValue_win": 0, - "expectedSet_None_AllocatedValue_win": 0, - "expectedMGet_None_AllocatedValue_win": 0, - "expectedMSet_None_AllocatedValue_win": 0, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" + "_comment": "All these values are the expected Allocated.", + "expectedGet_None": 0, + "expectedSet_None": 0, + "expectedMGet_None": 0, + "expectedMSet_None": 0 } diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json index 66006975c3..2676f419ba 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json @@ -1,31 +1,15 @@ { - "configuration": "Release", - "framework": "net8.0", "filter": "BDN.benchmark.Cluster.ClusterOperations.*", - "expectedGet_DSV_AllocatedValue_linux": 0, - "expectedSet_DSV_AllocatedValue_linux": 0, - "expectedMGet_DSV_AllocatedValue_linux": 0, - "expectedMSet_DSV_AllocatedValue_linux": 0, - "expectedCPBSet_DSV_AllocatedValue_linux": 0, - "expectedGet_None_AllocatedValue_linux": 0, - "expectedSet_None_AllocatedValue_linux": 0, - "expectedMGet_None_AllocatedValue_linux": 0, - "expectedMSet_None_AllocatedValue_linux": 0, - "expectedCPBSet_None_AllocatedValue_linux": 0, - - "expectedGet_DSV_AllocatedValue_win": 0, - "expectedSet_DSV_AllocatedValue_win": 0, - "expectedMGet_DSV_AllocatedValue_win": 0, - "expectedMSet_DSV_AllocatedValue_win": 0, - "expectedCPBSet_DSV_AllocatedValue_win": 0, - "expectedGet_None_AllocatedValue_win": 0, - "expectedSet_None_AllocatedValue_win": 0, - "expectedMGet_None_AllocatedValue_win": 0, - "expectedMSet_None_AllocatedValue_win": 0, - "expectedCPBSet_None_AllocatedValue_win": 0, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" + "_comment": "All these values are the expected Allocated.", + "expectedGet_DSV": 0, + "expectedSet_DSV": 0, + "expectedMGet_DSV": 0, + "expectedMSet_DSV": 0, + "expectedCPBSet_DSV": 0, + "expectedGet_None": 0, + "expectedSet_None": 0, + "expectedMGet_None": 0, + "expectedMSet_None": 0, + "expectedCPBSet_None": 0 } diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json index 34c64f8f7b..ab29c9aea7 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json @@ -1,19 +1,9 @@ { - "configuration": "Release", - "framework": "net8.0", "filter": "BDN.benchmark.Lua.*", - - "expectedScript1_None_AllocatedValue_linux": 24, - "expectedScript2_None_AllocatedValue_linux": 144, - "expectedScript3_None_AllocatedValue_linux": 240, - "expectedScript4_None_AllocatedValue_linux": 776, - - "expectedScript1_None_AllocatedValue_win": 24, - "expectedScript2_None_AllocatedValue_win": 144, - "expectedScript3_None_AllocatedValue_win": 240, - "expectedScript4_None_AllocatedValue_win": 776, - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" + "_comment": "All these values are the expected Allocated.", + "expectedScript1_None": 24, + "expectedScript2_None": 144, + "expectedScript3_None": 240, + "expectedScript4_None": 776 } diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json index 9dd402b8d9..1f699f47a8 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json @@ -1,17 +1,8 @@ { - "configuration": "Release", - "framework": "net8.0", "filter": "BDN.benchmark.Operations.BasicOperations.*", - "expectedInlinePing_ACL_AllocatedValue_linux": 0, - "expectedInlinePing_AOF_AllocatedValue_linux": 0, - "expectedInlinePing_None_AllocatedValue_linux": 0, - - "expectedInlinePing_ACL_AllocatedValue_win": 0, - "expectedInlinePing_AOF_AllocatedValue_win": 0, - "expectedInlinePing_None_AllocatedValue_win": 0, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" + "_comment": "All these values are the expected Allocated.", + "expectedInlinePing_ACL_Allocated": 0, + "expectedInlinePing_AOF_Allocated": 0, + "expectedInlinePing_None_Allocated": 0 } diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json index c413bc3e20..7227473db5 100644 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json +++ b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json @@ -1,53 +1,27 @@ { - "configuration": "Release", - "framework": "net8.0", "filter": "BDN.benchmark.Operations.ObjectOperations.*", - "expectedZAddRem_ACL_AllocatedValue_linux": 18400, - "expectedLPushPop_ACL_AllocatedValue_linux": 14400, - "expectedSAddRem_ACL_AllocatedValue_linux": 12800, - "expectedHSetDel_ACL_AllocatedValue_linux": 43201, - "expectedMyDictSetGet_ACL_AllocatedValue_linux": 24000, - "expectedCustomProcSet_ACL_AllocatedValue_linux": 0, + "_comment": "All these values are the expected Allocated.", + "expectedZAddRem_ACL": 18400, + "expectedLPushPop_ACL": 14400, + "expectedSAddRem_ACL": 12800, + "expectedHSetDel_ACL": 43201, + "expectedMyDictSetGet_ACL": 24000, + "expectedCustomProcSet_ACL": 0, + + "expectedZAddRem_AOF": 18400, + "expectedLPushPop_AOF": 14400, + "expectedSAddRem_AOF": 12800, + "expectedHSetDel_AOF": 43201, + "expectedMyDictSetGet_AOF": 24000, + "expectedCustomProcSet_AOF": 0, + + "expectedZAddRem_None": 18400, + "expectedLPushPop_None": 14400, + "expectedSAddRem_None": 12800, + "expectedHSetDel_None": 43201, + "expectedMyDictSetGet_None": 24000, + "expectedCustomProcSet_None": 0 - "expectedZAddRem_AOF_AllocatedValue_linux": 18400, - "expectedLPushPop_AOF_AllocatedValue_linux": 14400, - "expectedSAddRem_AOF_AllocatedValue_linux": 12800, - "expectedHSetDel_AOF_AllocatedValue_linux": 43201, - "expectedMyDictSetGet_AOF_AllocatedValue_linux": 24000, - "expectedCustomProcSet_AOF_AllocatedValue_linux": 0, - - "expectedZAddRem_None_AllocatedValue_linux": 18400, - "expectedLPushPop_None_AllocatedValue_linux": 14400, - "expectedSAddRem_None_AllocatedValue_linux": 12800, - "expectedHSetDel_None_AllocatedValue_linux": 43201, - "expectedMyDictSetGet_None_AllocatedValue_linux": 24000, - "expectedCustomProcSet_None_AllocatedValue_linux": 0, - - - "expectedZAddRem_ACL_AllocatedValue_win": 18400, - "expectedLPushPop_ACL_AllocatedValue_win": 14400, - "expectedSAddRem_ACL_AllocatedValue_win": 12800, - "expectedHSetDel_ACL_AllocatedValue_win": 43201, - "expectedMyDictSetGet_ACL_AllocatedValue_win": 24000, - "expectedCustomProcSet_ACL_AllocatedValue_win": 0, - - "expectedZAddRem_AOF_AllocatedValue_win": 18400, - "expectedLPushPop_AOF_AllocatedValue_win": 14400, - "expectedSAddRem_AOF_AllocatedValue_win": 12800, - "expectedHSetDel_AOF_AllocatedValue_win": 43201, - "expectedMyDictSetGet_AOF_AllocatedValue_win": 24000, - "expectedCustomProcSet_AOF_AllocatedValue_win": 0, - - "expectedZAddRem_None_AllocatedValue_win": 18400, - "expectedLPushPop_None_AllocatedValue_win": 14400, - "expectedSAddRem_None_AllocatedValue_win": 12800, - "expectedHSetDel_None_AllocatedValue_win": 43201, - "expectedMyDictSetGet_None_AllocatedValue_win": 24000, - "expectedCustomProcSet_None_AllocatedValue_win": 0, - - "acceptableAllocatedRange": 10, - "acceptableError": ".1", - "acceptableStdDev": ".1" } diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 971a82742f..54534fa629 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -126,99 +126,73 @@ if ($IsLinux) { Write-Host "************** Start BDN.benchmark Test Run ********************" Write-Host " " -# Set all the config options (args to benchmark app) based on values from config json file -$configuration = $object.configuration -$framework = $object.framework -$filter = $object.filter +# Set all the config options (args to benchmark app) +$configuration = "Release" +$framework = "net8.0" $allocatedColumn = "-1" # last one is allocated, just to ensure in case other column gets added $paramNone = "None" $paramDSV = "DSV" $paramACL = "ACL" $paramAOF = "AOF" +$acceptableAllocatedRange = "10" # percent allowed variance when comparing expected vs actual found value - same for linux and windows. +$filter = $object.filter -# Set the expected values based on the OS -if ($IsLinux) { - # Linux expected values - $expectedGet_None_AllocatedValue = $object.expectedGET_None_AllocatedValue_linux - $expectedSet_None_AllocatedValue = $object.expectedSET_None_AllocatedValue_linux - $expectedMGet_None_AllocatedValue = $object.expectedMGET_None_AllocatedValue_linux - $expectedMSet_None_AllocatedValue = $object.expectedMSET_None_AllocatedValue_linux - $expectedCPBSET_None_AllocatedValue = $object.expectedCPBSET_None_AllocatedValue_linux - - $expectedGet_DSV_AllocatedValue = $object.expectedGET_DSV_AllocatedValue_linux - $expectedSet_DSV_AllocatedValue = $object.expectedSET_DSV_AllocatedValue_linux - $expectedMGet_DSV_AllocatedValue = $object.expectedMGET_DSV_AllocatedValue_linux - $expectedMSet_DSV_AllocatedValue = $object.expectedMSET_DSV_AllocatedValue_linux - $expectedCPBSET_DSV_AllocatedValue = $object.expectedCPBSET_DSV_AllocatedValue_linux - - $expectedZAddRem_None_AllocatedValue = $object.expectedZAddRem_None_AllocatedValue_linux - $expectedLPushPop_None_AllocatedValue = $object.expectedLPushPop_None_AllocatedValue_linux - $expectedSAddRem_None_AllocatedValue = $object.expectedSAddRem_None_AllocatedValue_linux - $expectedHSetDel_None_AllocatedValue = $object.expectedHSetDel_None_AllocatedValue_linux - $expectedMyDictSetGet_None_AllocatedValue = $object.expectedMyDictSetGet_None_AllocatedValue_linux - $expectedZAddRem_ACL_AllocatedValue = $object.expectedZAddRem_ACL_AllocatedValue_linux - $expectedLPushPop_ACL_AllocatedValue = $object.expectedLPushPop_ACL_AllocatedValue_linux - $expectedSAddRem_ACL_AllocatedValue = $object.expectedSAddRem_ACL_AllocatedValue_linux - $expectedHSetDel_ACL_AllocatedValue = $object.expectedHSetDel_ACL_AllocatedValue_linux - $expectedMyDictSetGet_ACL_AllocatedValue = $object.expectedMyDictSetGet_ACL_AllocatedValue_linux - $expectedZAddRem_AOF_AllocatedValue = $object.expectedZAddRem_AOF_AllocatedValue_linux - $expectedLPushPop_AOF_AllocatedValue = $object.expectedLPushPop_AOF_AllocatedValue_linux - $expectedSAddRem_AOF_AllocatedValue = $object.expectedSAddRem_AOF_AllocatedValue_linux - $expectedHSetDel_AOF_AllocatedValue = $object.expectedHSetDel_AOF_AllocatedValue_linux - $expectedMyDictSetGet_AOF_AllocatedValue = $object.expectedMyDictSetGet_AOF_AllocatedValue_linux - - $expectedScript1_None_AllocatedValue = $object.expectedScript1_None_AllocatedValue_linux - $expectedScript2_None_AllocatedValue = $object.expectedScript2_None_AllocatedValue_linux - $expectedScript3_None_AllocatedValue = $object.expectedScript3_None_AllocatedValue_linux - $expectedScript4_None_AllocatedValue = $object.expectedScript4_None_AllocatedValue_linux - - $expectedInlinePing_ACL_AllocatedValue = $object.expectedInlinePing_ACL_AllocatedValue_linux - $expectedInlinePing_AOF_AllocatedValue = $object.expectedInlinePing_AOF_AllocatedValue_linux - $expectedInlinePing_None_AllocatedValue = $object.expectedInlinePing_None_AllocatedValue_linux -} -else { - # Windows expected values - $expectedGet_None_AllocatedValue = $object.expectedGET_None_AllocatedValue_win - $expectedSet_None_AllocatedValue = $object.expectedSET_None_AllocatedValue_win - $expectedMGet_None_AllocatedValue = $object.expectedMGET_None_AllocatedValue_win - $expectedMSet_None_AllocatedValue = $object.expectedMSET_None_AllocatedValue_win - $expectedCPBSET_None_AllocatedValue = $object.expectedCPBSET_None_AllocatedValue_win - - $expectedGet_DSV_AllocatedValue = $object.expectedGET_DSV_AllocatedValue_win - $expectedSet_DSV_AllocatedValue = $object.expectedSET_DSV_AllocatedValue_win - $expectedMGet_DSV_AllocatedValue = $object.expectedMGET_DSV_AllocatedValue_win - $expectedMSet_DSV_AllocatedValue = $object.expectedMSET_DSV_AllocatedValue_win - $expectedCPBSET_DSV_AllocatedValue = $object.expectedCPBSET_DSV_AllocatedValue_win - - $expectedZAddRem_None_AllocatedValue = $object.expectedZAddRem_None_AllocatedValue_win - $expectedLPushPop_None_AllocatedValue = $object.expectedLPushPop_None_AllocatedValue_win - $expectedSAddRem_None_AllocatedValue = $object.expectedSAddRem_None_AllocatedValue_win - $expectedHSetDel_None_AllocatedValue = $object.expectedHSetDel_None_AllocatedValue_win - $expectedMyDictSetGet_None_AllocatedValue = $object.expectedMyDictSetGet_None_AllocatedValue_win - $expectedZAddRem_ACL_AllocatedValue = $object.expectedZAddRem_ACL_AllocatedValue_win - $expectedLPushPop_ACL_AllocatedValue = $object.expectedLPushPop_ACL_AllocatedValue_win - $expectedSAddRem_ACL_AllocatedValue = $object.expectedSAddRem_ACL_AllocatedValue_win - $expectedHSetDel_ACL_AllocatedValue = $object.expectedHSetDel_ACL_AllocatedValue_win - $expectedMyDictSetGet_ACL_AllocatedValue = $object.expectedMyDictSetGet_ACL_AllocatedValue_win - $expectedZAddRem_AOF_AllocatedValue = $object.expectedZAddRem_AOF_AllocatedValue_win - $expectedLPushPop_AOF_AllocatedValue = $object.expectedLPushPop_AOF_AllocatedValue_win - $expectedSAddRem_AOF_AllocatedValue = $object.expectedSAddRem_AOF_AllocatedValue_win - $expectedHSetDel_AOF_AllocatedValue = $object.expectedHSetDel_AOF_AllocatedValue_win - $expectedMyDictSetGet_AOF_AllocatedValue = $object.expectedMyDictSetGet_AOF_AllocatedValue_win - - $expectedScript1_None_AllocatedValue = $object.expectedScript1_None_AllocatedValue_win - $expectedScript2_None_AllocatedValue = $object.expectedScript2_None_AllocatedValue_win - $expectedScript3_None_AllocatedValue = $object.expectedScript3_None_AllocatedValue_win - $expectedScript4_None_AllocatedValue = $object.expectedScript4_None_AllocatedValue_win - - $expectedInlinePing_ACL_AllocatedValue = $object.expectedInlinePing_ACL_AllocatedValue_win - $expectedInlinePing_AOF_AllocatedValue = $object.expectedInlinePing_AOF_AllocatedValue_win - $expectedInlinePing_None_AllocatedValue = $object.expectedInlinePing_None_AllocatedValue_win +# Expected values for Allocated amount +switch -Wildcard ($configFile) { + "CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json" { + $expectedGet_None = $object.expectedGET_None + $expectedSet_None = $object.expectedSET_None + $expectedMGet_None = $object.expectedMGET_None + $expectedMSet_None = $object.expectedMSET_None + } + "CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json" { + $expectedGet_DSV = $object.expectedGET_DSV + $expectedSet_DSV = $object.expectedSET_DSV + $expectedMGet_DSV = $object.expectedMGET_DSV + $expectedMSet_DSV = $object.expectedMSET_DSV + $expectedCPBSET_DSV = $object.expectedCPBSET_DSV + + $expectedGet_None = $object.expectedGET_None + $expectedSet_None = $object.expectedSET_None + $expectedMGet_None = $object.expectedMGET_None + $expectedMSet_None = $object.expectedMSET_None + $expectedCPBSET_None = $object.expectedCPBSET_None + } + "CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json" { + $expectedScript1_None = $object.expectedScript1_None + $expectedScript2_None = $object.expectedScript2_None + $expectedScript3_None = $object.expectedScript3_None + $expectedScript4_None = $object.expectedScript4_None + } + "CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json" { + $expectedInlinePing_ACL = $object.expectedInlinePing_ACL + $expectedInlinePing_AOF = $object.expectedInlinePing_AOF + $expectedInlinePing_None = $object.expectedInlinePing_None + } + "CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json" { + $expectedZAddRem_None = $object.expectedZAddRem_None + $expectedLPushPop_None = $object.expectedLPushPop_None + $expectedSAddRem_None = $object.expectedSAddRem_None + $expectedHSetDel_None = $object.expectedHSetDel_None + $expectedMyDictSetGet_None = $object.expectedMyDictSetGet_None + $expectedCustomProcSet_None = $object.expectedCustomProcSet_None + + $expectedZAddRem_ACL = $object.expectedZAddRem_ACL + $expectedLPushPop_ACL = $object.expectedLPushPop_ACL + $expectedSAddRem_ACL = $object.expectedSAddRem_ACL + $expectedHSetDel_ACL = $object.expectedHSetDel_ACL + $expectedMyDictSetGet_ACL = $object.expectedMyDictSetGet_ACL + $expectedCustomProcSet_ACL = $object.expectedCustomProcSet_ACL + + $expectedZAddRem_AOF = $object.expectedZAddRem_AOF + $expectedLPushPop_AOF = $object.expectedLPushPop_AOF + $expectedSAddRem_AOF = $object.expectedSAddRem_AOF + $expectedHSetDel_AOF = $object.expectedHSetDel_AOF + $expectedMyDictSetGet_AOF = $object.expectedMyDictSetGet_AOF + $expectedCustomProcSet_AOF = $object.expectedCustomProcSet_AOF + } } -# percent allowed variance when comparing expected vs actual found value - same for linux and windows. -$acceptableAllocatedRange = $object.acceptableAllocatedRange - # Set up the results dir and errorlog dir $resultsDir = "$basePath/test/BDNPerfTests/results" if (-not (Test-Path -Path $resultsDir)) { @@ -274,40 +248,40 @@ Get-Content $resultsFile | ForEach-Object { switch -Wildcard ($line) { "*| Set*$paramNone*" { Write-Host "** Set ($paramNone) Allocated Value test" - $foundSet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSet_None_AllocatedValue $expectedSet_None_AllocatedValue $acceptableAllocatedRange $true + $foundSet_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSet_None $expectedSet_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| Get*$paramNone*" { Write-Host "** Get ($paramNone) Allocated Value test" - $foundGet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundGet_None_AllocatedValue $expectedGet_None_AllocatedValue $acceptableAllocatedRange $true + $foundGet_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundGet_None $expectedGet_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MSet*$paramNone*" { Write-Host "** MSet ($paramNone) Allocated Value test" - $foundMSet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMSet_None_AllocatedValue $expectedMSet_None_AllocatedValue $acceptableAllocatedRange $true + $foundMSet_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMSet_None $expectedMSet_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MGet*$paramNone*" { Write-Host "** MGet ($paramNone) Allocated Value test" - $foundMGet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMGet_None_AllocatedValue $expectedMGet_None_AllocatedValue $acceptableAllocatedRange $true + $foundMGet_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMGet_None $expectedMGet_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| CPBSET*$paramNone*" { Write-Host "** CPBSET ($paramNone) Allocated Value test" - $foundCPBSET_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCPBSET_None_AllocatedValue $expectedCPBSET_None_AllocatedValue $acceptableAllocatedRange $true + $foundCPBSET_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCPBSET_None $expectedCPBSET_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } @@ -315,40 +289,40 @@ Get-Content $resultsFile | ForEach-Object { "*| Set*$paramDSV*" { Write-Host "** Set ($paramDSV) Allocated Value test" - $foundSet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSet_DSV_AllocatedValue $expectedSet_DSV_AllocatedValue $acceptableAllocatedRange $true + $foundSet_DSV = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSet_DSV $expectedSet_DSV $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| Get*$paramDSV*" { Write-Host "** Get ($paramDSV) Allocated Value test" - $foundGet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundGet_DSV_AllocatedValue $expectedGet_DSV_AllocatedValue $acceptableAllocatedRange $true + $foundGet_DSV = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundGet_DSV $expectedGet_DSV $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MSet*$paramDSV*" { Write-Host "** MSet ($paramDSV) Allocated Value test" - $foundMSet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMSet_DSV_AllocatedValue $expectedMSet_DSV_AllocatedValue $acceptableAllocatedRange $true + $foundMSet_DSV = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMSet_DSV $expectedMSet_DSV $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MGet*$paramDSV*" { Write-Host "** MGet ($paramDSV) Allocated Value test" - $foundMGet_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMGet_DSV_AllocatedValue $expectedMGet_DSV_AllocatedValue $acceptableAllocatedRange $true + $foundMGet_DSV = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMGet_DSV $expectedMGet_DSV $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| CPBSET*$paramDSV*" { Write-Host "** CPBSET ($paramDSV) Allocated Value test" - $foundCPBSET_DSV_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCPBSET_DSV_AllocatedValue $expectedCPBSET_DSV_AllocatedValue $acceptableAllocatedRange $true + $foundCPBSET_DSV = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCPBSET_DSV $expectedCPBSET_DSV $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } @@ -356,96 +330,96 @@ Get-Content $resultsFile | ForEach-Object { "*| ZAddRem*$paramNone*" { Write-Host "** ZAddRem ($paramNone) Allocated Value test" - $foundZAddRem_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRem_None_AllocatedValue $expectedZAddRem_None_AllocatedValue $acceptableAllocatedRange $true + $foundZAddRem_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRem_None $expectedZAddRem_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| LPushPop*$paramNone*" { Write-Host "** LPushPop ($paramNone) Allocated Value test" - $foundLPushPop_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPop_None_AllocatedValue $expectedLPushPop_None_AllocatedValue $acceptableAllocatedRange $true + $foundLPushPop_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPop_None $expectedLPushPop_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| SAddRem*$paramNone*" { Write-Host "** SAddRem ($paramNone) Allocated Value test" - $foundSAddRem_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRem_None_AllocatedValue $expectedSAddRem_None_AllocatedValue $acceptableAllocatedRange $true + $foundSAddRem_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRem_None $expectedSAddRem_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| HSetDel*$paramNone*" { Write-Host "** HSetDel ($paramNone) Allocated Value test" - $foundHSetDel_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDel_None_AllocatedValue $expectedHSetDel_None_AllocatedValue $acceptableAllocatedRange $true + $foundHSetDel_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDel_None $expectedHSetDel_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MyDictSetGet*$paramNone*" { Write-Host "** MyDictSetGet ($paramNone) Allocated Value test" - $foundMyDictSetGet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGet_None_AllocatedValue $expectedMyDictSetGet_None_AllocatedValue $acceptableAllocatedRange $true + $foundMyDictSetGet_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGet_None $expectedMyDictSetGet_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| CustomProcSet*$paramNone*" { Write-Host "** MyDictSetGet ($paramNone) Allocated Value test" - $foundCustomProcSet_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCustomProcSet_None_AllocatedValue $expectedCustomProcSet_None_AllocatedValue $acceptableAllocatedRange $true + $foundCustomProcSet_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCustomProcSet_None $expectedCustomProcSet_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| ZAddRem*$paramACL*" { Write-Host "** ZAddRem ($paramACL) Allocated Value test" - $foundZAddRem_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRem_ACL_AllocatedValue $expectedZAddRem_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundZAddRem_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRem_ACL $expectedZAddRem_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| LPushPop*$paramACL*" { Write-Host "** LPushPop ($paramACL) Allocated Value test" - $foundLPushPop_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPop_ACL_AllocatedValue $expectedLPushPop_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundLPushPop_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPop_ACL $expectedLPushPop_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| SAddRem*$paramACL*" { Write-Host "** SAddRem ($paramACL) Allocated Value test" - $foundSAddRem_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRem_ACL_AllocatedValue $expectedSAddRem_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundSAddRem_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRem_ACL $expectedSAddRem_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| HSetDel*$paramACL*" { Write-Host "** HSetDel ($paramACL) Allocated Value test" - $foundHSetDel_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDel_ACL_AllocatedValue $expectedHSetDel_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundHSetDel_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDel_ACL $expectedHSetDel_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MyDictSetGet*$paramACL*" { Write-Host "** MyDictSetGet ($paramACL) Allocated Value test" - $foundMyDictSetGet_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGet_ACL_AllocatedValue $expectedMyDictSetGet_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundMyDictSetGet_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGet_ACL $expectedMyDictSetGet_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| CustomProcSet*$paramACL*" { Write-Host "** MyDictSetGet ($paramACL) Allocated Value test" - $foundCustomProcSet_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCustomProcSet_ACL_AllocatedValue $expectedCustomProcSet_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundCustomProcSet_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundCustomProcSet_ACL $expectedCustomProcSet_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } @@ -453,48 +427,48 @@ Get-Content $resultsFile | ForEach-Object { "*| ZAddRem*$paramAOF*" { Write-Host "** ZAddRem ($paramAOF) Allocated Value test" - $foundZAddRem_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRem_AOF_AllocatedValue $expectedZAddRem_AOF_AllocatedValue $acceptableAllocatedRange $true + $foundZAddRem_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundZAddRem_AOF $expectedZAddRem_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| LPushPop*$paramAOF*" { Write-Host "** LPushPop ($paramAOF) Allocated Value test" - $foundLPushPop_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPop_AOF_AllocatedValue $expectedLPushPop_AOF_AllocatedValue $acceptableAllocatedRange $true + $foundLPushPop_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundLPushPop_AOF $expectedLPushPop_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| SAddRem*$paramAOF*" { Write-Host "** SAddRem ($paramAOF) Allocated Value test" - $foundSAddRem_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRem_AOF_AllocatedValue $expectedSAddRem_AOF_AllocatedValue $acceptableAllocatedRange $true + $foundSAddRem_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundSAddRem_AOF $expectedSAddRem_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| HSetDel*$paramAOF*" { Write-Host "** HSetDel ($paramAOF) Allocated Value test" - $foundHSetDel_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDel_AOF_AllocatedValue $expectedHSetDel_AOF_AllocatedValue $acceptableAllocatedRange $true + $foundHSetDel_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundHSetDel_AOF $expectedHSetDel_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| MyDictSetGet*$paramAOF*" { Write-Host "** MyDictSetGet ($paramAOF) Allocated Value test" - $foundMyDictSetGet_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGet_AOF_AllocatedValue $expectedMyDictSetGet_AOF_AllocatedValue $acceptableAllocatedRange $true + $foundMyDictSetGet_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyDictSetGet_AOF $expectedMyDictSetGet_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| CustomProcSet*$paramAOF*" { - Write-Host "** MyDictSetGet ($paramAOF) Allocated Value test" - $foundMyCustomProcSet_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyCustomProcSet_AOF_AllocatedValue $expectedCustomProcSetSetGet_AOF_AllocatedValue $acceptableAllocatedRange $true + Write-Host "** CustomProcSet ($paramAOF) Allocated Value test" + $foundMyCustomProcSet_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundMyCustomProcSet_AOF $expectedCustomProcSet_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } @@ -502,32 +476,32 @@ Get-Content $resultsFile | ForEach-Object { "*| Script1*$paramNone*" { Write-Host "** Lua Script1 ($paramNone) Allocated Value test" - $foundScript1_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript1_None_AllocatedValue $expectedScript1_None_AllocatedValue $acceptableAllocatedRange $true + $foundScript1_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript1_None $expectedScript1_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| Script2*$paramNone*" { Write-Host "** Lua Script2 ($paramNone) Allocated Value test" - $foundScript2_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript2_None_AllocatedValue $expectedScript2_None_AllocatedValue $acceptableAllocatedRange $true + $foundScript2_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript2_None $expectedScript2_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| Script3*$paramNone*" { Write-Host "** Lua Script3 ($paramNone) Allocated Value test" - $foundScript3_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript3_None_AllocatedValue $expectedScript3_None_AllocatedValue $acceptableAllocatedRange $true + $foundScript3_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript3_None $expectedScript3_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| Script4*$paramNone*" { Write-Host "** LuaScript4 ($paramNone) Allocated Value test" - $foundScript4_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript4_None_AllocatedValue $expectedScript4_None_AllocatedValue $acceptableAllocatedRange $true + $foundScript4_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundScript4_None $expectedScript4_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } @@ -536,8 +510,8 @@ Get-Content $resultsFile | ForEach-Object { "*| InlinePing*$paramNone*" { Write-Host "** InlinePing ($paramNone) Allocated Value test" - $foundInlinePing_None_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundInlinePing_None_AllocatedValue $expectedInlinePing_None_AllocatedValue $acceptableAllocatedRange $true + $foundInlinePing_None = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundInlinePing_None $expectedInlinePing_None $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } @@ -545,22 +519,20 @@ Get-Content $resultsFile | ForEach-Object { "*| InlinePing*$paramACL*" { Write-Host "** InlinePing ($paramACL) Allocated Value test" - $foundInlinePing_ACL_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundInlinePing_ACL_AllocatedValue $expectedInlinePing_ACL_AllocatedValue $acceptableAllocatedRange $true + $foundInlinePing_ACL = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundInlinePing_ACL $expectedInlinePing_ACL $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } "*| InlinePing*$paramAOF*" { Write-Host "** InlinePing ($paramAOF) Allocated Value test" - $foundInlinePing_AOF_AllocatedValue = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundInlinePing_AOF_AllocatedValue $expectedInlinePing_AOF_AllocatedValue $acceptableAllocatedRange $true + $foundInlinePing_AOF = ParseValueFromResults $line $allocatedColumn + $currentResults = AnalyzeResult $foundInlinePing_AOF $expectedInlinePing_AOF $acceptableAllocatedRange $true if ($currentResults -eq $false) { $testSuiteResult = $false } } - - } } From ef9d9f099db0c3fac5632414f8d660d02a04a654 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 2 Dec 2024 14:10:06 -0800 Subject: [PATCH 40/44] Big update to where moved to a single config file and the ps1 file does not need to be updated when new BDNs are added. --- .github/workflows/ci-bdnbenchmark.yml | 5 +- test/BDNPerfTests/BDN_Benchmark_Config.json | 48 ++ ..._BDN_Benchmark_Cluster.ClusterMigrate.json | 9 - ...N_Benchmark_Cluster.ClusterOperations.json | 15 - ...I_CONFIG_BDN_Benchmark_Lua.LuaScripts.json | 9 - ..._Benchmark_Operations.BasicOperations.json | 8 - ...Benchmark_Operations.ObjectOperations.json | 27 - test/BDNPerfTests/run_bdnperftest.ps1 | 513 ++++-------------- 8 files changed, 156 insertions(+), 478 deletions(-) create mode 100644 test/BDNPerfTests/BDN_Benchmark_Config.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json delete mode 100644 test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index bb0a6e5eb8..a5e5405ff7 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -42,7 +42,7 @@ jobs: os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0' ] configuration: [ 'Release' ] - test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts' ] + test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts' ] steps: - name: Check out code uses: actions/checkout@v4 @@ -90,7 +90,7 @@ jobs: benchmark-data-dir-path: 'website/static/charts' auto-push: true - # Run `github-action-benchmark` action for the Continuous Benchmarking Charts + # Run `github-action-benchmark` action for the Continuous Benchmarking Charts (https://microsoft.github.io/garnet/charts/) - name: Store benchmark result for charts uses: benchmark-action/github-action-benchmark@v1 with: @@ -103,3 +103,4 @@ jobs: benchmark-data-dir-path: 'website/static/charts' auto-push: true max-items-in-chart: 50 + \ No newline at end of file diff --git a/test/BDNPerfTests/BDN_Benchmark_Config.json b/test/BDNPerfTests/BDN_Benchmark_Config.json new file mode 100644 index 0000000000..146de35650 --- /dev/null +++ b/test/BDNPerfTests/BDN_Benchmark_Config.json @@ -0,0 +1,48 @@ +{ + "_comment": "NOTE: If adding a new BDN perf test to this file, you will need to add the name to the test: line in ce-bdnbenchmark.yml (~line 45). This ensures all the tests are not ran sequentially when ran as github action.", + "_comment2": "NOTE: The BDN Perf name in the yml test: line should NOT have the BDN.benchmark prefix and the .* ending. It needs to be this way for the reporting part of the BDN Charts.", + + "BDN.benchmark.Operations.BasicOperations.*": { + "expected_InlinePing_ACL": 0, + "expected_InlinePing_AOF": 0, + "expected_InlinePing_None": 0 + }, + "BDN.benchmark.Operations.ObjectOperations.*": { + "expected_ZAddRem_ACL": 17.97, + "expected_LPushPop_ACL": 14.06, + "expected_SAddRem_ACL": 12.5, + "expected_HSetDel_ACL": 42.19, + "expected_ZAddRem_AOF": 17.97, + "expected_LPushPop_AOF": 14.06, + "expected_SAddRem_AOF": 12.5, + "expected_HSetDel_AOF": 42.19, + "expected_ZAddRem_None": 17.97, + "expected_LPushPop_None": 14.06, + "expected_SAddRem_None": 12.5, + "expected_HSetDel_None": 42.19 + }, + "BDN.benchmark.Cluster.ClusterOperations.*": { + "expected_Get_DSV": 0, + "expected_Set_DSV": 0, + "expected_MGet_DSV": 0, + "expected_MSet_DSV": 0, + "expected_CPBSet_DSV": 0, + "expected_Get_None": 0, + "expected_Set_None": 0, + "expected_MGet_None": 0, + "expected_MSet_None": 0, + "expected_CPBSet_None": 0 + }, + "BDN.benchmark.Cluster.ClusterMigrate.*": { + "expected_Get_None": 0, + "expected_Set_None": 0, + "expected_MGet_None": 0, + "expected_MSet_None": 0 + }, + "BDN.benchmark.Lua.LuaScripts.*": { + "expected_Script1_None": 24, + "expected_Script2_None": 144, + "expected_Script3_None": 240, + "expected_Script4_None": 776 + } +} \ No newline at end of file diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json deleted file mode 100644 index 4e4b10edc2..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "filter": "BDN.benchmark.Cluster.ClusterMigrate.*", - - "_comment": "All these values are the expected Allocated.", - "expectedGet_None": 0, - "expectedSet_None": 0, - "expectedMGet_None": 0, - "expectedMSet_None": 0 -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json deleted file mode 100644 index 2676f419ba..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "filter": "BDN.benchmark.Cluster.ClusterOperations.*", - - "_comment": "All these values are the expected Allocated.", - "expectedGet_DSV": 0, - "expectedSet_DSV": 0, - "expectedMGet_DSV": 0, - "expectedMSet_DSV": 0, - "expectedCPBSet_DSV": 0, - "expectedGet_None": 0, - "expectedSet_None": 0, - "expectedMGet_None": 0, - "expectedMSet_None": 0, - "expectedCPBSet_None": 0 -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json deleted file mode 100644 index ab29c9aea7..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "filter": "BDN.benchmark.Lua.*", - - "_comment": "All these values are the expected Allocated.", - "expectedScript1_None": 24, - "expectedScript2_None": 144, - "expectedScript3_None": 240, - "expectedScript4_None": 776 -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json deleted file mode 100644 index 1f699f47a8..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "filter": "BDN.benchmark.Operations.BasicOperations.*", - - "_comment": "All these values are the expected Allocated.", - "expectedInlinePing_ACL_Allocated": 0, - "expectedInlinePing_AOF_Allocated": 0, - "expectedInlinePing_None_Allocated": 0 -} diff --git a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json b/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json deleted file mode 100644 index 7227473db5..0000000000 --- a/test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "filter": "BDN.benchmark.Operations.ObjectOperations.*", - - "_comment": "All these values are the expected Allocated.", - "expectedZAddRem_ACL": 18400, - "expectedLPushPop_ACL": 14400, - "expectedSAddRem_ACL": 12800, - "expectedHSetDel_ACL": 43201, - "expectedMyDictSetGet_ACL": 24000, - "expectedCustomProcSet_ACL": 0, - - "expectedZAddRem_AOF": 18400, - "expectedLPushPop_AOF": 14400, - "expectedSAddRem_AOF": 12800, - "expectedHSetDel_AOF": 43201, - "expectedMyDictSetGet_AOF": 24000, - "expectedCustomProcSet_AOF": 0, - - "expectedZAddRem_None": 18400, - "expectedLPushPop_None": 14400, - "expectedSAddRem_None": 12800, - "expectedHSetDel_None": 43201, - "expectedMyDictSetGet_None": 24000, - "expectedCustomProcSet_None": 0 - -} - diff --git a/test/BDNPerfTests/run_bdnperftest.ps1 b/test/BDNPerfTests/run_bdnperftest.ps1 index 54534fa629..82a1e075a5 100644 --- a/test/BDNPerfTests/run_bdnperftest.ps1 +++ b/test/BDNPerfTests/run_bdnperftest.ps1 @@ -1,26 +1,24 @@ <#$f .SYNOPSIS - This script is designed for to run BDN Benchmarking and use the results as a gate in the GitHub CIs to ensure performance isn't declining + This script is designed for to run BDN Benchmarking and use the results as a gate in the GitHub CIs to ensure performance isn't declining. .DESCRIPTION - Script to test for performance regressions in Allocated Memory using BDN Benchmark tool. There are configuration files (in /ConfigFiles dir) associated with each test that contains name and expected values of the BDN benchmark. - Any of these can be sent as the parameter to the file. - - CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json + Script to test for performance regressions in Allocated Memory using BDN Benchmark tool. The various tests are in the configuration file (BDN_Benchmark_Config.json) and are associated with each test that contains name and expected values of the BDN benchmark. + Any of the BDN benchmark tests (ie BDN.benchmark.Operations.BasicOperations.*) can be sent as the parameter to the file. NOTE: The expected values are specific for the CI Machine. If you run these on your machine, you will need to change the expected values. - NOTE: The acceptablerange* parameters in the config file is how far +/- X% the found value can be from the expected value and still say it is pass. Defaulted to 10% + + NOTE: If adding a new BDN perf test to the BDN_Benchmark_Config.json, then need to add to the "test: [..." line in the ce-bdnbenchmark.yml .EXAMPLE - ./run_bdnperftest.ps1 - ./run_bdnperftest.ps1 CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json + ./run_bdnperftest.ps1 + ./run_bdnperftest.ps1 BDN.benchmark.Operations.BasicOperations.* + ./run_bdnperftest.ps1 Operations.BasicOperations <-- can run this way but this is how specify in ci-bdnbenchmark.yml #> - -# Send the config file for the benchmark. Defaults to a simple one param ( - [string]$configFile = "CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json" + [string]$currentTest = "BDN.benchmark.Operations.BasicOperations.*" ) $OFS = "`r`n" @@ -29,93 +27,96 @@ $OFS = "`r`n" ################## AnalyzeResult ##################### # -# Takes the result and verifies it falls in the acceptable range (tolerance) based on the percentage. +# Takes the result and verifies it falls in the acceptable range (tolerance) based on the percentage. # ###################################################### function AnalyzeResult { param ($foundResultValue, $expectedResultValue, $acceptablePercentRange, $warnonly) # Calculate the upper bounds of the expected value - [double] $Tolerance = $acceptablePercentRange / 100 - [double] $UpperBound = $expectedResultValue * (1 + $Tolerance) - [double] $dblfoundResultValue = $foundResultValue + $Tolerance = [double]$acceptablePercentRange / 100 + $UpperBound = [double]$expectedResultValue * (1 + $Tolerance) + $dblfoundResultValue = [double]$foundResultValue # Check if the actual value is within the bounds if ($dblfoundResultValue -le $UpperBound) { - Write-Host "** ** PASS! ** The Allocated Value result ($dblfoundResultValue) is under the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" + Write-Host "** ** PASS! ** The found Allocated value ($dblfoundResultValue) is below the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " return $true # the values are close enough } else { if ($warnonly) { - Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" + Write-Host "** << PERF REGRESSION WARNING! >> The BDN benchmark found Allocated value ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " } else { - Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark Allocated Value result ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" + Write-Host "** << PERF REGRESSION FAIL! >> The BDN benchmark found Allocated value ($dblfoundResultValue) is above the acceptable threshold of $UpperBound (Expected value $expectedResultValue + $acceptablePercentRange%)" Write-Host "** " } return $false # the values are too different } } + ######### ParseValueFromResults ########### -# +# # Takes the line from the results file and returns the value from the requested column # strips off all the characters and just to return the actual value # -# NOTE: Example of ResultsLine from BDN benchmark: "| InlinePing | 2.343 us | 0.0135 us | 0.0113 us | - |" -# NOTE: columnNum is zero based +# NOTE: Example of ResultsLine from BDN benchmark: "| ZAddRem | ACL | 96.05 us | 1.018 us | 0.952 us | 17.97 KB |" # ###################################################### function ParseValueFromResults { param ($ResultsLine, $columnNum) # Remove the leading and trailing pipes and split the string by '|' - $columns = $ResultsLine.Trim('|').Split('|') + $columns = $ResultsLine.Trim('|').Split('|') $column = $columns | ForEach-Object { $_.Trim() } - $foundValue = $column[$columnNum] + $foundValue = $column[$columnNum] if ($foundValue -eq "-") { $foundValue = "0" } - $foundValue = $foundValue.Trim(' B') - $foundValue = $foundValue.Trim(' K') - $foundValue = $foundValue.Trim(' m') + $foundValue = $foundValue.Trim(' B') + $foundValue = $foundValue.Trim(' K') + $foundValue = $foundValue.Trim(' m') return $foundValue } # ******** BEGIN MAIN ********* +# Set all the config options +$configFile = "BDN_Benchmark_Config.json" +$configuration = "Release" +$framework = "net8.0" +$allocatedColumn = "-1" # last one is allocated, just to ensure in case other column gets added +$acceptableAllocatedRange = "10" # percent allowed variance when comparing expected vs actual found value - same for linux and windows. + +# For Actions the test in the yml does not get specified with .* because it used for file names there, so add .* here if not already on there. +if ($currentTest -notmatch "\.\*$") { + $currentTest += ".*" +} +# Actions also don't have BDN on beginning due to what files are saved +if ($currentTest -notlike "BDN.benchmark.*") { + $currentTest = "BDN.benchmark." + $currentTest +} + # Get base path since paths can differ from machine to machine $pathstring = $pwd.Path if ($pathstring.Contains("test")) { $position = $pathString.IndexOf("test") $basePath = $pathstring.Substring(0,$position-1) # take off slash off end as well } else { - $basePath = $pathstring # already in base as not in test + $basePath = $pathstring # already in base and not in test Set-Location .\test\BDNPerfTests\ } -# Add .json to end if not already there -if ($configFile -notlike "*.json") { - $configFile += ".json" -} - -# This is special case that allows passing test without specifying CI_CONFIG_BDN_ at the beginning - need for perf test -if ($configFile -notlike "CI_CONFIG_BDN_Benchmark_*") { - $configFile = "CI_CONFIG_BDN_Benchmark_" + $configFile -} - # Read the test config file and convert the JSON to a PowerShell object -$fullConfiFileAndPath = "ConfigFiles/$configFile" -if (-not (Test-Path -Path $fullConfiFileAndPath)) { - Write-Error -Message "The test config file $fullConfiFileAndPath does not exist." -Category ObjectNotFound +if (-not (Test-Path -Path $configFile)) { + Write-Error -Message "The test config file $configFile does not exist." -Category ObjectNotFound exit } -$json = Get-Content -Raw $fullConfiFileAndPath -$object = $json | ConvertFrom-Json # Use this in the file name to separate outputs when running in ADO $CurrentOS = "Windows" @@ -123,96 +124,59 @@ if ($IsLinux) { $CurrentOS = "Linux" } -Write-Host "************** Start BDN.benchmark Test Run ********************" +Write-Host "************** Start BDN.benchmark Test Run ********************" +Write-Host " " +Write-Host "** Current Test: $currentTest" Write-Host " " -# Set all the config options (args to benchmark app) -$configuration = "Release" -$framework = "net8.0" -$allocatedColumn = "-1" # last one is allocated, just to ensure in case other column gets added -$paramNone = "None" -$paramDSV = "DSV" -$paramACL = "ACL" -$paramAOF = "AOF" -$acceptableAllocatedRange = "10" # percent allowed variance when comparing expected vs actual found value - same for linux and windows. -$filter = $object.filter - -# Expected values for Allocated amount -switch -Wildcard ($configFile) { - "CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json" { - $expectedGet_None = $object.expectedGET_None - $expectedSet_None = $object.expectedSET_None - $expectedMGet_None = $object.expectedMGET_None - $expectedMSet_None = $object.expectedMSET_None - } - "CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json" { - $expectedGet_DSV = $object.expectedGET_DSV - $expectedSet_DSV = $object.expectedSET_DSV - $expectedMGet_DSV = $object.expectedMGET_DSV - $expectedMSet_DSV = $object.expectedMSET_DSV - $expectedCPBSET_DSV = $object.expectedCPBSET_DSV - - $expectedGet_None = $object.expectedGET_None - $expectedSet_None = $object.expectedSET_None - $expectedMGet_None = $object.expectedMGET_None - $expectedMSet_None = $object.expectedMSET_None - $expectedCPBSET_None = $object.expectedCPBSET_None - } - "CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json" { - $expectedScript1_None = $object.expectedScript1_None - $expectedScript2_None = $object.expectedScript2_None - $expectedScript3_None = $object.expectedScript3_None - $expectedScript4_None = $object.expectedScript4_None - } - "CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json" { - $expectedInlinePing_ACL = $object.expectedInlinePing_ACL - $expectedInlinePing_AOF = $object.expectedInlinePing_AOF - $expectedInlinePing_None = $object.expectedInlinePing_None - } - "CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json" { - $expectedZAddRem_None = $object.expectedZAddRem_None - $expectedLPushPop_None = $object.expectedLPushPop_None - $expectedSAddRem_None = $object.expectedSAddRem_None - $expectedHSetDel_None = $object.expectedHSetDel_None - $expectedMyDictSetGet_None = $object.expectedMyDictSetGet_None - $expectedCustomProcSet_None = $object.expectedCustomProcSet_None - - $expectedZAddRem_ACL = $object.expectedZAddRem_ACL - $expectedLPushPop_ACL = $object.expectedLPushPop_ACL - $expectedSAddRem_ACL = $object.expectedSAddRem_ACL - $expectedHSetDel_ACL = $object.expectedHSetDel_ACL - $expectedMyDictSetGet_ACL = $object.expectedMyDictSetGet_ACL - $expectedCustomProcSet_ACL = $object.expectedCustomProcSet_ACL - - $expectedZAddRem_AOF = $object.expectedZAddRem_AOF - $expectedLPushPop_AOF = $object.expectedLPushPop_AOF - $expectedSAddRem_AOF = $object.expectedSAddRem_AOF - $expectedHSetDel_AOF = $object.expectedHSetDel_AOF - $expectedMyDictSetGet_AOF = $object.expectedMyDictSetGet_AOF - $expectedCustomProcSet_AOF = $object.expectedCustomProcSet_AOF - } +# Access the properties under the specific test in the config file +$json = Get-Content -Raw -Path "$configFile" | ConvertFrom-Json +$testProperties = $json.$currentTest + +# create a matrix of expected results for specific test +$splitTextArray = New-Object 'string[]' 3 +$expectedResultsArray = New-Object 'string[,]' 20, 3 + +[int]$currentRow = 0 + +# Iterate through each property for the expected value - getting method, params and value +foreach ($property in $testProperties.PSObject.Properties) { + + # Split the property name into parts + $splitTextArray = $property.Name -split '_' + + # get the expected Method, Parameter and value + $expectedResultsArray[$currentRow,0] = "| " + $splitTextArray[1] # Add | at beginning so it is part of summary block and not every other line in the file that matches the Method name + $expectedResultsArray[$currentRow,1] = $splitTextArray[2] + $expectedResultsArray[$currentRow,2] = $property.value + + $currentRow += 1 } +$totalExpectedResultValues = $currentRow + # Set up the results dir and errorlog dir -$resultsDir = "$basePath/test/BDNPerfTests/results" +$resultsDir = "$basePath/test/BDNPerfTests/results" if (-not (Test-Path -Path $resultsDir)) { New-Item -Path $resultsDir -ItemType Directory } -$errorLogDir = "$basePath/test/BDNPerfTests/errorlog" +$errorLogDir = "$basePath/test/BDNPerfTests/errorlog" if (-not (Test-Path -Path $errorLogDir)) { New-Item -Path $errorLogDir -ItemType Directory } # Run the BDN.benchmark -$BDNbenchmarkPath = "$basePath/benchmark/BDN.benchmark" +$BDNbenchmarkPath = "$basePath/benchmark/BDN.benchmark" -# Create Results and all the log files using the the config file name as part of the name of the results \ logs -$justResultsFileNameNoExt = $configFile -replace ".{5}$" # strip off the .json -$resultsFileName = $justResultsFileNameNoExt + "_" + $CurrentOS + ".results" +# Create Results and all the log files using the the config file name as part of the name of the results \ logs - strip off the prefix and postfix +$prefix = "BDN.benchmark." +$justResultsFileNameNoExt = $currentTest -replace ".{2}$" # strip off the .* for log files +$currentTestStripped = $justResultsFileNameNoExt -replace [regex]::Escape($prefix), "" +$resultsFileName = $currentTestStripped + "_" + $CurrentOS + ".results" $resultsFile = "$resultsDir/$resultsFileName" -$BDNbenchmarkErrorFile = "$errorLogDir/$justResultsFileNameNoExt" + "_StandardError_" +$CurrentOS+".log" +$BDNbenchmarkErrorFile = "$errorLogDir/$currentTestStripped" + "_StandardError_" +$CurrentOS+".log" +$filter = $currentTest -Write-Output "** Start BDN Benchmark: $filter" Write-Output " " Write-Output "** Start: dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath --exporters json > $resultsFile 2> $BDNbenchmarkErrorFile" dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchmarkPath --exporters json > $resultsFile 2> $BDNbenchmarkErrorFile @@ -220,9 +184,9 @@ dotnet run -c $configuration -f $framework --filter $filter --project $BDNbenchm Write-Output "** BDN Benchmark for $filter finished" Write-Output " " -Write-Output "**** EVALUATE THE RESULTS FILE $resultsFile ****" +Write-Output "**** ANALYZE THE RESULTS FILE $resultsFile ****" -# First check if file is there and if not, error out gracefully +# First check if results file is there and if not, error out gracefully if (-not (Test-Path -Path $resultsFile)) { Write-Error -Message "The test results file $resultsFile does not exist. Check to make sure the test was ran." -Category ObjectNotFound exit @@ -235,315 +199,48 @@ if ($resultsFileSizeBytes -eq 0) { exit } +Write-Output " " Write-Output "************************" -Write-Output "** RESULTS " -Write-Output "** " +Write-Output "** RESULTS for test: $currentTest " +Write-Output "**" # Set the test suite to pass and if any one fails, then mark the suite as fail - just one result failure will mark the whole test as failed $testSuiteResult = $true -# Read the results file line by line and pull out the specific results if exists +# Go through the results and verify the summary found results value vs the expected results values Get-Content $resultsFile | ForEach-Object { $line = $_ - switch -Wildcard ($line) { - "*| Set*$paramNone*" { - Write-Host "** Set ($paramNone) Allocated Value test" - $foundSet_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSet_None $expectedSet_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| Get*$paramNone*" { - Write-Host "** Get ($paramNone) Allocated Value test" - $foundGet_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundGet_None $expectedGet_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MSet*$paramNone*" { - Write-Host "** MSet ($paramNone) Allocated Value test" - $foundMSet_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMSet_None $expectedMSet_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MGet*$paramNone*" { - Write-Host "** MGet ($paramNone) Allocated Value test" - $foundMGet_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMGet_None $expectedMGet_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| CPBSET*$paramNone*" { - Write-Host "** CPBSET ($paramNone) Allocated Value test" - $foundCPBSET_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCPBSET_None $expectedCPBSET_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - - "*| Set*$paramDSV*" { - Write-Host "** Set ($paramDSV) Allocated Value test" - $foundSet_DSV = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSet_DSV $expectedSet_DSV $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| Get*$paramDSV*" { - Write-Host "** Get ($paramDSV) Allocated Value test" - $foundGet_DSV = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundGet_DSV $expectedGet_DSV $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MSet*$paramDSV*" { - Write-Host "** MSet ($paramDSV) Allocated Value test" - $foundMSet_DSV = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMSet_DSV $expectedMSet_DSV $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MGet*$paramDSV*" { - Write-Host "** MGet ($paramDSV) Allocated Value test" - $foundMGet_DSV = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMGet_DSV $expectedMGet_DSV $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| CPBSET*$paramDSV*" { - Write-Host "** CPBSET ($paramDSV) Allocated Value test" - $foundCPBSET_DSV = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCPBSET_DSV $expectedCPBSET_DSV $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - - "*| ZAddRem*$paramNone*" { - Write-Host "** ZAddRem ($paramNone) Allocated Value test" - $foundZAddRem_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRem_None $expectedZAddRem_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| LPushPop*$paramNone*" { - Write-Host "** LPushPop ($paramNone) Allocated Value test" - $foundLPushPop_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPop_None $expectedLPushPop_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| SAddRem*$paramNone*" { - Write-Host "** SAddRem ($paramNone) Allocated Value test" - $foundSAddRem_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRem_None $expectedSAddRem_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| HSetDel*$paramNone*" { - Write-Host "** HSetDel ($paramNone) Allocated Value test" - $foundHSetDel_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDel_None $expectedHSetDel_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MyDictSetGet*$paramNone*" { - Write-Host "** MyDictSetGet ($paramNone) Allocated Value test" - $foundMyDictSetGet_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGet_None $expectedMyDictSetGet_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| CustomProcSet*$paramNone*" { - Write-Host "** MyDictSetGet ($paramNone) Allocated Value test" - $foundCustomProcSet_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCustomProcSet_None $expectedCustomProcSet_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| ZAddRem*$paramACL*" { - Write-Host "** ZAddRem ($paramACL) Allocated Value test" - $foundZAddRem_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRem_ACL $expectedZAddRem_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| LPushPop*$paramACL*" { - Write-Host "** LPushPop ($paramACL) Allocated Value test" - $foundLPushPop_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPop_ACL $expectedLPushPop_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| SAddRem*$paramACL*" { - Write-Host "** SAddRem ($paramACL) Allocated Value test" - $foundSAddRem_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRem_ACL $expectedSAddRem_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| HSetDel*$paramACL*" { - Write-Host "** HSetDel ($paramACL) Allocated Value test" - $foundHSetDel_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDel_ACL $expectedHSetDel_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MyDictSetGet*$paramACL*" { - Write-Host "** MyDictSetGet ($paramACL) Allocated Value test" - $foundMyDictSetGet_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGet_ACL $expectedMyDictSetGet_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| CustomProcSet*$paramACL*" { - Write-Host "** MyDictSetGet ($paramACL) Allocated Value test" - $foundCustomProcSet_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundCustomProcSet_ACL $expectedCustomProcSet_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| ZAddRem*$paramAOF*" { - Write-Host "** ZAddRem ($paramAOF) Allocated Value test" - $foundZAddRem_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundZAddRem_AOF $expectedZAddRem_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| LPushPop*$paramAOF*" { - Write-Host "** LPushPop ($paramAOF) Allocated Value test" - $foundLPushPop_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundLPushPop_AOF $expectedLPushPop_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| SAddRem*$paramAOF*" { - Write-Host "** SAddRem ($paramAOF) Allocated Value test" - $foundSAddRem_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundSAddRem_AOF $expectedSAddRem_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| HSetDel*$paramAOF*" { - Write-Host "** HSetDel ($paramAOF) Allocated Value test" - $foundHSetDel_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundHSetDel_AOF $expectedHSetDel_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| MyDictSetGet*$paramAOF*" { - Write-Host "** MyDictSetGet ($paramAOF) Allocated Value test" - $foundMyDictSetGet_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyDictSetGet_AOF $expectedMyDictSetGet_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| CustomProcSet*$paramAOF*" { - Write-Host "** CustomProcSet ($paramAOF) Allocated Value test" - $foundMyCustomProcSet_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundMyCustomProcSet_AOF $expectedCustomProcSet_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } + # Get a value + for ($currentExpectedProp = 0; $currentExpectedProp -lt $totalExpectedResultValues; $currentExpectedProp++) { - "*| Script1*$paramNone*" { - Write-Host "** Lua Script1 ($paramNone) Allocated Value test" - $foundScript1_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript1_None $expectedScript1_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| Script2*$paramNone*" { - Write-Host "** Lua Script2 ($paramNone) Allocated Value test" - $foundScript2_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript2_None $expectedScript2_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| Script3*$paramNone*" { - Write-Host "** Lua Script3 ($paramNone) Allocated Value test" - $foundScript3_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript3_None $expectedScript3_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| Script4*$paramNone*" { - Write-Host "** LuaScript4 ($paramNone) Allocated Value test" - $foundScript4_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundScript4_None $expectedScript4_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } + # Check if the line contains the method name + if ($line -match [regex]::Escape($expectedResultsArray[$currentExpectedProp, 0])) { + # Found the method in the results, now check the param we looking for is in the line + if ($line -match [regex]::Escape($expectedResultsArray[$currentExpectedProp, 1])) { - "*| InlinePing*$paramNone*" { - Write-Host "** InlinePing ($paramNone) Allocated Value test" - $foundInlinePing_None = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundInlinePing_None $expectedInlinePing_None $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } + # Found Method and Param so know this is one we want, so get value + $foundValue = ParseValueFromResults $line $allocatedColumn - "*| InlinePing*$paramACL*" { - Write-Host "** InlinePing ($paramACL) Allocated Value test" - $foundInlinePing_ACL = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundInlinePing_ACL $expectedInlinePing_ACL $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false - } - } - "*| InlinePing*$paramAOF*" { - Write-Host "** InlinePing ($paramAOF) Allocated Value test" - $foundInlinePing_AOF = ParseValueFromResults $line $allocatedColumn - $currentResults = AnalyzeResult $foundInlinePing_AOF $expectedInlinePing_AOF $acceptableAllocatedRange $true - if ($currentResults -eq $false) { - $testSuiteResult = $false + # Check if found value is not equal to expected value + Write-Host "** Config: "$expectedResultsArray[$currentExpectedProp, 0].Substring(2) $expectedResultsArray[$currentExpectedProp, 1] + $currentResults = AnalyzeResult $foundValue $expectedResultsArray[$currentExpectedProp, 2] $acceptableAllocatedRange $true + if ($currentResults -eq $false) { + $testSuiteResult = $false + } } } } } -Write-Output "** " Write-Output "************************" Write-Output "** Final summary:" -Write-Output "** " +Write-Output "**" if ($testSuiteResult) { - Write-Output "** PASS! All tests passed " + Write-Output "** PASS! All tests in the suite passed." } else { - Write-Error -Message "** BDN Benchmark PERFORMANCE REGRESSION FAIL! At least one test had benchmark value outside of expected range. NOTE: Expected results are based on CI machine and may differ from the machine that this was ran on." + Write-Error -Message "** BDN Benchmark PERFORMANCE REGRESSION FAIL! At least one test had benchmark Allocated Memory value outside of expected range. NOTE: Expected results are based on CI machine and may differ from the machine that this was ran on." } -Write-Output "** " -Write-Output "************************" +Write-Output "**" +Write-Output "************************" \ No newline at end of file From 0f0e6c9b29c161235c71575598181bacfbde32d2 Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Dec 2024 15:44:44 -0800 Subject: [PATCH 41/44] Added workflow_dispatch: to deploy website yml so we can manually deploy website if we want --- .github/workflows/deploy-website.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 5523260161..6ee6ee4cfb 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -1,6 +1,7 @@ name: Deploy to GitHub Pages on: + workflow_dispatch: push: branches: - main From 5571602ff9b22ee8e1f25920e454a35872d43fac Mon Sep 17 00:00:00 2001 From: darrenge Date: Mon, 9 Dec 2024 16:55:08 -0800 Subject: [PATCH 42/44] Updated alert-threshold from 150% to 140%. If too many false positives we can raise it back. Also, removed darrenge from the alert CC line as no need for that. --- .github/workflows/ci-bdnbenchmark.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index a5e5405ff7..0c418245a9 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -81,10 +81,9 @@ jobs: tool: 'benchmarkdotnet' output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '150%' + alert-threshold: '140%' comment-on-alert: true fail-on-alert: true - alert-comment-cc-users: '@darrenge' summary-always: false gh-pages-branch: 'continuousbenchmark' benchmark-data-dir-path: 'website/static/charts' From 7799136d86db1387b78db5c215d5d7e891c1b08b Mon Sep 17 00:00:00 2001 From: darrenge Date: Tue, 10 Dec 2024 11:38:01 -0800 Subject: [PATCH 43/44] Removed the commit comment / comparison to previous commit task. Still will get summary data in the specific workflow run and still uploads data to charts (when pushing to main). --- .github/workflows/ci-bdnbenchmark.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 0c418245a9..b91cb8b8f6 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -74,21 +74,6 @@ jobs: ./test/BDNPerfTests/errorlog if: ${{ always() }} - # Run `github-action-benchmark` action to get Commit Comment - - name: Store benchmark result for commit comment - uses: benchmark-action/github-action-benchmark@v1 - with: - tool: 'benchmarkdotnet' - output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '140%' - comment-on-alert: true - fail-on-alert: true - summary-always: false - gh-pages-branch: 'continuousbenchmark' - benchmark-data-dir-path: 'website/static/charts' - auto-push: true - # Run `github-action-benchmark` action for the Continuous Benchmarking Charts (https://microsoft.github.io/garnet/charts/) - name: Store benchmark result for charts uses: benchmark-action/github-action-benchmark@v1 From 4e1975f7976a36705cc280033d8f2d6c6beeeb5f Mon Sep 17 00:00:00 2001 From: darrenge Date: Wed, 11 Dec 2024 09:24:05 -0800 Subject: [PATCH 44/44] Minor update - comments only --- .github/workflows/ci-bdnbenchmark.yml | 4 ++-- test/BDNPerfTests/BDN_Benchmark_Config.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index b91cb8b8f6..f3c0494039 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -11,8 +11,8 @@ env: permissions: actions: write - deployments: write #permission to deploy GitHub pages website - contents: write # permission to update benchmark contents in gh-pages branch + deployments: write # permission to deploy GitHub pages website + contents: write # permission to update benchmark contents in continuousbenchmark branch jobs: changes: diff --git a/test/BDNPerfTests/BDN_Benchmark_Config.json b/test/BDNPerfTests/BDN_Benchmark_Config.json index 146de35650..9782e13aec 100644 --- a/test/BDNPerfTests/BDN_Benchmark_Config.json +++ b/test/BDNPerfTests/BDN_Benchmark_Config.json @@ -1,6 +1,6 @@ { - "_comment": "NOTE: If adding a new BDN perf test to this file, you will need to add the name to the test: line in ce-bdnbenchmark.yml (~line 45). This ensures all the tests are not ran sequentially when ran as github action.", - "_comment2": "NOTE: The BDN Perf name in the yml test: line should NOT have the BDN.benchmark prefix and the .* ending. It needs to be this way for the reporting part of the BDN Charts.", + "_comment": "NOTE: If adding a new BDN perf test to this file, you will need to add the name to the test: line in ci-bdnbenchmark.yml (~line 45). This ensures all the tests are not ran sequentially when ran as github action.", + "_comment2": "NOTE: The BDN Perf name in the ci-bdnbenchmark.yml file test: line should NOT have the BDN.benchmark prefix and the .* ending. It needs to be this way for the reporting part of the BDN Charts.", "BDN.benchmark.Operations.BasicOperations.*": { "expected_InlinePing_ACL": 0,