diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9994df0a..5f4a76a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - missing_eval jobs: test: @@ -22,7 +23,7 @@ jobs: sparse-checkout-cone-mode: false path: 'bench_data' - - run: tree +# - run: tree - name: Evaluate benchmark uses: ./ @@ -33,26 +34,23 @@ jobs: file_with_bench_data: data.json folder_with_current_benchmark_results: resultsFolder bench_group_name: "MongoDB Benchmark" - evaluation_method: "threshold" - threshold_values: 2500,700,50000,5000,30000 - comparison_operators: tolerance,tolerance,tolerance,tolerance,tolerance - comparison_margins: 10,10,10,10,10 - result_files_merge_strategy_for_each_metric: "sum, average, average, average, average" + evaluation_method: "jump_detection" + jump_detection_thresholds: "10, 10, 10, 10, 10" save_curr_bench_res: true github_token: ${{ secrets.GH_TOKEN }} github_context: ${{ toJson(github) }} - - name: Commit files - working-directory: ./bench_data - run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git commit -a -m "Add changes" - git push origin bench_data - - - name: Check output and fail if needed - if: steps.eval.outputs.should_fail == 'true' - run: | - echo "Failing as indicated by evaluation action" - exit 1 +# - name: Commit files +# working-directory: ./bench_data +# run: | +# git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" +# git config --local user.name "github-actions[bot]" +# git commit -a -m "Add changes" +# git push origin bench_data +# +# - name: Check output and fail if needed +# if: steps.eval.outputs.should_fail == 'true' +# run: | +# echo "Failing as indicated by evaluation action" +# exit 1 diff --git a/dist/index.js b/dist/index.js index 1aa6fa69..07995f20 100644 --- a/dist/index.js +++ b/dist/index.js @@ -30073,7 +30073,6 @@ module.exports.createCurrBench = function (config) { }).filter(result => result !== null); let commit; - // if config.eventName === schedule, then we will not have if (config.eventName === 'schedule') { core.info('The workflow was triggered by a scheduled event.'); commit = getCommitReplacementWhenTriggeredByScheduledEvent(config.runId); @@ -30125,7 +30124,6 @@ module.exports.addCompleteBenchmarkToFile = async ( core.debug(`Reading file at ${pathToPreviousDataFile}`) try { const data = await fs.readFile(pathToPreviousDataFile, 'utf8') - //core.debug('Read file: ' + data) // -> can be very long... jsonData = JSON.parse(data) } catch (err) { core.debug( @@ -30328,13 +30326,9 @@ module.exports.getBenchFromWeekAgo = function ( ); let benchmarks = data.entries[benchmarkGroupToCompare]; - // Print the amount of benchmarks - let closestBenchmark = null; let smallestDifference = Infinity; - - benchmarks.forEach(benchmark => { let difference = Math.abs(now - benchmark.date - ONE_WEEK_IN_MS); if (difference < smallestDifference) { @@ -30397,7 +30391,6 @@ module.exports.getBenchmarkOfStableBranch = function (benchmarkGroupToCompare, f folderWithBenchData, fileNameWithBenchData ); let benchmarks = data.entries[benchmarkGroupToCompare]; - // find benchmark with commit sha == latestBenchSha let benchmark = benchmarks.find(benchmark => benchmark.commit.id === latestBenchSha); core.debug(`Benchmark of stable branch: ${JSON.stringify(benchmark)}`); @@ -30438,13 +30431,13 @@ module.exports.createComment = function ( commentBody = module.exports.createBodyForComparisonWithPrev(evaluationResult, completeConfig); break; case 'previous_successful': - commentBody = module.exports.createBodyForComparisonWithPrevSucc(evaluationResult, completeConfig); + commentBody = module.exports.createBodyForComparisonWithPrev(evaluationResult, completeConfig); break; case 'threshold_range': commentBody = module.exports.createBodyForComparisonWithThresholdRange(evaluationResult, completeConfig); break; case 'jump_detection': - commentBody = module.exports.createBodyForComparisonWithJumpDet(evaluationResult, completeConfig); + commentBody = module.exports.createBodyForComparisonWithJumpDeltas(evaluationResult, completeConfig); break; case 'trend_detection_moving_ave': commentBody = module.exports.createBodyForComparisonWithTrendDetMovAve(evaluationResult, completeConfig); @@ -30661,6 +30654,83 @@ module.exports.createBodyForComparisonWithTrendDetDeltas = function(evaluationRe return lines.join('\n') } + +module.exports.createBodyForComparisonWithJumpDeltas = function(evaluationResult, completeConfig) { + core.debug('------ start createBodyForComparisonWithJumpDeltas ------') + const currentBenchmark = evaluationResult.referenceBenchmarks.current; + const previousBenchmark = evaluationResult.referenceBenchmarks.previous; + + const lines = [] + lines.push('## Benchmark results') + lines.push('') + lines.push(`Benchmark group: ${currentBenchmark.benchmarkGroupName}`) + lines.push('') + lines.push(`The chosen evaluation method is jump_detection.`) + lines.push(`For each metric, there is the following condition: + The current value should not change more than X% (Max. ch in the table below) from the value measured for the previous benchmark.`) + + const currentBenchmarkGroupName = currentBenchmark.benchmarkGroupName + const previousBenchmarkGroupName = previousBenchmark.benchmarkGroupName + + if (currentBenchmarkGroupName !== previousBenchmarkGroupName) { + lines.push( + "Note: Benchmarks from different groups are being compared." + ) + } + const benchDataText = module.exports.createBenchDataTextForCompWithPrev( + currentBenchmark, + previousBenchmark + ) + lines.push(benchDataText) + + lines.push('', '', '', '', '') + lines.push('## Results') + lines.push('', '', '', '', '') + + lines.push( + `| Metric | Curr: ${currentBenchmark.commitInfo.id} | Prev: ${previousBenchmark.commitInfo.id} | Max. Jump | Was | Res | ` + ) + lines.push('|-|-|-|-|-|-|') + + const evaluationResults = evaluationResult.results.result + const evaluationParameters = evaluationResult.evalParameters + const evaluationConfiguration = completeConfig.evaluationConfig + for (let i = 0; i < evaluationResults.length; i++) { + + const resultStatus = evaluationResults[i]; + const metricName = evaluationParameters.metricNames[i]; + const metricUnit = evaluationParameters.metricUnits[i]; + + const currValue = currentBenchmark.simpleMetricResults[i].value; + const prevValue = previousBenchmark.simpleMetricResults[i].value; + + const currPlusUnit = currValue + ' ' + metricUnit; + const prevPlusUnit = prevValue + ' ' + metricUnit; + + const shouldBe = evaluationParameters.shouldBe[i]; + const ratio = evaluationParameters.is[i]; + + + let line + + + if (resultStatus === 'failed' || resultStatus === 'passed') { + let betterOrWorse = resultStatus === 'passed' ? '🟢' : '🔴' + line = `| \`${metricName}\` | \`${currPlusUnit}\` | \`${prevPlusUnit}\` | ${shouldBe} | ${ratio} | ${betterOrWorse} |` + } else { + line = `| \`${metricName}\` | \'${currPlusUnit}\' | N/A | N/A | N/A | 🔘 |` + } + + lines.push(line) + } + + const benchmarkPassed = module.exports.addInfoAboutBenchRes(lines, completeConfig, evaluationResults); + module.exports.alertUsersIfBenchFailed(benchmarkPassed, completeConfig, lines); + return lines.join('\n') +} + + + module.exports.createBenchDataText = function (currentBenchmark) { core.info('------ start createBenchDataText ------') const benchInfo = currentBenchmark.benchmarkInfo @@ -30897,7 +30967,7 @@ module.exports.createBodyForComparisonWithThresholdRange = function ( /////////////////////// /////////////////////// Summary /////////////////////// -module.exports.createWorkflowSummaryForCompWithPrev = function (evaluationResult, completeConfig) { +module.exports.createWorkflowSummaryForCompWithPrev = function (evaluationResult, completeConfig, successful) { const currentBenchmark = evaluationResult.referenceBenchmarks.current; const previousBenchmark = evaluationResult.referenceBenchmarks.previous; @@ -31185,6 +31255,97 @@ module.exports.addSummary = function (evaluationMethod, headers, rows, summaryMe .write(); } +module.exports.createWorkflowSummaryForJumpDetection = function (evaluationResult, completeConfig) { + const currentBenchmark = evaluationResult.referenceBenchmarks.current; + const previousBenchmark = evaluationResult.referenceBenchmarks.previous; + + const currentCommitId = completeConfig.eventName === 'schedule' ? currentBenchmark.commitInfo.id : currentBenchmark.commitInfo.id.substring(0, 7); + const previousCommitId = previousBenchmark.commitInfo.eventName === 'schedule' ? previousBenchmark.commitInfo.id : previousBenchmark.commitInfo.id.substring(0, 7); + + const headers = [ + { + data: 'Metric', + header: true, + }, + { + data: `Current: "${currentCommitId}"`, + header: true, + }, + { + data: `Previous: "${previousCommitId}"`, + header: true, + }, + + { + data: 'Jump', + header: true, + }, + { + data: 'Max. change [%]', + header: true, + }, + { + data: 'Result', + header: true, + } + + ]; + + const rows = []; + const evaluationResults = evaluationResult.results.result + const evaluationParameters = evaluationResult.evalParameters + const evaluationConfiguration = completeConfig.evaluationConfig + for (let i = 0; i < evaluationResults.length; i++) { + const resultStatus = evaluationResults[i]; + const metricName = evaluationParameters.metricNames[i]; + const metricUnit = evaluationParameters.metricUnits[i]; + + const currValue = currentBenchmark.simpleMetricResults[i].value; + const prevValue = previousBenchmark.simpleMetricResults[i].value; + + const currPlusUnit = currValue + ' ' + metricUnit; + const prevPlusUnit = prevValue + ' ' + metricUnit; + + const shouldBe = evaluationParameters.shouldBe[i]; + const ratio = evaluationParameters.is[i]; + + + let graphicalRepresentationOfRes; + if (resultStatus === 'failed' || resultStatus === 'passed') { + graphicalRepresentationOfRes = resultStatus === 'passed' ? '🟢' : '🔴' + } else { + graphicalRepresentationOfRes= '🔘'; + } + + rows.push([ + { + data: metricName, + }, + { + data: currPlusUnit, + }, + { + data: prevPlusUnit, + }, + { + data: ratio, + }, + { + data: shouldBe, + }, + { + data: graphicalRepresentationOfRes + }, + + ]) + } + let summaryMessage = module.exports.createSummaryMessage(evaluationResult); + const evaluationMethod = evaluationResult.evalParameters.evaluationMethod; + module.exports.addSummary(evaluationMethod, headers, rows, summaryMessage, completeConfig.linkToTemplatedGhPageWithResults, + completeConfig.eventName); +} + + module.exports.getEvaluationMethodSpecificDescriptionOfEvalMethod = function (evaluationMethod) { switch (evaluationMethod) { @@ -32337,7 +32498,7 @@ module.exports.evaluateWithJumpDetection = function (currentBenchmarkData, confi shouldBe.push(threshold); if (previousResult) { - const ratio = (currentValue / previousResult.value - 1) * 100; + const ratio = Math.abs((currentValue / previousResult.value - 1) * 100); ratios.push(ratio.toFixed(2)); const isPassed = Math.abs(ratio) < threshold; evaluationResults.push(isPassed ? 'passed' : 'failed'); @@ -32572,7 +32733,7 @@ const { createCurrBench} = __nccwpck_require__(501) const { createComment, createWorkflowSummaryForCompWithPrev, createWorkflowSummaryThreshold, summaryForMethodNotSupported, createWorkflowSummaryForThresholdRange, - createWorkflowSummaryForTrendDetDeltas} = __nccwpck_require__(3732) + createWorkflowSummaryForTrendDetDeltas, createWorkflowSummaryForJumpDetection} = __nccwpck_require__(3732) const { addCompleteBenchmarkToFile, @@ -32665,13 +32826,17 @@ async function run() { if (addJobSummary === 'on' || (addJobSummary === 'if_failed' && shouldFail)) { if (evaluationConfig.evaluationMethod === 'previous') { - createWorkflowSummaryForCompWithPrev(evaluationResult, completeConfig); + createWorkflowSummaryForCompWithPrev(evaluationResult, completeConfig, false); + } else if (evaluationConfig.evaluationMethod === 'previous_successful') { + createWorkflowSummaryForCompWithPrev(evaluationResult, completeConfig, true); } else if (evaluationConfig.evaluationMethod === 'threshold') { createWorkflowSummaryThreshold(evaluationResult, completeConfig); } else if (evaluationConfig.evaluationMethod === 'threshold_range') { createWorkflowSummaryForThresholdRange(evaluationResult, completeConfig) } else if (evaluationConfig.evaluationMethod === 'trend_detection_deltas') { createWorkflowSummaryForTrendDetDeltas(evaluationResult, completeConfig); + } else if (evaluationConfig.evaluationMethod === 'jump_detection') { + createWorkflowSummaryForJumpDetection(evaluationResult, completeConfig); } else { diff --git a/resultsFolder/result_wg-medium2.json b/resultsFolder/result_wg-medium2.json deleted file mode 100644 index a46aa452..00000000 --- a/resultsFolder/result_wg-medium2.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "benchInfo": { - "executionTime": "0m 11s", - "otherInfo": "YCSB Parameters: workload=workloada, recordcount=200000, threads=32, ", - "parametrization": { - "storageEngine": "N/A", - "logVerbosity": "N/A", - "bindIp": "0.0.0.0", - "port": 27017 - } - }, - "results": [ - { - "unit": "ops/sec", - "value": 16884.761502743775, - "name": "OVERALL Throughput" - }, - { - "unit": "us", - "value": 1765.6541831135073, - "name": "[READ], AverageLatency(us)" - }, - { - "unit": "us", - "value": 65407.0, - "name": "[READ], MaxLatency(us)" - }, - { - "unit": "us", - "value": 1866.0130910511919, - "name": "[UPDATE], AverageLatency(us)" - }, - { - "unit": "us", - "value": 156159.0, - "name": "[UPDATE], MaxLatency(us)" - } - ] -} \ No newline at end of file diff --git a/resultsFolder/result_wg-medium3.json b/resultsFolder/result_wg-medium3.json deleted file mode 100644 index 9b2a400e..00000000 --- a/resultsFolder/result_wg-medium3.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "benchInfo": { - "executionTime": "0m 11s", - "otherInfo": "YCSB Parameters: workload=workloada, recordcount=200000, threads=32, ", - "parametrization": { - "storageEngine": "N/A", - "logVerbosity": "N/A", - "bindIp": "0.0.0.0", - "port": 27017 - } - }, - "results": [ - { - "unit": "ops/sec", - "value": 16770.08217340265, - "name": "OVERALL Throughput" - }, - { - "unit": "us", - "value": 1817.7090485888857, - "name": "[READ], AverageLatency(us)" - }, - { - "unit": "us", - "value": 158847.0, - "name": "[READ], MaxLatency(us)" - }, - { - "unit": "us", - "value": 1861.29883843717, - "name": "[UPDATE], AverageLatency(us)" - }, - { - "unit": "us", - "value": 155135.0, - "name": "[UPDATE], MaxLatency(us)" - } - ] -} \ No newline at end of file diff --git a/resultsFolder/result_wg-medium4.json b/resultsFolder/result_wg-medium4.json deleted file mode 100644 index 7ca039ff..00000000 --- a/resultsFolder/result_wg-medium4.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "benchInfo": { - "executionTime": "0m 11s", - "otherInfo": "YCSB Parameters: workload=workloada, recordcount=200000, threads=32, ", - "parametrization": { - "storageEngine": "N/A", - "logVerbosity": "N/A", - "bindIp": "0.0.0.0", - "port": 27017 - } - }, - "results": [ - { - "unit": "ops/sec", - "value": 16810.960746406658, - "name": "OVERALL Throughput" - }, - { - "unit": "us", - "value": 1834.0088426016898, - "name": "[READ], AverageLatency(us)" - }, - { - "unit": "us", - "value": 159487.0, - "name": "[READ], MaxLatency(us)" - }, - { - "unit": "us", - "value": 1814.1897455816663, - "name": "[UPDATE], AverageLatency(us)" - }, - { - "unit": "us", - "value": 51807.0, - "name": "[UPDATE], MaxLatency(us)" - } - ] -} \ No newline at end of file diff --git a/src/bench.js b/src/bench.js index aaf7dc04..4a8ae447 100644 --- a/src/bench.js +++ b/src/bench.js @@ -30,7 +30,6 @@ module.exports.createCurrBench = function (config) { }).filter(result => result !== null); let commit; - // if config.eventName === schedule, then we will not have if (config.eventName === 'schedule') { core.info('The workflow was triggered by a scheduled event.'); commit = getCommitReplacementWhenTriggeredByScheduledEvent(config.runId); diff --git a/src/bench_data.js b/src/bench_data.js index 00eb9e48..6ebe439b 100644 --- a/src/bench_data.js +++ b/src/bench_data.js @@ -26,7 +26,6 @@ module.exports.addCompleteBenchmarkToFile = async ( core.debug(`Reading file at ${pathToPreviousDataFile}`) try { const data = await fs.readFile(pathToPreviousDataFile, 'utf8') - //core.debug('Read file: ' + data) // -> can be very long... jsonData = JSON.parse(data) } catch (err) { core.debug( @@ -229,13 +228,9 @@ module.exports.getBenchFromWeekAgo = function ( ); let benchmarks = data.entries[benchmarkGroupToCompare]; - // Print the amount of benchmarks - let closestBenchmark = null; let smallestDifference = Infinity; - - benchmarks.forEach(benchmark => { let difference = Math.abs(now - benchmark.date - ONE_WEEK_IN_MS); if (difference < smallestDifference) { @@ -298,7 +293,6 @@ module.exports.getBenchmarkOfStableBranch = function (benchmarkGroupToCompare, f folderWithBenchData, fileNameWithBenchData ); let benchmarks = data.entries[benchmarkGroupToCompare]; - // find benchmark with commit sha == latestBenchSha let benchmark = benchmarks.find(benchmark => benchmark.commit.id === latestBenchSha); core.debug(`Benchmark of stable branch: ${JSON.stringify(benchmark)}`); diff --git a/src/comment.js b/src/comment.js index cdee7b75..43dcf14c 100644 --- a/src/comment.js +++ b/src/comment.js @@ -19,13 +19,13 @@ module.exports.createComment = function ( commentBody = module.exports.createBodyForComparisonWithPrev(evaluationResult, completeConfig); break; case 'previous_successful': - commentBody = module.exports.createBodyForComparisonWithPrevSucc(evaluationResult, completeConfig); + commentBody = module.exports.createBodyForComparisonWithPrev(evaluationResult, completeConfig); break; case 'threshold_range': commentBody = module.exports.createBodyForComparisonWithThresholdRange(evaluationResult, completeConfig); break; case 'jump_detection': - commentBody = module.exports.createBodyForComparisonWithJumpDet(evaluationResult, completeConfig); + commentBody = module.exports.createBodyForComparisonWithJumpDeltas(evaluationResult, completeConfig); break; case 'trend_detection_moving_ave': commentBody = module.exports.createBodyForComparisonWithTrendDetMovAve(evaluationResult, completeConfig); @@ -242,6 +242,83 @@ module.exports.createBodyForComparisonWithTrendDetDeltas = function(evaluationRe return lines.join('\n') } + +module.exports.createBodyForComparisonWithJumpDeltas = function(evaluationResult, completeConfig) { + core.debug('------ start createBodyForComparisonWithJumpDeltas ------') + const currentBenchmark = evaluationResult.referenceBenchmarks.current; + const previousBenchmark = evaluationResult.referenceBenchmarks.previous; + + const lines = [] + lines.push('## Benchmark results') + lines.push('') + lines.push(`Benchmark group: ${currentBenchmark.benchmarkGroupName}`) + lines.push('') + lines.push(`The chosen evaluation method is jump_detection.`) + lines.push(`For each metric, there is the following condition: + The current value should not change more than X% (Max. ch in the table below) from the value measured for the previous benchmark.`) + + const currentBenchmarkGroupName = currentBenchmark.benchmarkGroupName + const previousBenchmarkGroupName = previousBenchmark.benchmarkGroupName + + if (currentBenchmarkGroupName !== previousBenchmarkGroupName) { + lines.push( + "Note: Benchmarks from different groups are being compared." + ) + } + const benchDataText = module.exports.createBenchDataTextForCompWithPrev( + currentBenchmark, + previousBenchmark + ) + lines.push(benchDataText) + + lines.push('', '', '', '', '') + lines.push('## Results') + lines.push('', '', '', '', '') + + lines.push( + `| Metric | Curr: ${currentBenchmark.commitInfo.id} | Prev: ${previousBenchmark.commitInfo.id} | Max. Jump | Was | Res | ` + ) + lines.push('|-|-|-|-|-|-|') + + const evaluationResults = evaluationResult.results.result + const evaluationParameters = evaluationResult.evalParameters + const evaluationConfiguration = completeConfig.evaluationConfig + for (let i = 0; i < evaluationResults.length; i++) { + + const resultStatus = evaluationResults[i]; + const metricName = evaluationParameters.metricNames[i]; + const metricUnit = evaluationParameters.metricUnits[i]; + + const currValue = currentBenchmark.simpleMetricResults[i].value; + const prevValue = previousBenchmark.simpleMetricResults[i].value; + + const currPlusUnit = currValue + ' ' + metricUnit; + const prevPlusUnit = prevValue + ' ' + metricUnit; + + const shouldBe = evaluationParameters.shouldBe[i]; + const ratio = evaluationParameters.is[i]; + + + let line + + + if (resultStatus === 'failed' || resultStatus === 'passed') { + let betterOrWorse = resultStatus === 'passed' ? '🟢' : '🔴' + line = `| \`${metricName}\` | \`${currPlusUnit}\` | \`${prevPlusUnit}\` | ${shouldBe} | ${ratio} | ${betterOrWorse} |` + } else { + line = `| \`${metricName}\` | \'${currPlusUnit}\' | N/A | N/A | N/A | 🔘 |` + } + + lines.push(line) + } + + const benchmarkPassed = module.exports.addInfoAboutBenchRes(lines, completeConfig, evaluationResults); + module.exports.alertUsersIfBenchFailed(benchmarkPassed, completeConfig, lines); + return lines.join('\n') +} + + + module.exports.createBenchDataText = function (currentBenchmark) { core.info('------ start createBenchDataText ------') const benchInfo = currentBenchmark.benchmarkInfo @@ -478,7 +555,7 @@ module.exports.createBodyForComparisonWithThresholdRange = function ( /////////////////////// /////////////////////// Summary /////////////////////// -module.exports.createWorkflowSummaryForCompWithPrev = function (evaluationResult, completeConfig) { +module.exports.createWorkflowSummaryForCompWithPrev = function (evaluationResult, completeConfig, successful) { const currentBenchmark = evaluationResult.referenceBenchmarks.current; const previousBenchmark = evaluationResult.referenceBenchmarks.previous; @@ -766,6 +843,97 @@ module.exports.addSummary = function (evaluationMethod, headers, rows, summaryMe .write(); } +module.exports.createWorkflowSummaryForJumpDetection = function (evaluationResult, completeConfig) { + const currentBenchmark = evaluationResult.referenceBenchmarks.current; + const previousBenchmark = evaluationResult.referenceBenchmarks.previous; + + const currentCommitId = completeConfig.eventName === 'schedule' ? currentBenchmark.commitInfo.id : currentBenchmark.commitInfo.id.substring(0, 7); + const previousCommitId = previousBenchmark.commitInfo.eventName === 'schedule' ? previousBenchmark.commitInfo.id : previousBenchmark.commitInfo.id.substring(0, 7); + + const headers = [ + { + data: 'Metric', + header: true, + }, + { + data: `Current: "${currentCommitId}"`, + header: true, + }, + { + data: `Previous: "${previousCommitId}"`, + header: true, + }, + + { + data: 'Jump', + header: true, + }, + { + data: 'Max. change [%]', + header: true, + }, + { + data: 'Result', + header: true, + } + + ]; + + const rows = []; + const evaluationResults = evaluationResult.results.result + const evaluationParameters = evaluationResult.evalParameters + const evaluationConfiguration = completeConfig.evaluationConfig + for (let i = 0; i < evaluationResults.length; i++) { + const resultStatus = evaluationResults[i]; + const metricName = evaluationParameters.metricNames[i]; + const metricUnit = evaluationParameters.metricUnits[i]; + + const currValue = currentBenchmark.simpleMetricResults[i].value; + const prevValue = previousBenchmark.simpleMetricResults[i].value; + + const currPlusUnit = currValue + ' ' + metricUnit; + const prevPlusUnit = prevValue + ' ' + metricUnit; + + const shouldBe = evaluationParameters.shouldBe[i]; + const ratio = evaluationParameters.is[i]; + + + let graphicalRepresentationOfRes; + if (resultStatus === 'failed' || resultStatus === 'passed') { + graphicalRepresentationOfRes = resultStatus === 'passed' ? '🟢' : '🔴' + } else { + graphicalRepresentationOfRes= '🔘'; + } + + rows.push([ + { + data: metricName, + }, + { + data: currPlusUnit, + }, + { + data: prevPlusUnit, + }, + { + data: ratio, + }, + { + data: shouldBe, + }, + { + data: graphicalRepresentationOfRes + }, + + ]) + } + let summaryMessage = module.exports.createSummaryMessage(evaluationResult); + const evaluationMethod = evaluationResult.evalParameters.evaluationMethod; + module.exports.addSummary(evaluationMethod, headers, rows, summaryMessage, completeConfig.linkToTemplatedGhPageWithResults, + completeConfig.eventName); +} + + module.exports.getEvaluationMethodSpecificDescriptionOfEvalMethod = function (evaluationMethod) { switch (evaluationMethod) { diff --git a/src/evaluate.js b/src/evaluate.js index 5c1f4dc4..79161a1d 100644 --- a/src/evaluate.js +++ b/src/evaluate.js @@ -240,7 +240,7 @@ module.exports.evaluateWithJumpDetection = function (currentBenchmarkData, confi shouldBe.push(threshold); if (previousResult) { - const ratio = (currentValue / previousResult.value - 1) * 100; + const ratio = Math.abs((currentValue / previousResult.value - 1) * 100); ratios.push(ratio.toFixed(2)); const isPassed = Math.abs(ratio) < threshold; evaluationResults.push(isPassed ? 'passed' : 'failed'); diff --git a/src/main.js b/src/main.js index f6764b99..5de0574a 100644 --- a/src/main.js +++ b/src/main.js @@ -16,7 +16,7 @@ const { createCurrBench} = require('./bench') const { createComment, createWorkflowSummaryForCompWithPrev, createWorkflowSummaryThreshold, summaryForMethodNotSupported, createWorkflowSummaryForThresholdRange, - createWorkflowSummaryForTrendDetDeltas} = require('./comment') + createWorkflowSummaryForTrendDetDeltas, createWorkflowSummaryForJumpDetection} = require('./comment') const { addCompleteBenchmarkToFile, @@ -109,13 +109,17 @@ async function run() { if (addJobSummary === 'on' || (addJobSummary === 'if_failed' && shouldFail)) { if (evaluationConfig.evaluationMethod === 'previous') { - createWorkflowSummaryForCompWithPrev(evaluationResult, completeConfig); + createWorkflowSummaryForCompWithPrev(evaluationResult, completeConfig, false); + } else if (evaluationConfig.evaluationMethod === 'previous_successful') { + createWorkflowSummaryForCompWithPrev(evaluationResult, completeConfig, true); } else if (evaluationConfig.evaluationMethod === 'threshold') { createWorkflowSummaryThreshold(evaluationResult, completeConfig); } else if (evaluationConfig.evaluationMethod === 'threshold_range') { createWorkflowSummaryForThresholdRange(evaluationResult, completeConfig) } else if (evaluationConfig.evaluationMethod === 'trend_detection_deltas') { createWorkflowSummaryForTrendDetDeltas(evaluationResult, completeConfig); + } else if (evaluationConfig.evaluationMethod === 'jump_detection') { + createWorkflowSummaryForJumpDetection(evaluationResult, completeConfig); } else {