From ea2648f865797902b9ed2d8d23014734d6324a84 Mon Sep 17 00:00:00 2001 From: cindyyuanjiang Date: Sun, 23 Feb 2025 22:06:36 -0800 Subject: [PATCH] updated PR due to recent changes in tools repo Signed-off-by: cindyyuanjiang --- .../tool/analysis/AppSQLPlanAnalyzer.scala | 1 + .../tool/analysis/StatisticsMetrics.scala | 2 +- ...g_filtereddiagnosticmetrics_expectation.csv | 18 +++++++++--------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSQLPlanAnalyzer.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSQLPlanAnalyzer.scala index 68f72130d..9041efdd8 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSQLPlanAnalyzer.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/AppSQLPlanAnalyzer.scala @@ -501,6 +501,7 @@ class AppSQLPlanAnalyzer(app: AppBase, appIndex: Int) extends AppAnalysisBase(ap stageAccumResult.min, stageAccumResult.median, stageAccumResult.max, + 0, stageAccumResult.total ) } diff --git a/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/StatisticsMetrics.scala b/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/StatisticsMetrics.scala index 5c7e505cb..f8c8e79c9 100644 --- a/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/StatisticsMetrics.scala +++ b/core/src/main/scala/com/nvidia/spark/rapids/tool/analysis/StatisticsMetrics.scala @@ -18,7 +18,7 @@ package com.nvidia.spark.rapids.tool.analysis import org.apache.spark.sql.rapids.tool.util.InPlaceMedianArrView.{chooseMidpointPivotInPlace, findMedianInPlace} -// Store (min, median, max, total) for a given metric +// Store (min, median, max, count, total) for a given metric case class StatisticsMetrics(min: Long, med: Long, max: Long, count: Long, total: Long) object StatisticsMetrics { diff --git a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_filtereddiagnosticmetrics_expectation.csv b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_filtereddiagnosticmetrics_expectation.csv index 540fdd9e9..75dcef894 100644 --- a/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_filtereddiagnosticmetrics_expectation.csv +++ b/core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_filtereddiagnosticmetrics_expectation.csv @@ -1,15 +1,15 @@ appIndex,appName,appId,sqlId,stageId,stageDurationMs,nodeId,nodeName,numFilesReadMin,numFilesReadMedian,numFilesReadMax,numFilesReadTotal,numPartitionsMin,numPartitionsMedian,numPartitionsMax,numPartitionsTotal,metadataTimeMin,metadataTimeMedian,metadataTimeMax,metadataTimeTotal,outputBatchesMin,outputBatchesMedian,outputBatchesMax,outputBatchesTotal,inputBatchesMin,inputBatchesMedian,inputBatchesMax,inputBatchesTotal,outputRowsMin,outputRowsMedian,outputRowsMax,outputRowsTotal,sortTimeMin,sortTimeMedian,sortTimeMax,sortTimeTotal,peakMemoryMin,peakMemoryMedian,peakMemoryMax,peakMemoryTotal,shuffleBytesWrittenMin,shuffleBytesWrittenMedian,shuffleBytesWrittenMax,shuffleBytesWrittenTotal,shuffleWriteTimeMin,shuffleWriteTimeMedian,shuffleWriteTimeMax,shuffleWriteTimeTotal -1,Spark shell,local-1622814619968,0,0,1743,16,"GpuColumnarExchange",0,0,0,0,0,0,0,0,0,0,0,0,200,200,200,1200,0,0,0,0,1666666,1666667,1666667,10000000,0,0,0,0,0,0,0,0,6688608,6688702,6688825,40132250,41434653,60830365,100858775,400284505 -1,Spark shell,local-1622814619968,0,0,1743,21,"Scan",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1666666,1666667,1666667,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,Spark shell,local-1622814619968,0,1,1631,8,"GpuColumnarExchange",0,0,0,0,0,0,0,0,0,0,0,0,200,200,200,1200,0,0,0,0,1666666,1666667,1666667,10000000,0,0,0,0,0,0,0,0,6688602,6688708,6688833,40132258,37444140,92128351,108992798,508750471 -1,Spark shell,local-1622814619968,0,1,1631,13,"Scan",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1666666,1666667,1666667,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,Spark shell,local-1622814619968,0,2,688,3,"GpuColumnarExchange",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,1,1,1,200,0,0,0,0,0,0,0,0,77,77,77,15400,139875,230038,9747416,93193331 +1,Spark shell,local-1622814619968,0,0,1743,16,"GpuColumnarExchange",0,0,0,0,0,0,0,0,0,0,0,0,200,200,200,1200,0,0,0,0,1666666,1666666,1666667,10000000,0,0,0,0,0,0,0,0,6688608,6688707,6688825,40132250,41434653,66714083,100858775,400284505 +1,Spark shell,local-1622814619968,0,0,1743,21,"Scan",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1666666,1666666,1666667,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,Spark shell,local-1622814619968,0,1,1631,8,"GpuColumnarExchange",0,0,0,0,0,0,0,0,0,0,0,0,200,200,200,1200,0,0,0,0,1666666,1666666,1666667,10000000,0,0,0,0,0,0,0,0,6688602,6688708,6688833,40132258,37444140,84791745,108992798,508750471 +1,Spark shell,local-1622814619968,0,1,1631,13,"Scan",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1666666,1666666,1666667,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,Spark shell,local-1622814619968,0,2,688,3,"GpuColumnarExchange",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,1,1,1,200,0,0,0,0,0,0,0,0,77,77,77,15400,139875,465911,9747416,93193331 1,Spark shell,local-1622814619968,0,2,688,4,"GpuHashAggregate",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,1,1,1,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,Spark shell,local-1622814619968,0,2,688,6,"GpuShuffledHashJoin",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,50017,50559,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,Spark shell,local-1622814619968,0,2,688,7,"GpuShuffleCoalesce",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,50017,50559,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,Spark shell,local-1622814619968,0,2,688,6,"GpuShuffledHashJoin",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,49948,50559,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,Spark shell,local-1622814619968,0,2,688,7,"GpuShuffleCoalesce",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,49948,50559,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 1,Spark shell,local-1622814619968,0,2,688,8,"GpuColumnarExchange",0,0,0,0,1,1,1,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,Spark shell,local-1622814619968,0,2,688,14,"GpuCoalesceBatches",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,50017,50559,10000000,0,0,0,0,395840,400140,404472,80000000,0,0,0,0,0,0,0,0 -1,Spark shell,local-1622814619968,0,2,688,15,"GpuShuffleCoalesce",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,50017,50559,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,Spark shell,local-1622814619968,0,2,688,14,"GpuCoalesceBatches",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,49948,50559,10000000,0,0,0,0,395840,399950,404472,80000000,0,0,0,0,0,0,0,0 +1,Spark shell,local-1622814619968,0,2,688,15,"GpuShuffleCoalesce",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,200,0,0,0,0,49480,49948,50559,10000000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 1,Spark shell,local-1622814619968,0,2,688,16,"GpuColumnarExchange",0,0,0,0,1,1,1,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 1,Spark shell,local-1622814619968,0,3,83,1,"GpuHashAggregate",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 1,Spark shell,local-1622814619968,0,3,83,2,"GpuShuffleCoalesce",0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,200,200,200,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0