diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/metrics/MetricsSparkListener.scala b/flint-core/src/main/scala/org/opensearch/flint/core/metrics/MetricsSparkListener.scala index 1138545c7..2ee941260 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/metrics/MetricsSparkListener.scala +++ b/flint-core/src/main/scala/org/opensearch/flint/core/metrics/MetricsSparkListener.scala @@ -34,14 +34,9 @@ class MetricsSparkListener extends SparkListener with Logging { recordsWritten += outputMetrics.recordsWritten totalJvmGcTime += taskEnd.taskMetrics.jvmGCTime - MetricsUtil.addHistoricGauge(MetricConstants.TASK_JVM_GC_TIME_METRIC, taskEnd.taskMetrics.jvmGCTime) - } - - override def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit = { - executorMetricsUpdate.executorUpdates.foreach { case (taskId, metrics) => - val totalGcTime = metrics.getMetricValue("totalGCTime") - logInfo(s"ExecutorID: ${executorMetricsUpdate.execId}, Task ID: $taskId, Executor totalGcTime: $totalGcTime") - } + MetricsUtil.addHistoricGauge( + MetricConstants.TASK_JVM_GC_TIME_METRIC, + taskEnd.taskMetrics.jvmGCTime) } def emitMetrics(): Unit = { diff --git a/spark-sql-application/src/main/scala/org/apache/spark/sql/JobOperator.scala b/spark-sql-application/src/main/scala/org/apache/spark/sql/JobOperator.scala index 316b91c2e..8582d3037 100644 --- a/spark-sql-application/src/main/scala/org/apache/spark/sql/JobOperator.scala +++ b/spark-sql-application/src/main/scala/org/apache/spark/sql/JobOperator.scala @@ -14,7 +14,7 @@ import scala.util.{Failure, Success, Try} import org.opensearch.flint.common.model.FlintStatement import org.opensearch.flint.common.scheduler.model.LangType -import org.opensearch.flint.core.metrics.{MetricConstants, MetricsUtil, MetricsSparkListener} +import org.opensearch.flint.core.metrics.{MetricConstants, MetricsSparkListener, MetricsUtil} import org.opensearch.flint.core.metrics.MetricsUtil.incrementCounter import org.opensearch.flint.spark.FlintSpark