From 73936362fe9eda1e2a2ee27a44164eea05ad2a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pascal=20Spo=CC=88rri?= Date: Tue, 27 Jun 2023 17:23:08 +0200 Subject: [PATCH] Cleanup unused variables and imports. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Pascal SpoĢˆrri --- .../apache/spark/shuffle/helper/S3ShuffleDispatcher.scala | 6 ------ .../org/apache/spark/shuffle/sort/S3ShuffleManager.scala | 3 --- .../org/apache/spark/shuffle/S3ShuffleManagerTest.scala | 1 - .../scala/org/apache/spark/shuffle/S3SortShuffleTest.scala | 1 - 4 files changed, 11 deletions(-) diff --git a/src/main/scala/org/apache/spark/shuffle/helper/S3ShuffleDispatcher.scala b/src/main/scala/org/apache/spark/shuffle/helper/S3ShuffleDispatcher.scala index bb902c6..41c5910 100644 --- a/src/main/scala/org/apache/spark/shuffle/helper/S3ShuffleDispatcher.scala +++ b/src/main/scala/org/apache/spark/shuffle/helper/S3ShuffleDispatcher.scala @@ -34,9 +34,6 @@ class S3ShuffleDispatcher extends Logging { val alwaysCreateIndex: Boolean = conf.getBoolean("spark.shuffle.s3.alwaysCreateIndex", defaultValue = false) val useBlockManager: Boolean = conf.getBoolean("spark.shuffle.s3.useBlockManager", defaultValue = true) val forceBatchFetch: Boolean = conf.getBoolean("spark.shuffle.s3.forceBatchFetch", defaultValue = false) - val allowSerializedShuffle: Boolean = conf.getBoolean("spark.shuffle.s3.allowSerializedShuffle", defaultValue = true) - val forceBypassMergeSort: Boolean = conf.getBoolean("spark.shuffle.s3.forceBypassMergeSort", defaultValue = false) - val sortShuffleCloneRecords: Boolean = conf.getBoolean("spark.shuffle.s3.sort.cloneRecords", defaultValue = false) val appDir = f"/${startTime}-${appId}/" val fs: FileSystem = FileSystem.get(URI.create(rootDir), { @@ -49,9 +46,6 @@ class S3ShuffleDispatcher extends Logging { logInfo(s"- spark.shuffle.s3.alwaysCreateIndex=${alwaysCreateIndex}") logInfo(s"- spark.shuffle.s3.useBlockManager=${useBlockManager}") logInfo(s"- spark.shuffle.s3.forceBatchFetch=${forceBatchFetch}") - logInfo(s"- spark.shuffle.s3.allowSerializedShuffle=${allowSerializedShuffle}") - logInfo(s"- spark.shuffle.s3.forceBypassMergeSort=${forceBypassMergeSort}") - logInfo(s"- spark.shuffle.s3.sort.cloneRecords=${sortShuffleCloneRecords}") def removeRoot(): Boolean = { Range(0, 10).map(idx => { diff --git a/src/main/scala/org/apache/spark/shuffle/sort/S3ShuffleManager.scala b/src/main/scala/org/apache/spark/shuffle/sort/S3ShuffleManager.scala index 1bd0bca..40ade35 100644 --- a/src/main/scala/org/apache/spark/shuffle/sort/S3ShuffleManager.scala +++ b/src/main/scala/org/apache/spark/shuffle/sort/S3ShuffleManager.scala @@ -27,12 +27,9 @@ import org.apache.spark._ import org.apache.spark.internal.Logging import org.apache.spark.shuffle._ import org.apache.spark.shuffle.api.ShuffleExecutorComponents -import org.apache.spark.shuffle.helper.S3ShuffleHelper.dispatcher import org.apache.spark.shuffle.helper.{S3ShuffleDispatcher, S3ShuffleHelper} import org.apache.spark.storage.S3ShuffleReader -import org.apache.spark.util.collection.OpenHashSet -import java.util.concurrent.ConcurrentHashMap import scala.collection.JavaConverters._ import scala.collection.mutable import scala.concurrent.ExecutionContext.Implicits.global diff --git a/src/test/scala/org/apache/spark/shuffle/S3ShuffleManagerTest.scala b/src/test/scala/org/apache/spark/shuffle/S3ShuffleManagerTest.scala index 1ad7409..e43451c 100644 --- a/src/test/scala/org/apache/spark/shuffle/S3ShuffleManagerTest.scala +++ b/src/test/scala/org/apache/spark/shuffle/S3ShuffleManagerTest.scala @@ -252,5 +252,4 @@ class S3ShuffleManagerTest { .set("spark.local.dir", "./spark-temp") // Configure the working dir. .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.S3ShuffleManager") .set("spark.shuffle.sort.io.plugin.class", "org.apache.spark.shuffle.S3ShuffleDataIO") - .set("spark.shuffle.s3.forceBypassMergeSort", "false") } diff --git a/src/test/scala/org/apache/spark/shuffle/S3SortShuffleTest.scala b/src/test/scala/org/apache/spark/shuffle/S3SortShuffleTest.scala index c56da60..c9d9047 100644 --- a/src/test/scala/org/apache/spark/shuffle/S3SortShuffleTest.scala +++ b/src/test/scala/org/apache/spark/shuffle/S3SortShuffleTest.scala @@ -116,7 +116,6 @@ class S3SortShuffleTest { .set("spark.local.dir", "./spark-temp") // Configure the working dir. .set("spark.shuffle.sort.io.plugin.class", "org.apache.spark.shuffle.S3ShuffleDataIO") .set("spark.shuffle.manager", "org.apache.spark.shuffle.sort.S3ShuffleManager") - .set("spark.shuffle.s3.forceBypassMergeSort", "false") .set("spark.shuffle.s3.cleanup", "false") // Avoid issues with cleanup. def fakeTaskContext(env: SparkEnv): TaskContext = {