From 786a70e710369b195d7c117b33fe9983044014d6 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sat, 12 Mar 2022 21:23:46 -0800 Subject: [PATCH] [SPARK-38537][K8S] Unify `Statefulset*` to `StatefulSet*` ### What changes were proposed in this pull request? K8s has [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) and Apache Spark is currently using both `Statefulset*` and `StatefulSet*`. The worst case is a mixed case like `class StatefulSetAllocatorSuite` in `StatefulsetAllocatorSuite.scala`. This PR aims to unify them to a K8s original name, `StatefulSet`. https://github.com/apache/spark/blob/c032928515e74367137c668ce692d8fd53696485/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala#L39 To sum up, two files are renamed and five files are changed. ``` $ git diff master --stat resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala | 5 ++--- resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala | 2 +- resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetPodsAllocator.scala => StatefulSetPodsAllocator.scala} | 2 +- resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala | 4 ++-- resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetAllocatorSuite.scala => StatefulSetAllocatorSuite.scala} | 4 ++-- 5 files changed, 8 insertions(+), 9 deletions(-) ``` ### Why are the changes needed? To be consistent not only inside Apache Spark, but also with K8s. ### Does this PR introduce _any_ user-facing change? No. This is a new code in Apache Spark 3.3. ### How was this patch tested? Pass the CIs. Closes #35827 from dongjoon-hyun/SPARK-38537. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala | 5 ++--- .../scheduler/cluster/k8s/KubernetesClusterManager.scala | 2 +- ...setPodsAllocator.scala => StatefulSetPodsAllocator.scala} | 2 +- .../cluster/k8s/KubernetesClusterManagerSuite.scala | 4 ++-- ...tAllocatorSuite.scala => StatefulSetAllocatorSuite.scala} | 4 ++-- 5 files changed, 8 insertions(+), 9 deletions(-) rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetPodsAllocator.scala => StatefulSetPodsAllocator.scala} (99%) rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetAllocatorSuite.scala => StatefulSetAllocatorSuite.scala} (98%) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala index 2e0d4fa7ca00b..cc081202cf89a 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala @@ -26,9 +26,8 @@ import org.apache.spark.resource.ResourceProfile * :: DeveloperApi :: * A abstract interface for allowing different types of pods allocation. * - * The internal Spark implementations are [[StatefulsetPodsAllocator]] - * and [[ExecutorPodsAllocator]]. This may be useful for folks integrating with custom schedulers - * such as Volcano, Yunikorn, etc. + * The internal Spark implementations are [[StatefulSetPodsAllocator]] + * and [[ExecutorPodsAllocator]]. This may be useful for folks integrating with custom schedulers. * * This API may change or be removed at anytime. * diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 9497349569efc..10ea3a8cb0e46 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -137,7 +137,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit snapshotsStore: ExecutorPodsSnapshotsStore) = { val executorPodsAllocatorName = sc.conf.get(KUBERNETES_ALLOCATION_PODS_ALLOCATOR) match { case "statefulset" => - classOf[StatefulsetPodsAllocator].getName + classOf[StatefulSetPodsAllocator].getName case "direct" => classOf[ExecutorPodsAllocator].getName case fullClass => diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetPodsAllocator.scala similarity index 99% rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetPodsAllocator.scala rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetPodsAllocator.scala index 0d00d9678048e..294ee70168b23 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetPodsAllocator.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetPodsAllocator.scala @@ -34,7 +34,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.resource.ResourceProfile import org.apache.spark.util.{Clock, Utils} -class StatefulsetPodsAllocator( +class StatefulSetPodsAllocator( conf: SparkConf, secMgr: SecurityManager, executorBuilder: KubernetesExecutorBuilder, diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala index ae1477e51bdf6..2b6bfe851dbd3 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala @@ -47,8 +47,8 @@ class KubernetesClusterManagerSuite extends SparkFunSuite with BeforeAndAfter { test("constructing a AbstractPodsAllocator works") { val validConfigs = List("statefulset", "direct", - "org.apache.spark.scheduler.cluster.k8s.StatefulsetPodsAllocator", - "org.apache.spark.scheduler.cluster.k8s.ExecutorPodsAllocator") + classOf[StatefulSetPodsAllocator].getName, + classOf[ExecutorPodsAllocator].getName) validConfigs.foreach { c => val manager = new KubernetesClusterManager() when(sc.conf.get(KUBERNETES_ALLOCATION_PODS_ALLOCATOR)).thenReturn(c) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetAllocatorSuite.scala similarity index 98% rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetAllocatorSuite.scala index 5f8ceb2d3ffc5..748f509e01303 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetAllocatorSuite.scala @@ -80,7 +80,7 @@ class StatefulSetAllocatorSuite extends SparkFunSuite with BeforeAndAfter { @Mock private var driverPodOperations: PodResource[Pod] = _ - private var podsAllocatorUnderTest: StatefulsetPodsAllocator = _ + private var podsAllocatorUnderTest: StatefulSetPodsAllocator = _ private var snapshotsStore: DeterministicExecutorPodsSnapshotsStore = _ @@ -111,7 +111,7 @@ class StatefulSetAllocatorSuite extends SparkFunSuite with BeforeAndAfter { when(executorBuilder.buildFromFeatures(any(classOf[KubernetesExecutorConf]), meq(secMgr), meq(kubernetesClient), any(classOf[ResourceProfile]))).thenAnswer(executorPodAnswer()) snapshotsStore = new DeterministicExecutorPodsSnapshotsStore() - podsAllocatorUnderTest = new StatefulsetPodsAllocator( + podsAllocatorUnderTest = new StatefulSetPodsAllocator( conf, secMgr, executorBuilder, kubernetesClient, snapshotsStore, null) when(schedulerBackend.getExecutorIds).thenReturn(Seq.empty) podsAllocatorUnderTest.start(TEST_SPARK_APP_ID, schedulerBackend)