[SPARK-38537][K8S] Unify Statefulset* to StatefulSet*

### What changes were proposed in this pull request? K8s has [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) and Apache Spark is currently using both `Statefulset*` and `StatefulSet*`. The worst case is a mixed case like `class StatefulSetAllocatorSuite` in `StatefulsetAllocatorSuite.scala`. This PR aims to unify them to a K8s original name, `StatefulSet`. https://github.com/apache/spark/blob/c032928515e74367137c668ce692d8fd53696485/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala#L39 To sum up, two files are renamed and five files are changed. ``` $ git diff master --stat resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala | 5 ++--- resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala | 2 +- resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetPodsAllocator.scala => StatefulSetPodsAllocator.scala} | 2 +- resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala | 4 ++-- resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetAllocatorSuite.scala => StatefulSetAllocatorSuite.scala} | 4 ++-- 5 files changed, 8 insertions(+), 9 deletions(-) ``` ### Why are the changes needed? To be consistent not only inside Apache Spark, but also with K8s. ### Does this PR introduce _any_ user-facing change? No. This is a new code in Apache Spark 3.3. ### How was this patch tested? Pass the CIs. Closes apache#35827 from dongjoon-hyun/SPARK-38537. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
mantov5 · Mar 13, 2022 · 786a70e · 786a70e
1 parent 6b64e5d
commit 786a70e
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 9 deletions.
diff --git a/...es/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala b/...es/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala
@@ -26,9 +26,8 @@ import org.apache.spark.resource.ResourceProfile
  * :: DeveloperApi ::
  * A abstract interface for allowing different types of pods allocation.
  *
- * The internal Spark implementations are [[StatefulsetPodsAllocator]]
- * and [[ExecutorPodsAllocator]]. This may be useful for folks integrating with custom schedulers
- * such as Volcano, Yunikorn, etc.
+ * The internal Spark implementations are [[StatefulSetPodsAllocator]]
+ * and [[ExecutorPodsAllocator]]. This may be useful for folks integrating with custom schedulers.
  *
  * This API may change or be removed at anytime.
  *

diff --git a/...core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/...core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
@@ -137,7 +137,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
       snapshotsStore: ExecutorPodsSnapshotsStore) = {
     val executorPodsAllocatorName = sc.conf.get(KUBERNETES_ALLOCATION_PODS_ALLOCATOR) match {
       case "statefulset" =>
-        classOf[StatefulsetPodsAllocator].getName
+        classOf[StatefulSetPodsAllocator].getName
       case "direct" =>
         classOf[ExecutorPodsAllocator].getName
       case fullClass =>

diff --git a/...luster/k8s/StatefulsetPodsAllocator.scala → ...luster/k8s/StatefulSetPodsAllocator.scala b/...luster/k8s/StatefulsetPodsAllocator.scala → ...luster/k8s/StatefulSetPodsAllocator.scala
@@ -34,7 +34,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.resource.ResourceProfile
 import org.apache.spark.util.{Clock, Utils}
 
-class StatefulsetPodsAllocator(
+class StatefulSetPodsAllocator(
     conf: SparkConf,
     secMgr: SecurityManager,
     executorBuilder: KubernetesExecutorBuilder,

diff --git a/...src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala b/...src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala
@@ -47,8 +47,8 @@ class KubernetesClusterManagerSuite extends SparkFunSuite with BeforeAndAfter {
 
   test("constructing a AbstractPodsAllocator works") {
     val validConfigs = List("statefulset", "direct",
-      "org.apache.spark.scheduler.cluster.k8s.StatefulsetPodsAllocator",
-      "org.apache.spark.scheduler.cluster.k8s.ExecutorPodsAllocator")
+      classOf[StatefulSetPodsAllocator].getName,
+      classOf[ExecutorPodsAllocator].getName)
     validConfigs.foreach { c =>
       val manager = new KubernetesClusterManager()
         when(sc.conf.get(KUBERNETES_ALLOCATION_PODS_ALLOCATOR)).thenReturn(c)

diff --git a/...uster/k8s/StatefulsetAllocatorSuite.scala → ...uster/k8s/StatefulSetAllocatorSuite.scala b/...uster/k8s/StatefulsetAllocatorSuite.scala → ...uster/k8s/StatefulSetAllocatorSuite.scala
@@ -80,7 +80,7 @@ class StatefulSetAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
   @Mock
   private var driverPodOperations: PodResource[Pod] = _
 
-  private var podsAllocatorUnderTest: StatefulsetPodsAllocator = _
+  private var podsAllocatorUnderTest: StatefulSetPodsAllocator = _
 
   private var snapshotsStore: DeterministicExecutorPodsSnapshotsStore = _
 
@@ -111,7 +111,7 @@ class StatefulSetAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
     when(executorBuilder.buildFromFeatures(any(classOf[KubernetesExecutorConf]), meq(secMgr),
       meq(kubernetesClient), any(classOf[ResourceProfile]))).thenAnswer(executorPodAnswer())
     snapshotsStore = new DeterministicExecutorPodsSnapshotsStore()
-    podsAllocatorUnderTest = new StatefulsetPodsAllocator(
+    podsAllocatorUnderTest = new StatefulSetPodsAllocator(
       conf, secMgr, executorBuilder, kubernetesClient, snapshotsStore, null)
     when(schedulerBackend.getExecutorIds).thenReturn(Seq.empty)
     podsAllocatorUnderTest.start(TEST_SPARK_APP_ID, schedulerBackend)