From f12d6080ad15b21c88074daad2f35519bd23092d Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 3 Feb 2025 16:46:04 -0800 Subject: [PATCH] [SPARK-51075][K8S] Use `container resource` for HPA --- examples/cluster-with-hpa-template.yaml | 5 +++-- .../spark/k8s/operator/SparkClusterResourceSpec.java | 7 ++++--- .../spark/k8s/operator/SparkClusterResourceSpecTest.java | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/cluster-with-hpa-template.yaml b/examples/cluster-with-hpa-template.yaml index cee5b18a..5220690e 100644 --- a/examples/cluster-with-hpa-template.yaml +++ b/examples/cluster-with-hpa-template.yaml @@ -39,9 +39,10 @@ spec: memory: "3Gi" horizontalPodAutoscalerSpec: metrics: - - type: Resource - resource: + - type: ContainerResource + containerResource: name: cpu + container: worker target: type: Utilization averageUtilization: 10 diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java index 158e0ee7..89ff9a5d 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java @@ -307,14 +307,15 @@ private static Optional buildHorizontalPodAutoscaler( new HorizontalPodAutoscalerSpecBuilder() .addToMetrics( new MetricSpecBuilder() - .withType("Resource") - .withNewResource() + .withType("ContainerResource") + .withNewContainerResource() .withName("cpu") + .withContainer("worker") .withNewTarget() .withType("Utilization") .withAverageUtilization(30) .endTarget() - .endResource() + .endContainerResource() .build()) .withNewBehavior() .withNewScaleUp() diff --git a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java index 3c0bbe73..ace31384 100644 --- a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java +++ b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java @@ -304,5 +304,7 @@ void testHorizontalPodAutoscaler() { assertEquals("4.0.0", hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME)); assertEquals(1, hpa.getSpec().getMinReplicas()); assertEquals(3, hpa.getSpec().getMaxReplicas()); + assertEquals(1, hpa.getSpec().getMetrics().size()); + assertEquals("worker", hpa.getSpec().getMetrics().get(0).getContainerResource().getContainer()); } }