diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java index 10b218db..ecca1af9 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/BaseResource.java @@ -27,6 +27,16 @@ import org.apache.spark.k8s.operator.status.BaseState; import org.apache.spark.k8s.operator.status.BaseStatus; +/** + * Base class for Spark Kubernetes Operator's custom resources like SparkApplication and + * SparkCluster. + * + * @param The type of the state. + * @param The type of the attempt summary. + * @param The type of the state. + * @param The type of the spec. + * @param The type of the status. + */ public class BaseResource< S, AS extends BaseAttemptSummary, diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java index c300041a..2c074f11 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/Constants.java @@ -19,6 +19,7 @@ package org.apache.spark.k8s.operator; +/** Constants used in the Spark Kubernetes Operator. */ @SuppressWarnings("PMD.DataClass") public class Constants { public static final String API_GROUP = "spark.apache.org"; diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java index 3717ee71..a95241d1 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplication.java @@ -32,6 +32,7 @@ import org.apache.spark.k8s.operator.status.ApplicationStateSummary; import org.apache.spark.k8s.operator.status.ApplicationStatus; +/** SparkApplication is the Custom Resource Definition (CRD) for a Spark application. */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonDeserialize() @Group(Constants.API_GROUP) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java index 9da21e6b..db54f6bd 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkApplicationList.java @@ -22,5 +22,6 @@ import io.fabric8.kubernetes.api.model.DefaultKubernetesResourceList; import lombok.NoArgsConstructor; +/** List of SparkApplications. */ @NoArgsConstructor public class SparkApplicationList extends DefaultKubernetesResourceList {} diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java index 4f0c6898..2d0ed8d0 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkCluster.java @@ -32,6 +32,7 @@ import org.apache.spark.k8s.operator.status.ClusterStateSummary; import org.apache.spark.k8s.operator.status.ClusterStatus; +/** SparkCluster is the Custom Resource Definition (CRD) for a Spark cluster. */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonDeserialize() @Group(Constants.API_GROUP) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java index 035b78c1..78883f50 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/SparkClusterList.java @@ -22,5 +22,6 @@ import io.fabric8.kubernetes.api.model.DefaultKubernetesResourceList; import lombok.NoArgsConstructor; +/** List of SparkClusters. */ @NoArgsConstructor public class SparkClusterList extends DefaultKubernetesResourceList {} diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java index 9d7722f5..f8fd531c 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationSpec.java @@ -31,6 +31,7 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +/** Spec for a Spark application. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java index 2bf96326..b2bbcc7c 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTimeoutConfig.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Application timeout configuration. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java index f1fadf79..8e2bef0d 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ApplicationTolerations.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Toleration settings for a Spark application. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java index 7fd7e3a5..35c72269 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseApplicationTemplateSpec.java @@ -27,6 +27,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Base class for application template specs for driver and executors. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java index 3522ad34..bfef359f 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java @@ -28,6 +28,7 @@ import org.apache.spark.k8s.operator.diff.Diffable; +/** Base spec for all custom resources. */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java index 408f8d62..12b06448 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterSpec.java @@ -28,6 +28,7 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +/** Spec for a Spark cluster. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java index d7e546bf..e59cd24e 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ClusterTolerations.java @@ -25,6 +25,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Toleration settings for a Spark cluster. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java index 8cce3fde..bdf00a14 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/ConfigMapSpec.java @@ -28,6 +28,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Spec for a ConfigMap. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java index 9712a652..5ddccf1d 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/DriverServiceIngressSpec.java @@ -28,6 +28,7 @@ import lombok.Builder; import lombok.Data; +/** Spec for a driver service ingress. */ @Data @Builder @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java index c04a2be9..a459a9ab 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/MasterSpec.java @@ -28,6 +28,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Spec for a Spark master. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java index 2c19cd23..f724b43e 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RestartConfig.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Restart configuration for a Spark application. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java index 819cca8c..11abbe90 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/RuntimeVersions.java @@ -27,6 +27,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Runtime versions for a Spark application or cluster. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java index 5adca3ec..914ccca6 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerInstanceConfig.java @@ -25,6 +25,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Config tolerations of worker instances for the cluster. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java index 78b86e3c..7094195b 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/WorkerSpec.java @@ -29,6 +29,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** Spec for a Spark worker. */ @Data @NoArgsConstructor @AllArgsConstructor diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java index a9019358..2d3c6a70 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationAttemptSummary.java @@ -27,6 +27,7 @@ import lombok.Getter; import lombok.ToString; +/** Summary of a Spark application attempt. */ @Getter @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java index 33614f83..b4dcdfb2 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationState.java @@ -32,6 +32,7 @@ import lombok.Setter; import lombok.ToString; +/** Represents the state of a Spark application. */ @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java index 28134409..87bc31f2 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ApplicationStatus.java @@ -34,6 +34,7 @@ import org.apache.spark.k8s.operator.spec.RestartConfig; import org.apache.spark.k8s.operator.spec.RestartPolicy; +/** Represents the status of a Spark application. */ @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java index 84c8b766..1222820e 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/AttemptInfo.java @@ -28,6 +28,7 @@ import lombok.NoArgsConstructor; import lombok.ToString; +/** Information about an attempt. */ @NoArgsConstructor @AllArgsConstructor @Builder diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java index 8078529e..ad7673ee 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseAttemptSummary.java @@ -26,6 +26,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; +/** Base class for attempt summaries. */ @EqualsAndHashCode @ToString @Getter diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java index e73cd355..82d8623d 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseState.java @@ -26,6 +26,11 @@ import lombok.AllArgsConstructor; import lombok.Data; +/** + * Base class for state information. + * + * @param The type of the current state summary. + */ @Data @AllArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java index ab1d6425..455e76ff 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/BaseStatus.java @@ -29,6 +29,13 @@ import lombok.Getter; import lombok.ToString; +/** + * Base class for status information. + * + * @param The type of the state summary. + * @param The type of the state. + * @param The type of the attempt summary. + */ @ToString @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java index e0fb7cd4..4a79c88d 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterAttemptSummary.java @@ -27,6 +27,7 @@ import lombok.Getter; import lombok.ToString; +/** Summary of a Spark cluster attempt. */ @Getter @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java index 2d6a832a..1ccd8e6b 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterState.java @@ -29,6 +29,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; +/** Represents the state of a Spark cluster. */ @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java index 73c92077..5cc97b3f 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/status/ClusterStatus.java @@ -27,6 +27,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; +/** Represents the status of a Spark cluster. */ @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java index e9ed22e7..5351594f 100644 --- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java +++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/utils/ModelUtils.java @@ -38,6 +38,7 @@ import org.apache.spark.k8s.operator.SparkApplication; import org.apache.spark.k8s.operator.spec.ApplicationSpec; +/** Utility class for Kubernetes model operations. */ public final class ModelUtils { public static final ObjectMapper objectMapper = new ObjectMapper(); diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java index 00619d7a..4eaf2106 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java @@ -57,7 +57,7 @@ /** * Entry point for Spark Operator. Bootstrap the operator app by starting watch and reconciler for * SparkApps, starting watch for hot property loading, if enabled, and starting metrics server with - * sentinel monitor if enabled + * sentinel monitor if enabled. */ @Slf4j public class SparkOperator { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java index 766c1c54..5e103dfc 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/client/KubernetesClientFactory.java @@ -27,7 +27,7 @@ import okhttp3.Interceptor; import okhttp3.OkHttpClient; -/** Build Kubernetes Client with metrics configured */ +/** Factory for building Kubernetes clients with metrics configured. */ public final class KubernetesClientFactory { private KubernetesClientFactory() {} diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java index e0726c4f..4f7b03de 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/ConfigOption.java @@ -31,7 +31,11 @@ import org.apache.spark.k8s.operator.utils.ModelUtils; -/** Config options for Spark Operator. Supports primitive and serialized JSON */ +/** + * Config options for Spark Operator. Supports primitive and serialized JSON. + * + * @param The type of the config option's value. + */ @RequiredArgsConstructor @AllArgsConstructor @EqualsAndHashCode diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/context/BaseContext.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/context/BaseContext.java index 98bddb5a..82b23478 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/context/BaseContext.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/context/BaseContext.java @@ -23,6 +23,11 @@ import org.apache.spark.k8s.operator.BaseResource; +/** + * Base class for context objects. + * + * @param The type of the custom resource. + */ public abstract class BaseContext> { public abstract CR getResource(); diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/listeners/BaseStatusListener.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/listeners/BaseStatusListener.java index 17076b8b..f3d12c53 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/listeners/BaseStatusListener.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/listeners/BaseStatusListener.java @@ -21,7 +21,12 @@ import org.apache.spark.k8s.operator.BaseResource; import org.apache.spark.k8s.operator.status.BaseStatus; -/** Custom listeners, if added, would be listening to resource status change */ +/** + * Custom listeners, if added, would be listening to resource status change. + * + * @param The type of the status. + * @param The type of the custom resource. + */ public abstract class BaseStatusListener< STATUS extends BaseStatus, CR extends BaseResource> { public abstract void listenStatus(CR resource, STATUS prevStatus, STATUS updatedStatus); diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/JVMMetricSet.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/JVMMetricSet.java index a87d5bda..8a4a27ae 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/JVMMetricSet.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/JVMMetricSet.java @@ -31,6 +31,7 @@ import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; +/** A MetricSet for JVM metrics. */ public class JVMMetricSet implements MetricSet { /** Refer codahale FileDescriptorRatioGauge for the definition */ diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsService.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsService.java index c97818ea..ea5e6bdc 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsService.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsService.java @@ -28,7 +28,7 @@ import com.sun.net.httpserver.HttpServer; import lombok.extern.slf4j.Slf4j; -/** Start Http service at endpoint /prometheus, exposing operator metrics */ +/** Start Http service at endpoint /prometheus, exposing operator metrics. */ @Slf4j public class MetricsService { HttpServer server; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystem.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystem.java index 4194144c..1b212dc0 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystem.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystem.java @@ -36,6 +36,7 @@ import org.apache.spark.metrics.sink.Sink; import org.apache.spark.metrics.source.Source; +/** Manages the metrics system for the Spark Operator. */ @Slf4j public class MetricsSystem { private final AtomicBoolean running = new AtomicBoolean(false); diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystemFactory.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystemFactory.java index f93c1cca..0eb1de9d 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystemFactory.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/MetricsSystemFactory.java @@ -28,7 +28,7 @@ import org.apache.spark.k8s.operator.config.SparkOperatorConfManager; -/** Factory for MetricsSystem */ +/** Factory for MetricsSystem. */ public final class MetricsSystemFactory { public static final String METRIC_PREFIX = "spark.metrics.conf.operator."; public static final String SINK = "sink."; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/PrometheusPullModelHandler.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/PrometheusPullModelHandler.java index fc935c5a..d5a72a8c 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/PrometheusPullModelHandler.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/PrometheusPullModelHandler.java @@ -37,7 +37,7 @@ import org.apache.spark.metrics.sink.PrometheusServlet; -/** Serves as simple Prometheus sink (pull model), presenting metrics snapshot as HttpHandler */ +/** Serves as simple Prometheus sink (pull model), presenting metrics snapshot as HttpHandler. */ @Slf4j public class PrometheusPullModelHandler extends PrometheusServlet implements HttpHandler { private static final String EMPTY_RECORD_VALUE = "[]"; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/healthcheck/SentinelManager.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/healthcheck/SentinelManager.java index 05255555..ec5e32e6 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/healthcheck/SentinelManager.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/healthcheck/SentinelManager.java @@ -43,7 +43,7 @@ import org.apache.spark.k8s.operator.utils.Utils; /** - * Sentinel manager monitors dedicated sentinel resources to make sure the operator is healthy + * Sentinel manager monitors dedicated sentinel resources to make sure the operator is healthy. * * @param custom resource type */ diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/KubernetesMetricsInterceptor.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/KubernetesMetricsInterceptor.java index 58240790..aab1c0e8 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/KubernetesMetricsInterceptor.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/KubernetesMetricsInterceptor.java @@ -40,6 +40,7 @@ import org.apache.spark.metrics.source.Source; +/** Interceptor for Kubernetes client to collect metrics. */ @Slf4j public class KubernetesMetricsInterceptor implements Interceptor, Source { final MetricRegistry metricRegistry; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java index 530848e8..86a34ff5 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java @@ -48,6 +48,7 @@ import org.apache.spark.util.Clock; import org.apache.spark.util.SystemClock; +/** Metrics for the Java Operator SDK. */ @Slf4j public class OperatorJosdkMetrics implements Source, Metrics { public static final String FINISHED = "finished"; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJvmSource.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJvmSource.java index 8cf7df07..ccbb4785 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJvmSource.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJvmSource.java @@ -24,6 +24,7 @@ import org.apache.spark.k8s.operator.metrics.JVMMetricSet; import org.apache.spark.metrics.source.Source; +/** Source for JVM metrics. */ public class OperatorJvmSource implements Source { @Override public String sourceName() { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/HealthProbe.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/HealthProbe.java index 4df568aa..5b33a9ed 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/HealthProbe.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/HealthProbe.java @@ -43,6 +43,7 @@ import org.apache.spark.k8s.operator.metrics.healthcheck.SentinelManager; +/** Health probe for the operator. */ @Getter @Slf4j @RequiredArgsConstructor diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ProbeService.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ProbeService.java index d562e7b9..2446be63 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ProbeService.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ProbeService.java @@ -33,6 +33,7 @@ import org.apache.spark.k8s.operator.metrics.healthcheck.SentinelManager; +/** Service for health and readiness probes. */ @Slf4j public class ProbeService { public static final String HEALTHZ = "/healthz"; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ReadinessProbe.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ReadinessProbe.java index 19c0cdf4..d66d9bf4 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ReadinessProbe.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/probe/ReadinessProbe.java @@ -32,6 +32,7 @@ import io.javaoperatorsdk.operator.Operator; import lombok.extern.slf4j.Slf4j; +/** Readiness probe for the operator. */ @Slf4j public class ReadinessProbe implements HttpHandler { private final List operators; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkAppResourceSpecFactory.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkAppResourceSpecFactory.java index 35871fa1..98ba0fbf 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkAppResourceSpecFactory.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkAppResourceSpecFactory.java @@ -44,6 +44,7 @@ import org.apache.spark.k8s.operator.decorators.DriverDecorator; import org.apache.spark.k8s.operator.utils.ModelUtils; +/** Factory for creating SparkAppResourceSpec objects. */ @Slf4j public final class SparkAppResourceSpecFactory { @@ -74,7 +75,6 @@ private static Map overrideDependencyConf(final SparkApplication if (app.getSpec().getSparkConf().containsKey("spark.app.name")) { confOverrides.put("spark.app.name", app.getMetadata().getName()); } - // FIXME: avoid this file flushing confOverrides.putAll(getOrCreateLocalFileForDriverSpec(app, confOverrides)); confOverrides.putAll(getOrCreateLocalFileForExecutorSpec(app, confOverrides)); return confOverrides; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkClusterResourceSpecFactory.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkClusterResourceSpecFactory.java index 8518019f..45d63722 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkClusterResourceSpecFactory.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/SparkClusterResourceSpecFactory.java @@ -29,6 +29,7 @@ import org.apache.spark.k8s.operator.SparkClusterSubmissionWorker; import org.apache.spark.k8s.operator.decorators.ClusterDecorator; +/** Factory for creating SparkClusterResourceSpec objects. */ @Slf4j public final class SparkClusterResourceSpecFactory { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverReadyObserver.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverReadyObserver.java index 92f9d7c9..7a506f12 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverReadyObserver.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverReadyObserver.java @@ -31,7 +31,7 @@ import org.apache.spark.k8s.operator.status.ApplicationStatus; import org.apache.spark.k8s.operator.utils.PodUtils; -/** Observes whether driver is ready */ +/** Observes whether driver is ready. */ public class AppDriverReadyObserver extends BaseAppDriverObserver { @Override public Optional observe( diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverStartObserver.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverStartObserver.java index 656c81db..66c92576 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverStartObserver.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverStartObserver.java @@ -31,6 +31,7 @@ import org.apache.spark.k8s.operator.status.ApplicationStatus; import org.apache.spark.k8s.operator.utils.PodUtils; +/** Observes whether the driver pod has started. */ public class AppDriverStartObserver extends BaseAppDriverObserver { @Override public Optional observe( diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverTimeoutObserver.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverTimeoutObserver.java index b971fe9f..6b26c028 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverTimeoutObserver.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/AppDriverTimeoutObserver.java @@ -31,7 +31,7 @@ import org.apache.spark.k8s.operator.status.ApplicationStatus; import org.apache.spark.k8s.operator.utils.SparkAppStatusUtils; -/** Observes driver status and time-out as configured in app spec */ +/** Observes driver status and time-out as configured in app spec. */ public class AppDriverTimeoutObserver extends BaseAppDriverObserver { /** diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseAppDriverObserver.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseAppDriverObserver.java index 6548cce1..dd8ba643 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseAppDriverObserver.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseAppDriverObserver.java @@ -40,7 +40,7 @@ import org.apache.spark.k8s.operator.utils.PodPhase; import org.apache.spark.k8s.operator.utils.PodUtils; -/** Observes driver pod status and update Application status as needed */ +/** Observes driver pod status and update Application status as needed. */ @Slf4j public abstract class BaseAppDriverObserver extends BaseSecondaryResourceObserver< diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseSecondaryResourceObserver.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseSecondaryResourceObserver.java index 3c91ead6..1bb3a963 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseSecondaryResourceObserver.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/observers/BaseSecondaryResourceObserver.java @@ -31,6 +31,13 @@ /** * Observe given secondary resource, return state to be updated if applicable. These observers only * observe secondary resource status and update the status of owner, SparkApplication, if needed. + * + * @param The type of the state summary. + * @param The type of the attempt summary. + * @param The type of the state. + * @param The type of the spec. + * @param The type of the status. + * @param The type of the secondary resource. */ public abstract class BaseSecondaryResourceObserver< S, diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppCleanUpStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppCleanUpStep.java index f75967a1..3f01134b 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppCleanUpStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppCleanUpStep.java @@ -48,7 +48,7 @@ /** * Cleanup all secondary resources when application is deleted, or at the end of each attempt. - * Update Application status to indicate whether another attempt would be made + * Update Application status to indicate whether another attempt would be made. */ @AllArgsConstructor @NoArgsConstructor diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppInitStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppInitStep.java index 583d93b9..1e9f2cf7 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppInitStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppInitStep.java @@ -44,7 +44,7 @@ import org.apache.spark.k8s.operator.utils.ReconcilerUtils; import org.apache.spark.k8s.operator.utils.SparkAppStatusRecorder; -/** Request all driver and its resources when starting an attempt */ +/** Request all driver and its resources when starting an attempt. */ @Slf4j public class AppInitStep extends AppReconcileStep { @Override diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppReconcileStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppReconcileStep.java index 91542ce3..6c414d46 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppReconcileStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppReconcileStep.java @@ -37,7 +37,7 @@ import org.apache.spark.k8s.operator.status.ApplicationStatus; import org.apache.spark.k8s.operator.utils.SparkAppStatusRecorder; -/** Basic reconcile step for application */ +/** Basic reconcile step for application. */ public abstract class AppReconcileStep { public abstract ReconcileProgress reconcile( SparkAppContext context, SparkAppStatusRecorder statusRecorder); diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppResourceObserveStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppResourceObserveStep.java index 0db6dfdf..83333f41 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppResourceObserveStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppResourceObserveStep.java @@ -28,7 +28,7 @@ import org.apache.spark.k8s.operator.reconciler.observers.BaseAppDriverObserver; import org.apache.spark.k8s.operator.utils.SparkAppStatusRecorder; -/** Observes secondary resource and update app status if needed */ +/** Observes secondary resource and update app status if needed. */ @RequiredArgsConstructor public class AppResourceObserveStep extends AppReconcileStep { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppRunningStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppRunningStep.java index 774cf616..524c1cf3 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppRunningStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppRunningStep.java @@ -36,7 +36,7 @@ import org.apache.spark.k8s.operator.utils.PodUtils; import org.apache.spark.k8s.operator.utils.SparkAppStatusRecorder; -/** Observe whether app acquires enough executors as configured in spec */ +/** Observe whether app acquires enough executors as configured in spec. */ public class AppRunningStep extends AppReconcileStep { @Override public ReconcileProgress reconcile( diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppUnknownStateStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppUnknownStateStep.java index 715c235a..ef113749 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppUnknownStateStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/AppUnknownStateStep.java @@ -30,7 +30,7 @@ import org.apache.spark.k8s.operator.status.ApplicationStateSummary; import org.apache.spark.k8s.operator.utils.SparkAppStatusRecorder; -/** Abnormal state handler */ +/** Abnormal state handler. */ public class AppUnknownStateStep extends AppReconcileStep { @Override public ReconcileProgress reconcile( diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterInitStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterInitStep.java index 29c8cd0c..db2fc3df 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterInitStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterInitStep.java @@ -40,7 +40,7 @@ import org.apache.spark.k8s.operator.status.ClusterStatus; import org.apache.spark.k8s.operator.utils.SparkClusterStatusRecorder; -/** Request cluster master and its resources when starting an attempt */ +/** Request cluster master and its resources when starting an attempt. */ @Slf4j public class ClusterInitStep extends ClusterReconcileStep { @Override diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterReconcileStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterReconcileStep.java index 444c20f0..0a3199c0 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterReconcileStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterReconcileStep.java @@ -27,7 +27,7 @@ import org.apache.spark.k8s.operator.status.ClusterStatus; import org.apache.spark.k8s.operator.utils.SparkClusterStatusRecorder; -/** Basic reconcile step for cluster */ +/** Basic reconcile step for cluster. */ public abstract class ClusterReconcileStep { public abstract ReconcileProgress reconcile( SparkClusterContext context, SparkClusterStatusRecorder statusRecorder); diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterUnknownStateStep.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterUnknownStateStep.java index 87eb8eb0..6e70cd82 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterUnknownStateStep.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/reconciler/reconcilesteps/ClusterUnknownStateStep.java @@ -26,7 +26,7 @@ import org.apache.spark.k8s.operator.status.ClusterStateSummary; import org.apache.spark.k8s.operator.utils.SparkClusterStatusRecorder; -/** Abnormal state handler */ +/** Abnormal state handler for clusters. */ public class ClusterUnknownStateStep extends ClusterReconcileStep { @Override public ReconcileProgress reconcile( diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ClassLoadingUtils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ClassLoadingUtils.java index 8d49fb40..f6e98c04 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ClassLoadingUtils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ClassLoadingUtils.java @@ -29,6 +29,7 @@ import org.apache.spark.k8s.operator.listeners.BaseStatusListener; +/** Utility class for loading classes. */ @Slf4j public final class ClassLoadingUtils { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/LoggingUtils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/LoggingUtils.java index 1958edb6..bbe30f0b 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/LoggingUtils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/LoggingUtils.java @@ -28,6 +28,7 @@ import org.apache.spark.k8s.operator.SparkApplication; import org.apache.spark.k8s.operator.status.ApplicationAttemptSummary; +/** Utility class for logging. */ public class LoggingUtils { public static final class TrackedMDC { public static final String AppAttemptIdKey = "resource.app.attemptId"; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/PodUtils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/PodUtils.java index 65893590..81a4cf32 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/PodUtils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/PodUtils.java @@ -28,6 +28,7 @@ import org.apache.spark.k8s.operator.spec.ApplicationSpec; +/** Utility class for Pod operations. */ public final class PodUtils { public static final String POD_READY_CONDITION_TYPE = "ready"; diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ReconcilerUtils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ReconcilerUtils.java index 2d7bd282..3a901f03 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ReconcilerUtils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/ReconcilerUtils.java @@ -45,6 +45,7 @@ import org.apache.spark.k8s.operator.BaseResource; import org.apache.spark.k8s.operator.reconciler.ReconcileProgress; +/** Utility class for reconciler operations. */ @Slf4j public final class ReconcilerUtils { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusRecorder.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusRecorder.java index 4ca31909..b03d6e3d 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusRecorder.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusRecorder.java @@ -27,6 +27,7 @@ import org.apache.spark.k8s.operator.status.ApplicationState; import org.apache.spark.k8s.operator.status.ApplicationStatus; +/** Records the status of a Spark application. */ public class SparkAppStatusRecorder extends StatusRecorder { public SparkAppStatusRecorder(List statusListeners) { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusUtils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusUtils.java index 56b3e076..ecbe3e20 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusUtils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkAppStatusUtils.java @@ -24,7 +24,7 @@ import org.apache.spark.k8s.operator.status.ApplicationState; import org.apache.spark.k8s.operator.status.ApplicationStateSummary; -/** Handy utils for create and manage Application Status */ +/** Handy utils for create and manage Application Status. */ public final class SparkAppStatusUtils { private SparkAppStatusUtils() {} diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkClusterStatusRecorder.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkClusterStatusRecorder.java index 2495d052..82808c34 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkClusterStatusRecorder.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkClusterStatusRecorder.java @@ -27,6 +27,7 @@ import org.apache.spark.k8s.operator.status.ClusterState; import org.apache.spark.k8s.operator.status.ClusterStatus; +/** Records the status of a Spark cluster. */ public class SparkClusterStatusRecorder extends StatusRecorder { public SparkClusterStatusRecorder(List statusListeners) { diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkExceptionUtils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkExceptionUtils.java index 8221b4bb..dabac325 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkExceptionUtils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/SparkExceptionUtils.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +/** Utility class for Spark exceptions. */ public final class SparkExceptionUtils { private SparkExceptionUtils() {} diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/StatusRecorder.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/StatusRecorder.java index 214f88bf..7a4f114c 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/StatusRecorder.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/StatusRecorder.java @@ -42,6 +42,10 @@ /** * Enables additional (extendable) observers for Spark App status. Cache and version locking might * be removed in future version as batch app does not expect spec change after submitted. + * + * @param The type of the status. + * @param The type of the custom resource. + * @param The type of the status listener. */ @Slf4j public class StatusRecorder< diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/Utils.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/Utils.java index 28c1cf6b..c6b2b7fe 100644 --- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/Utils.java +++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/utils/Utils.java @@ -46,6 +46,7 @@ import org.apache.spark.k8s.operator.listeners.SparkAppStatusListener; import org.apache.spark.k8s.operator.listeners.SparkClusterStatusListener; +/** Utility class for common operations. */ public final class Utils { private Utils() {} diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppDriverConf.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppDriverConf.java index 30ed0418..cbe3f43e 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppDriverConf.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppDriverConf.java @@ -28,6 +28,7 @@ import org.apache.spark.deploy.k8s.submit.KubernetesClientUtils; import org.apache.spark.deploy.k8s.submit.MainAppResource; +/** Spark application driver configuration. */ public final class SparkAppDriverConf extends KubernetesDriverConf { private SparkAppDriverConf( SparkConf sparkConf, diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java index 8fe2bba8..e21647dd 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java @@ -57,7 +57,7 @@ *
  • Converts scala types to Java for easier reference from operator * * - *

    This is not thread safe and not expected to be shared among reconciler threads + *

    This is not thread safe and not expected to be shared among reconciler threads. */ public class SparkAppResourceSpec { @Getter private final Pod configuredPod; diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java index d204c5d7..acccb8cd 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java @@ -44,7 +44,7 @@ /** * Similar to org.apache.spark.deploy.k8s.submit.KubernetesClientApplication. This reads args from - * SparkApplication instead of starting separate spark-submit process + * SparkApplication instead of starting separate spark-submit process. */ public class SparkAppSubmissionWorker { // Default length limit for generated app id. Generated id is used as resource-prefix when diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java index f35cd68f..834739c3 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java @@ -46,7 +46,7 @@ import org.apache.spark.k8s.operator.spec.MasterSpec; import org.apache.spark.k8s.operator.spec.WorkerSpec; -/** Spark Cluster Resource Spec: Master Service, Master StatefulSet, Worker StatefulSet */ +/** Spark Cluster Resource Spec: Master Service, Master StatefulSet, Worker StatefulSet. */ public class SparkClusterResourceSpec { @Getter private final Service masterService; @Getter private final Service workerService; diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterSubmissionWorker.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterSubmissionWorker.java index 5cf77cfc..0193f623 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterSubmissionWorker.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterSubmissionWorker.java @@ -25,6 +25,7 @@ import org.apache.spark.SparkConf; +/** Worker for submitting Spark clusters. */ public class SparkClusterSubmissionWorker { public SparkClusterResourceSpec getResourceSpec( SparkCluster cluster, Map confOverrides) { diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/ConfigMapSpecUtils.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/ConfigMapSpecUtils.java index fed1d1b6..8c01ef29 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/ConfigMapSpecUtils.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/ConfigMapSpecUtils.java @@ -28,6 +28,7 @@ import org.apache.spark.k8s.operator.spec.ConfigMapSpec; +/** Utility class for ConfigMap specifications. */ public final class ConfigMapSpecUtils { private ConfigMapSpecUtils() {} diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/DriverServiceIngressUtils.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/DriverServiceIngressUtils.java index 48d16c98..b794b963 100644 --- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/DriverServiceIngressUtils.java +++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/utils/DriverServiceIngressUtils.java @@ -35,6 +35,7 @@ import org.apache.spark.k8s.operator.spec.DriverServiceIngressSpec; +/** Utility class for driver service ingress. */ public final class DriverServiceIngressUtils { private DriverServiceIngressUtils() {}