[SPARK-53824] Ban org.apache.commons.collections4 package

### What changes were proposed in this pull request?

This PR aims to ban `org.apache.commons.collections4` package because our use cases can use more simpler Java API `isEmpty` directly.

```java
  Map<String, String> confFromSpec = cluster.getSpec().getSparkConf();
- if (MapUtils.isNotEmpty(confFromSpec)) {
+ if (!confFromSpec.isEmpty()) {
```

### Why are the changes needed?

To reduce the 3rd-party library dependency.

### Does this PR introduce _any_ user-facing change?

No behavior change.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #372 from dongjoon-hyun/SPARK-53824.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: dongjoon-hyun

config/checkstyle/checkstyle.xml

@@ -201,10 +201,11 @@
                 value="Please use the `assertThrows` method to test for exceptions."/>
     </module>
     <module name="IllegalImport">
-      <property name="illegalPkgs" value="org.apache.log4j"/>
-      <property name="illegalPkgs" value="org.apache.commons.lang"/>
       <property name="illegalPkgs" value="org.apache.commons.collections"/>
+      <property name="illegalPkgs" value="org.apache.commons.collections4" />
+      <property name="illegalPkgs" value="org.apache.commons.lang"/>
       <property name="illegalPkgs" value="org.apache.commons.lang3" />
+      <property name="illegalPkgs" value="org.apache.log4j"/>
     </module>
   </module>
 </module>

spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java

@@ -27,7 +27,6 @@
 
 import io.fabric8.kubernetes.client.KubernetesClient;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.collections4.MapUtils;
 
 import org.apache.spark.SparkConf;
 import org.apache.spark.deploy.k8s.KubernetesDriverSpec;
@@ -111,12 +110,12 @@ protected SparkAppDriverConf buildDriverConf(
       SparkApplication app, Map<String, String> confOverrides) {
     ApplicationSpec applicationSpec = app.getSpec();
     SparkConf effectiveSparkConf = new SparkConf();
-    if (MapUtils.isNotEmpty(applicationSpec.getSparkConf())) {
+    if (!applicationSpec.getSparkConf().isEmpty()) {
       for (String confKey : applicationSpec.getSparkConf().keySet()) {
         effectiveSparkConf.set(confKey, applicationSpec.getSparkConf().get(confKey));
       }
     }
-    if (MapUtils.isNotEmpty(confOverrides)) {
+    if (!confOverrides.isEmpty()) {
       for (Map.Entry<String, String> entry : confOverrides.entrySet()) {
         effectiveSparkConf.set(entry.getKey(), entry.getValue());
       }

spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterSubmissionWorker.java

@@ -21,8 +21,6 @@
 
 import java.util.Map;
 
-import org.apache.commons.collections4.MapUtils;
-
 import org.apache.spark.SparkConf;
 
 /** Worker for submitting Spark clusters. */
@@ -39,13 +37,13 @@ public SparkClusterResourceSpec getResourceSpec(
     SparkConf effectiveSparkConf = new SparkConf();
 
     Map<String, String> confFromSpec = cluster.getSpec().getSparkConf();
-    if (MapUtils.isNotEmpty(confFromSpec)) {
+    if (!confFromSpec.isEmpty()) {
       for (Map.Entry<String, String> entry : confFromSpec.entrySet()) {
         effectiveSparkConf.set(entry.getKey(), entry.getValue());
       }
     }
 
-    if (MapUtils.isNotEmpty(confOverrides)) {
+    if (!confOverrides.isEmpty()) {
       for (Map.Entry<String, String> entry : confOverrides.entrySet()) {
         effectiveSparkConf.set(entry.getKey(), entry.getValue());
       }