[SPARK-54753][SQL][4.0] Fix memory leak of ArtifactManager

### What changes were proposed in this pull request?

backport the fix in trunk branch to branch-4.0
5b5ff6d

dongjoon-hyun pranavdev022 hvanhovell vicennial HyukjinKwon cc Ngone51 LuciferYang

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

### Was this patch authored or co-authored using generative AI tooling?

Closes #53653 from scottme/SPARK-54753-4.0.

Authored-by: xihuan_mstr <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: xihuan_mstr

sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala

@@ -234,6 +234,8 @@ object CheckConnectJvmClientCompatibility {
         "org.apache.spark.sql.artifact.ArtifactManager$"),
       ProblemFilters.exclude[MissingClassProblem](
         "org.apache.spark.sql.artifact.ArtifactManager$SparkContextResourceType$"),
+      ProblemFilters.exclude[MissingClassProblem](
+        "org.apache.spark.sql.artifact.ArtifactManager$StateCleanupRunner"),
 
       // ColumnNode conversions
       ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.sql.SparkSession"),

sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala

@@ -384,8 +384,7 @@ class ArtifactManager(session: SparkSession) extends AutoCloseable with Logging
     artifactPath)
   // Ensure that no reference to `this` is captured/help by the cleanup lambda
   private def getCleanable: Cleaner.Cleanable = cleaner.register(
-    this,
-    () => ArtifactManager.cleanUpGlobalResources(cleanUpStateForGlobalResources)
+    this, new StateCleanupRunner(cleanUpStateForGlobalResources)
   )
   private var cleanable = getCleanable
 
@@ -491,6 +490,12 @@ object ArtifactManager extends Logging {
     }
   }
 
+  private class StateCleanupRunner(cleanupState: ArtifactStateForCleanup) extends Runnable {
+    override def run(): Unit = {
+      ArtifactManager.cleanUpGlobalResources(cleanupState)
+    }
+  }
+
   // Shared cleaner instance
   private val cleaner: Cleaner = Cleaner.create()