Support Spark 4.1 by removing mention to StreamExecution class (#10228)

* Do not rely on StreamExecution class

* re-enable Spark 4.1 in CI

Author: aboitreaud

dd-java-agent/instrumentation/spark/spark_2.13/build.gradle

@@ -16,7 +16,6 @@ muzzle {
     group = "org.apache.spark"
     module = "spark-sql_$scalaVersion"
     versions = "[4.0.0,)"
-    skipVersions += "4.1.0" // FIXME: Temporary skip `4.1.0` as we need more time to support it.
     javaVersion = 17
   }
 }

dd-java-agent/instrumentation/spark/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java

@@ -58,7 +58,6 @@
 import org.apache.spark.sql.execution.SQLExecution;
 import org.apache.spark.sql.execution.SparkPlanInfo;
 import org.apache.spark.sql.execution.metric.SQLMetricInfo;
-import org.apache.spark.sql.execution.streaming.StreamExecution;
 import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd;
 import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart;
 import org.apache.spark.sql.streaming.SourceProgress;
@@ -1264,7 +1263,7 @@ private static String getStreamingBatchKey(Properties properties) {
       return null;
     }
 
-    Object queryId = properties.get(StreamExecution.QUERY_ID_KEY());
+    Object queryId = properties.get("sql.streaming.queryId");
     Object batchId = properties.get("streaming.sql.batchId");
 
     if (queryId == null || batchId == null) {