feat: support complex spark conf for taskmanager (#2170)

* Support complex spark conf for taskmanager

* Add spark file coding config in release taskmanager conf

Author: tobegit3hub

java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java

@@ -180,11 +180,11 @@ public static void parse() throws IOException, NumberFormatException, ConfigExce
 
         SPARK_DEFAULT_CONF = prop.getProperty("spark.default.conf", "");
         if (!SPARK_DEFAULT_CONF.isEmpty()) {
-            String[] defaultSparkConfs = TaskManagerConfig.SPARK_DEFAULT_CONF.split(",");
+            String[] defaultSparkConfs = TaskManagerConfig.SPARK_DEFAULT_CONF.split(";");
             for (String sparkConfMap: defaultSparkConfs) {
                 if (!sparkConfMap.isEmpty()) {
                     String[] kv = sparkConfMap.split("=");
-                    if (kv.length != 2) {
+                    if (kv.length < 2) {
                         throw new ConfigException("spark.default.conf", String.format("error format of %s", sparkConfMap));
                     } else if (!kv[0].startsWith("spark")) {
                         throw new ConfigException("spark.default.conf", String.format("config key should start with 'spark' but get %s", kv[0]));

java/openmldb-taskmanager/src/main/scala/com/_4paradigm/openmldb/taskmanager/spark/SparkJobManager.scala

@@ -94,13 +94,14 @@ object SparkJobManager {
       launcher.setConf("spark.yarn.maxAppAttempts", TaskManagerConfig.SPARK_YARN_MAXAPPATTEMPTS.toString)
     }
 
-    // TODO: Support escape delimiter
     // Set default Spark conf by TaskManager configuration file
-    val defaultSparkConfs = TaskManagerConfig.SPARK_DEFAULT_CONF.split(",")
+    val defaultSparkConfs = TaskManagerConfig.SPARK_DEFAULT_CONF.split(";")
     defaultSparkConfs.map(sparkConf => {
       if (sparkConf.nonEmpty) {
         val kvList = sparkConf.split("=")
-        launcher.setConf(kvList(0), kvList(1))
+        val key = kvList(0)
+        val value = kvList.drop(1).mkString("=")
+        launcher.setConf(key, value)
       }
     })
 

release/conf/taskmanager.properties

@@ -10,4 +10,5 @@ zookeeper.root_path=/openmldb
 # Spark Config
 spark.home=
 spark.master=local
-offline.data.prefix=file:///tmp/openmldb_offline_storage/
+offline.data.prefix=file:///tmp/openmldb_offline_storage/
+spark.default.conf=spark.driver.extraJavaOptions=-Dfile.encoding=utf-8;spark.executor.extraJavaOptions=-Dfile.encoding=utf-8