Make K8s tasks persist task report + Fix MSQ INSERT/REPLACE querying problem.

distribution/docker/peon.sh

@@ -39,11 +39,12 @@
 # - DRUID_CONFIG_${service} -- full path to a file for druid 'service' properties
 
 # This script is very similar to druid.sh, used exclusively for the kubernetes-overlord-extension.
+# Specifically, it is used by K8sTaskAdapter and PodTemplateTaskAdapter to start up Peon tasks.
 
 set -e
 SERVICE="overlord"
 
-echo "$(date -Is) startup service $SERVICE"
+echo "$(date -Is) startup service peon"
 
 # We put all the config in /tmp/conf to allow for a
 # read-only root filesystem
@@ -161,8 +162,10 @@ fi
 # If TASK_JSON is not set, CliPeon will pull the task.json file from deep storage.
 mkdir -p ${TASK_DIR}; [ -n "$TASK_JSON" ] && echo ${TASK_JSON} | base64 -d | gzip -d > ${TASK_DIR}/task.json;
 
+# Start peon using CliPeon, with variables `Main internal peon TASK_DIR ATTEMPT_ID`
 if [ -n "$TASK_ID" ]; then
-    exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main internal peon --taskId "${TASK_ID}" "$@"
+    # TASK_ID is only set from PodTemplateTaskAdapter
+    exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main internal peon "${TASK_DIR}" 1 --taskId "${TASK_ID}" "$@"
 else
-    exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main internal peon "$@"
+    exec bin/run-java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main internal peon "${TASK_DIR}" 1 "$@"
 fi

docs/development/extensions-core/k8s-jobs.md

@@ -372,11 +372,12 @@ The below runtime properties need to be passed to the Job's peon process.
 druid.port=8100 (what port the peon should run on)
 druid.peon.mode=remote
 druid.service=druid/peon (for metrics reporting)
-druid.indexer.task.baseTaskDir=/druid/data (this should match the argument to the ./peon.sh run command in the PodTemplate)
 druid.indexer.runner.type=k8s
 druid.indexer.task.encapsulatedTask=true
 ```
 
+**Note**: Prior to Druid 35.0.0, you will need the `druid.indexer.task.baseTaskDir` runtime property, along with the `TASK_DIR` and `attemptId` arguments to `/peon.sh` to run your jobs. There is no need for that now as Druid will automatically configure the task directory. You can still choose to customize the target task directory by adjusting `druid.indexer.task.baseTaskDir` on the Overlord service.
+
 #### Example 1: Using a Pod Template that retrieves values from a ConfigMap 
 
 <details>
@@ -398,7 +399,7 @@ template:
         - sh
         - -c
         - |
-          /peon.sh /druid/data 1
+          /peon.sh
       env:
       - name: CUSTOM_ENV_VARIABLE
         value: "hello"
@@ -492,7 +493,6 @@ data:
         druid.port=8100
         druid.service=druid/peon
         druid.server.http.numThreads=5
-        druid.indexer.task.baseTaskDir=/druid/data
         druid.indexer.runner.type=k8s
         druid.peon.mode=remote
         druid.indexer.task.encapsulatedTask=true
@@ -544,7 +544,7 @@ data:
             - sh
             - -c
             - |
-              /peon.sh /druid/data 1
+              /peon.sh
           env:
             - name: druid_port
               value: 8100
@@ -556,8 +556,6 @@ data:
               value: remote
             - name: druid_service
               value: "druid/peon"
-            - name: druid_indexer_task_baseTaskDir
-              value: /druid/data
             - name: druid_indexer_runner_type
               value: k8s
             - name: druid_indexer_task_encapsulatedTask

extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java

@@ -124,7 +124,7 @@ public Job fromTask(Task task) throws IOException
     PeonCommandContext context = new PeonCommandContext(
         generateCommand(task),
         javaOpts(task),
-        taskConfig.getBaseTaskDir(),
+        taskConfig.getTaskDir(task.getId()),
         taskRunnerConfig.getCpuCoreInMicro(),
         node.isEnableTlsPort()
     );
@@ -408,6 +408,9 @@ private List<String> javaOpts(Task task)
       );
     }
 
+    javaOpts.add(org.apache.druid.java.util.common.StringUtils.format(
+        "-Ddruid.indexer.task.baseTaskDir=%s", taskConfig.getBaseTaskDir().getAbsolutePath()
+    ));
     javaOpts.add(org.apache.druid.java.util.common.StringUtils.format("-Ddruid.port=%d", DruidK8sConstants.PORT));
     javaOpts.add(org.apache.druid.java.util.common.StringUtils.format(
         "-Ddruid.plaintextPort=%d",
@@ -435,8 +438,6 @@ private List<String> generateCommand(Task task)
   {
     final List<String> command = new ArrayList<>();
     command.add("/peon.sh");
-    command.add(taskConfig.getBaseTaskDir().getAbsolutePath());
-    command.add("1"); // the attemptId is always 1, we never run the task twice on the same pod.
 
     String nodeType = task.getNodeType();
     if (nodeType != null) {

extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/PodTemplateTaskAdapter.java

@@ -49,6 +49,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
+import java.nio.file.Paths;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -217,6 +218,10 @@ private Collection<EnvVar> getEnv(Task task) throws IOException
     List<EnvVar> envVars = Lists.newArrayList(
         new EnvVarBuilder()
             .withName(DruidK8sConstants.TASK_DIR_ENV)
+            .withValue(Paths.get(taskConfig.getBaseDir()).resolve(task.getId()).toString())
+            .build(),
+        new EnvVarBuilder()
+            .withName("druid_indexer_task_baseTaskDir")
             .withValue(taskConfig.getBaseDir())
             .build(),
         new EnvVarBuilder()

extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/MultiContainerTaskAdapterTest.java

@@ -105,7 +105,7 @@ public void testMultiContainerSupport() throws IOException
         task,
         new PeonCommandContext(
             Collections.singletonList(
-                "/peon.sh /druid/data/baseTaskDir/noop_2022-09-26T22:08:00.582Z_352988d2-5ff7-4b70-977c-3de96f9bfca6 1"),
+                "/peon.sh"),
             new ArrayList<>(),
             new File("/tmp"),
             config.getCpuCoreInMicro()
@@ -159,7 +159,7 @@ public void testMultiContainerSupportWithNamedContainer() throws IOException
         task,
         new PeonCommandContext(
             Collections.singletonList(
-                "/peon.sh /druid/data/baseTaskDir/noop_2022-09-26T22:08:00.582Z_352988d2-5ff7-4b70-977c-3de96f9bfca6 1"),
+                "/peon.sh"),
             new ArrayList<>(),
             new File("/tmp"),
             config.getCpuCoreInMicro()
@@ -215,7 +215,7 @@ public void testOverridingPeonMonitors() throws IOException
         task,
         new PeonCommandContext(
             Collections.singletonList(
-                "/peon.sh /druid/data/baseTaskDir/noop_2022-09-26T22:08:00.582Z_352988d2-5ff7-4b70-977c-3de96f9bfca6 1"),
+                "/peon.sh"),
             new ArrayList<>(),
             new File("/tmp"),
             config.getCpuCoreInMicro()

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutput.yaml

@@ -21,7 +21,7 @@ spec:
       hostname: "id-3e70afe5cd823dfc7dd308eea616426b"
       containers:
         - args:
-            - "/kubexit/kubexit /bin/sh -c \"/peon.sh /druid/data/baseTaskDir/noop_2022-09-26T22:08:00.582Z_352988d2-5ff7-4b70-977c-3de96f9bfca6 1\""
+            - "/kubexit/kubexit /bin/sh -c \"/peon.sh\""
           command:
             - "/bin/sh"
             - "-c"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedMultiContainerOutputOrder.yaml

@@ -21,7 +21,7 @@ spec:
       hostname: "id-3e70afe5cd823dfc7dd308eea616426b"
       containers:
         - args:
-            - "/kubexit/kubexit /bin/sh -c \"/peon.sh /druid/data/baseTaskDir/noop_2022-09-26T22:08:00.582Z_352988d2-5ff7-4b70-977c-3de96f9bfca6 1\""
+            - "/kubexit/kubexit /bin/sh -c \"/peon.sh\""
           command:
             - "/bin/sh"
             - "-c"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedNoopJob.yaml

@@ -4,11 +4,11 @@ metadata:
   name: "id-3e70afe5cd823dfc7dd308eea616426b"
   labels:
     druid.k8s.peons: "true"
+    druid.overlord.namespace: "namespace"
     druid.task.id: "id"
     druid.task.type: "noop"
     druid.task.group.id: "id"
     druid.task.datasource: "datasource"
-    druid.overlord.namespace: "namespace"
   annotations:
     task.id: "id"
     task.type: "noop"
@@ -23,11 +23,11 @@ spec:
     metadata:
       labels:
         druid.k8s.peons: "true"
+        druid.overlord.namespace: "namespace"
         druid.task.id: "id"
         druid.task.type: "noop"
         druid.task.group.id: "id"
         druid.task.datasource: "datasource"
-        druid.overlord.namespace: "namespace"
       annotations:
         task: "H4sIAAAAAAAAAD2MvQ4CIRCE32VqijsTG1qLi7W+wArEbHICrmC8EN7dJf40k/lmJtNQthxgEVPKMGCvXsXgKqnm4x89FTqlKm6MBzw+YCA1nvmm8W4/TQYuxRJeBbZ17cJ3ZhvoSbzShVcu2zLOf9cS7pUl+ANlclrCzr2/AQUK0FqZAAAA"
         tls.enabled: "false"
@@ -43,6 +43,8 @@ spec:
             - "3600"
           env:
             - name: "TASK_DIR"
+              value: "/tmp/id"
+            - name: "druid_indexer_task_baseTaskDir"
               value: "/tmp"
             - name: "TASK_ID"
               value: "id"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobLongIds.yaml

@@ -4,11 +4,11 @@ metadata:
   name: "apiissuedkillwikipedia3omjobnb-18ed64f09a02fab468b9bba38739871f"
   labels:
     druid.k8s.peons: "true"
+    druid.overlord.namespace: "namespace"
     druid.task.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
     druid.task.type: "noop"
     druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
     druid.task.datasource: "data_source"
-    druid.overlord.namespace: "namespace"
   annotations:
     task.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
     task.type: "noop"
@@ -23,11 +23,11 @@ spec:
     metadata:
       labels:
         druid.k8s.peons: "true"
+        druid.overlord.namespace: "namespace"
         druid.task.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
         druid.task.type: "noop"
         druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
         druid.task.datasource: "data_source"
-        druid.overlord.namespace: "namespace"
       annotations:
         task: "H4sIAAAAAAAAAMVQPa/CMAz8L55b5KRUSFkZnt5Mpy6R20RPhtKENOFDVf87KbC+GekG352l83mG+PAWFIzOeSiATZ7Jc8nTlKzRJx4GfeMTe2uYKu3OR9eNXa8FIpYoMhpE9cImS62WKKsS61Js/zPqRuwUVgrFRkpsc+pfcMn/fiXaUKSDS6Ffn7ASPb1ZASGNDZ+zLmvEAno3RnuPoOYle/azpmagK/FAHQ8cHz9rk2/0CPaSOFizJ099PgSUWJYnqMIU2d4BAAA="
         tls.enabled: "false"
@@ -43,6 +43,8 @@ spec:
             - "3600"
           env:
             - name: "TASK_DIR"
+              value: "/tmp/api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
+            - name: "druid_indexer_task_baseTaskDir"
               value: "/tmp"
             - name: "TASK_ID"
               value: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobNoTaskJson.yaml

@@ -4,11 +4,11 @@ metadata:
   name: "id-3e70afe5cd823dfc7dd308eea616426b"
   labels:
     druid.k8s.peons: "true"
+    druid.overlord.namespace: "namespace"
     druid.task.id: "id"
     druid.task.type: "noop"
     druid.task.group.id: "id"
     druid.task.datasource: "datasource"
-    druid.overlord.namespace: "namespace"
   annotations:
     task.id: "id"
     task.type: "noop"
@@ -23,11 +23,11 @@ spec:
     metadata:
       labels:
         druid.k8s.peons: "true"
+        druid.overlord.namespace: "namespace"
         druid.task.id: "id"
         druid.task.type: "noop"
         druid.task.group.id: "id"
         druid.task.datasource: "datasource"
-        druid.overlord.namespace: "namespace"
       annotations:
         tls.enabled: "false"
         task.id: "id"
@@ -42,6 +42,8 @@ spec:
             - "3600"
           env:
             - name: "TASK_DIR"
+              value: "/tmp/id"
+            - name: "druid_indexer_task_baseTaskDir"
               value: "/tmp"
             - name: "TASK_ID"
               value: "id"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabled.yaml

@@ -4,11 +4,11 @@ metadata:
   name: "id-3e70afe5cd823dfc7dd308eea616426b"
   labels:
     druid.k8s.peons: "true"
+    druid.overlord.namespace: "namespace"
     druid.task.id: "id"
     druid.task.type: "noop"
     druid.task.group.id: "id"
     druid.task.datasource: "datasource"
-    druid.overlord.namespace: "namespace"
   annotations:
     task.id: "id"
     task.type: "noop"
@@ -23,11 +23,11 @@ spec:
     metadata:
       labels:
         druid.k8s.peons: "true"
+        druid.overlord.namespace: "namespace"
         druid.task.id: "id"
         druid.task.type: "noop"
         druid.task.group.id: "id"
         druid.task.datasource: "datasource"
-        druid.overlord.namespace: "namespace"
       annotations:
         task: "H4sIAAAAAAAAAD2MvQ4CIRCE32VqijsTG1qLi7W+wArEbHICrmC8EN7dJf40k/lmJtNQthxgEVPKMGCvXsXgKqnm4x89FTqlKm6MBzw+YCA1nvmm8W4/TQYuxRJeBbZ17cJ3ZhvoSbzShVcu2zLOf9cS7pUl+ANlclrCzr2/AQUK0FqZAAAA"
         tls.enabled: "true"
@@ -43,6 +43,8 @@ spec:
             - "3600"
           env:
             - name: "TASK_DIR"
+              value: "/tmp/id"
+            - name: "druid_indexer_task_baseTaskDir"
               value: "/tmp"
             - name: "TASK_ID"
               value: "id"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedNoopJobTlsEnabledBase.yaml

@@ -4,11 +4,11 @@ metadata:
   name: "id-3e70afe5cd823dfc7dd308eea616426b"
   labels:
     druid.k8s.peons: "true"
+    druid.overlord.namespace: "namespace"
     druid.task.id: "id"
     druid.task.type: "noop"
     druid.task.group.id: "id"
     druid.task.datasource: "datasource"
-    druid.overlord.namespace: "namespace"
   annotations:
     task.id: "id"
     task.type: "noop"
@@ -23,11 +23,11 @@ spec:
     metadata:
       labels:
         druid.k8s.peons: "true"
+        druid.overlord.namespace: "namespace"
         druid.task.id: "id"
         druid.task.type: "noop"
         druid.task.group.id: "id"
         druid.task.datasource: "datasource"
-        druid.overlord.namespace: "namespace"
       annotations:
         task: "H4sIAAAAAAAAAD2MvQ4CIRCE32VqijsTG1qLi7W+wArEbHICrmC8EN7dJf40k/lmJtNQthxgEVPKMGCvXsXgKqnm4x89FTqlKm6MBzw+YCA1nvmm8W4/TQYuxRJeBbZ17cJ3ZhvoSbzShVcu2zLOf9cS7pUl+ANlclrCzr2/AQUK0FqZAAAA"
         tls.enabled: "true"
@@ -43,6 +43,8 @@ spec:
             - "3600"
           env:
             - name: "TASK_DIR"
+              value: "/tmp/id"
+            - name: "druid_indexer_task_baseTaskDir"
               value: "/tmp"
             - name: "TASK_ID"
               value: "id"

extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedPodSpec.yaml

@@ -20,8 +20,7 @@ spec:
     spec:
       containers:
         - args:
-            - "/kubexit/kubexit /bin/sh -c \"/peon.sh /druid/data/baseTaskDir/noop_2022-09-26T22:08:00.582Z_352988d2-5ff7-4b70-977c-3de96f9bfca6\
-          \ 1\""
+            - "/kubexit/kubexit /bin/sh -c \"/peon.sh\""
           command:
             - "/bin/sh"
             - "-c"