feat(helm): Improve modularity and add Presto support refinements

- Replace `query_engine` conditions with modular flags in Helm templates.
- Refactor Presto configurations to use dedicated flags and scripts.
- Enhance worker and reducer node scheduling logic.
- Update documentation to reflect Presto integration and better configuration flexibility.

Author: junhaoliao

docs/src/user-docs/guides-k8s-deployment.md

@@ -323,12 +323,21 @@ clpConfig:
     storage_engine: "clp-s"
     query_engine: "presto"
 
+  # Disable the clp-s query pipeline since Presto replaces it.
+  # NOTE: The API server currently depends on the clp-s query pipeline and does not work with
+  # Presto. Keep it enabled if you need the API server; disable it if not.
+  api_server: null
+  query_scheduler: null
+  query_worker: null
+  reducer: null
+
   # Disable results cache retention since the Presto integration doesn't yet support garbage
   # collection of search results.
   results_cache:
     retention_period: null
 
   presto:
+    port: 30889
     coordinator:
       logging_level: "INFO"
       query_max_memory_gb: 1
@@ -386,6 +395,9 @@ To run compression workers, query workers, and reducers in separate node pools:
 
    # Label query nodes
    kubectl label nodes node3 node4 yscope.io/nodeType=query
+
+   # Label Presto nodes (if using Presto as the query engine)
+   kubectl label nodes node5 node6 yscope.io/nodeType=presto
    ```
 
 2. Configure scheduling:
@@ -399,19 +411,27 @@ To run compression workers, query workers, and reducers in separate node pools:
      replicas: 2
      scheduling:
        nodeSelector:
-         yscope.io/nodeType: compression
+         yscope.io/nodeType: "compression"
 
    queryWorker:
      replicas: 2
      scheduling:
        nodeSelector:
-         yscope.io/nodeType: query
+         yscope.io/nodeType: "query"
 
    reducer:
      replicas: 2
      scheduling:
        nodeSelector:
-         yscope.io/nodeType: query
+         yscope.io/nodeType: "query"
+
+   # If using Presto as the query engine, configure prestoWorker instead of
+   # queryWorker and reducer.
+   prestoWorker:
+     replicas: 2
+     scheduling:
+       nodeSelector:
+         yscope.io/nodeType: "presto"
    ```
 
 3. Install:
@@ -441,7 +461,7 @@ To run all worker types in the same node pool:
      replicas: 2
      scheduling:
        nodeSelector:
-         yscope.io/nodeType: compute
+         yscope.io/nodeType: "compute"
        topologySpreadConstraints:
          - maxSkew: 1
            topologyKey: "kubernetes.io/hostname"
@@ -454,13 +474,21 @@ To run all worker types in the same node pool:
      replicas: 2
      scheduling:
        nodeSelector:
-         yscope.io/nodeType: compute
+         yscope.io/nodeType: "compute"
 
    reducer:
      replicas: 2
      scheduling:
        nodeSelector:
-         yscope.io/nodeType: compute
+         yscope.io/nodeType: "compute"
+
+   # If using Presto as the query engine, configure prestoWorker instead of
+   # queryWorker and reducer.
+   prestoWorker:
+     replicas: 2
+     scheduling:
+       nodeSelector:
+         yscope.io/nodeType: "compute"
    ```
 
 3. Install:

docs/src/user-docs/guides-using-presto.md

@@ -43,12 +43,28 @@ When deploying CLP on Kubernetes using Helm, Presto can be enabled by setting th
      package:
        query_engine: "presto"
 
+     # Disable the clp-s query pipeline since Presto replaces it.
+     # NOTE: The API server currently depends on the clp-s query pipeline and does not work
+     # with Presto. Keep it enabled if you need the API server; disable it if not.
+     api_server: null
+     query_scheduler: null
+     query_worker: null
+     reducer: null
+
      # Disable results cache retention since the Presto integration doesn't yet support
      # garbage collection of search results.
      results_cache:
        retention_period: null
 
      presto:
+       port: 30889
+       coordinator:
+         logging_level: "INFO"
+         query_max_memory_gb: 1
+         query_max_memory_per_node_gb: 1
+       worker:
+         query_memory_gb: 4
+         system_memory_gb: 8
        # Split filter config for the Presto CLP connector. For each dataset, add a filter entry.
        # Replace <dataset> with the dataset name (use "default" if you didn't specify one when
        # compressing) and <timestamp-key> with the timestamp key used during compression.

tools/deployment/package-helm/.set-up-common.sh

@@ -82,6 +82,9 @@ nodes:
   - containerPort: 30800
     hostPort: 30800
     protocol: TCP
+  - containerPort: 30889
+    hostPort: 30889
+    protocol: TCP
 EOF
 
     for ((i = 0; i < num_workers; i++)); do

tools/deployment/package-helm/set-up-multi-dedicated-test.sh

@@ -10,9 +10,11 @@ CLP_HOME="${CLP_HOME:-/tmp/clp}"
 CLUSTER_NAME="${CLUSTER_NAME:-clp-test}"
 NUM_COMPRESSION_NODES="${NUM_COMPRESSION_NODES:-2}"
 NUM_QUERY_NODES="${NUM_QUERY_NODES:-2}"
+NUM_PRESTO_NODES="${NUM_PRESTO_NODES:-2}"
 COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}"
 QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}"
 REDUCER_REPLICAS="${REDUCER_REPLICAS:-2}"
+PRESTO_WORKER_REPLICAS="${PRESTO_WORKER_REPLICAS:-2}"
 
 # shellcheck source=.set-up-common.sh
 source "${script_dir}/.set-up-common.sh"
@@ -21,14 +23,16 @@ echo "=== Multi-node setup with dedicated worker nodes ==="
 echo "Cluster: ${CLUSTER_NAME}"
 echo "Compression nodes: ${NUM_COMPRESSION_NODES}"
 echo "Query nodes: ${NUM_QUERY_NODES}"
+echo "Presto nodes: ${NUM_PRESTO_NODES}"
 echo "Compression workers: ${COMPRESSION_WORKER_REPLICAS}"
 echo "Query workers: ${QUERY_WORKER_REPLICAS}"
 echo "Reducers: ${REDUCER_REPLICAS}"
+echo "Presto workers: ${PRESTO_WORKER_REPLICAS}"
 echo ""
 
 prepare_environment "${CLUSTER_NAME}"
 
-total_workers=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES))
+total_workers=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES + NUM_PRESTO_NODES))
 
 echo "Creating kind cluster..."
 generate_kind_config "${total_workers}" | kind create cluster --name "${CLUSTER_NAME}" --config=-
@@ -43,11 +47,18 @@ for ((i = 0; i < NUM_COMPRESSION_NODES; i++)); do
 done
 
 # Label query nodes
-for ((i = NUM_COMPRESSION_NODES; i < total_workers; i++)); do
+query_end=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES))
+for ((i = NUM_COMPRESSION_NODES; i < query_end; i++)); do
     echo "Labeling ${worker_nodes[$i]} as query node"
     kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=query --overwrite
 done
 
+# Label Presto nodes
+for ((i = query_end; i < total_workers; i++)); do
+    echo "Labeling ${worker_nodes[$i]} as presto node"
+    kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=presto --overwrite
+done
+
 echo "Installing Helm chart..."
 helm uninstall test --ignore-not-found
 sleep 2
@@ -57,6 +68,9 @@ helm install test "${script_dir}" \
     --set "compressionWorker.scheduling.nodeSelector.yscope\.io/nodeType=compression" \
     --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" \
     --set "queryWorker.scheduling.nodeSelector.yscope\.io/nodeType=query" \
-    --set "reducer.replicas=${REDUCER_REPLICAS}"
+    --set "reducer.replicas=${REDUCER_REPLICAS}" \
+    --set "reducer.scheduling.nodeSelector.yscope\.io/nodeType=query" \
+    --set "prestoWorker.replicas=${PRESTO_WORKER_REPLICAS}" \
+    --set "prestoWorker.scheduling.nodeSelector.yscope\.io/nodeType=presto"
 
 wait_for_cluster_ready

tools/deployment/package-helm/set-up-multi-shared-test.sh

@@ -12,6 +12,7 @@ NUM_WORKER_NODES="${NUM_WORKER_NODES:-2}"
 COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}"
 QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}"
 REDUCER_REPLICAS="${REDUCER_REPLICAS:-2}"
+PRESTO_WORKER_REPLICAS="${PRESTO_WORKER_REPLICAS:-2}"
 
 # shellcheck source=.set-up-common.sh
 source "${script_dir}/.set-up-common.sh"
@@ -22,6 +23,7 @@ echo "Worker nodes: ${NUM_WORKER_NODES}"
 echo "Compression workers: ${COMPRESSION_WORKER_REPLICAS}"
 echo "Query workers: ${QUERY_WORKER_REPLICAS}"
 echo "Reducers: ${REDUCER_REPLICAS}"
+echo "Presto workers: ${PRESTO_WORKER_REPLICAS}"
 echo ""
 
 prepare_environment "${CLUSTER_NAME}"
@@ -36,6 +38,7 @@ helm install test "${script_dir}" \
     --set "distributedDeployment=true" \
     --set "compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \
     --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" \
-    --set "reducer.replicas=${REDUCER_REPLICAS}"
+    --set "reducer.replicas=${REDUCER_REPLICAS}" \
+    --set "prestoWorker.replicas=${PRESTO_WORKER_REPLICAS}"
 
 wait_for_cluster_ready

tools/deployment/package-helm/templates/api-server-deployment.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.clpConfig.api_server (ne .Values.clpConfig.package.query_engine "presto") }}
+{{- if .Values.clpConfig.api_server }}
 apiVersion: "apps/v1"
 kind: "Deployment"
 metadata:

tools/deployment/package-helm/templates/api-server-logs-pv.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.clpConfig.api_server (ne .Values.clpConfig.package.query_engine "presto") }}
+{{- if .Values.clpConfig.api_server }}
 {{- include "clp.createStaticPv" (dict
   "root" .
   "component_category" "api-server"

tools/deployment/package-helm/templates/api-server-logs-pvc.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.clpConfig.api_server (ne .Values.clpConfig.package.query_engine "presto") }}
+{{- if .Values.clpConfig.api_server }}
 {{- include "clp.createPvc" (dict
   "root" .
   "component_category" "api-server"

tools/deployment/package-helm/templates/api-server-service.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.clpConfig.api_server (ne .Values.clpConfig.package.query_engine "presto") }}
+{{- if .Values.clpConfig.api_server }}
 apiVersion: "v1"
 kind: "Service"
 metadata:

tools/deployment/package-helm/templates/configmap.yaml

@@ -129,15 +129,18 @@ data:
     package:
       query_engine: {{ .Values.clpConfig.package.query_engine | quote }}
       storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }}
+    {{- with .Values.clpConfig.query_scheduler }}
     query_scheduler:
-      host: "{{ include "clp.fullname" . }}-query-scheduler"
-      jobs_poll_delay: {{ .Values.clpConfig.query_scheduler.jobs_poll_delay }}
-      logging_level: {{ .Values.clpConfig.query_scheduler.logging_level | quote }}
-      num_archives_to_search_per_sub_job: {{
-        .Values.clpConfig.query_scheduler.num_archives_to_search_per_sub_job | int }}
+      host: "{{ include "clp.fullname" $ }}-query-scheduler"
+      jobs_poll_delay: {{ .jobs_poll_delay }}
+      logging_level: {{ .logging_level | quote }}
+      num_archives_to_search_per_sub_job: {{ .num_archives_to_search_per_sub_job | int }}
       port: 7000
+    {{- end }}
+    {{- with .Values.clpConfig.query_worker }}
     query_worker:
-      logging_level: {{ .Values.clpConfig.query_worker.logging_level | quote }}
+      logging_level: {{ .logging_level | quote }}
+    {{- end }}
     queue:
       host: "{{ include "clp.fullname" . }}-queue"
       port: 5672
@@ -146,11 +149,13 @@ data:
       host: "{{ include "clp.fullname" . }}-redis"
       port: 6379
       query_backend_database: {{ .Values.clpConfig.redis.query_backend_database | int }}
+    {{- with .Values.clpConfig.reducer }}
     reducer:
       base_port: 14009
-      host: "{{ include "clp.fullname" . }}-reducer"
-      logging_level: {{ .Values.clpConfig.reducer.logging_level | quote }}
-      upsert_interval: {{ .Values.clpConfig.reducer.upsert_interval | int }}
+      host: "{{ include "clp.fullname" $ }}-reducer"
+      logging_level: {{ .logging_level | quote }}
+      upsert_interval: {{ .upsert_interval | int }}
+    {{- end }}
     results_cache:
       db_name: {{ .Values.clpConfig.results_cache.db_name | quote }}
       host: "{{ include "clp.fullname" . }}-results-cache"
@@ -219,10 +224,10 @@ data:
     {{- else }}
     mcp_server: null
     {{- end }}
-    {{- if eq .Values.clpConfig.package.query_engine "presto" }}
+    {{- if .Values.clpConfig.presto }}
     presto:
       host: "{{ include "clp.fullname" . }}-presto-coordinator"
-      port: 8080
+      port: 8889
     {{- else }}
     presto: null
     {{- end }}
@@ -328,16 +333,15 @@ data:
         {{ .Values.clpConfig.archive_output.target_segment_size | int }},
       "ClpQueryEngine": {{ .Values.clpConfig.package.query_engine | quote }},
       "ClpStorageEngine": {{ .Values.clpConfig.package.storage_engine | quote }},
-      {{- if eq .Values.clpConfig.package.query_engine "presto" }}
+      {{- if .Values.clpConfig.presto }}
       "PrestoHost": "{{ include "clp.fullname" . }}-presto-coordinator",
-      "PrestoPort": 8080
+      "PrestoPort": 8889
       {{- else }}
       "PrestoHost": null,
       "PrestoPort": null
       {{- end }}
     }
 
-{{- if eq .Values.clpConfig.package.query_engine "presto" }}
 {{- with .Values.clpConfig.presto }}
   presto-coordinator-catalog-clp.properties: |
     connector.name=clp
@@ -354,11 +358,11 @@ data:
   presto-coordinator-config-config.properties: |
     coordinator=true
     node-scheduler.include-coordinator=false
-    http-server.http.port=8080
+    http-server.http.port=8889
     query.max-memory={{ .coordinator.query_max_memory_gb }}GB
     query.max-memory-per-node={{ .coordinator.query_max_memory_per_node_gb }}GB
     discovery-server.enabled=true
-    discovery.uri=http://{{ include "clp.fullname" $ }}-presto-coordinator:8080
+    discovery.uri=http://{{ include "clp.fullname" $ }}-presto-coordinator:8889
     optimizer.optimize-hash-generation=false
     regex-library=RE2J
     use-alternative-function-signatures=true
@@ -400,8 +404,8 @@ data:
     {{- end }}{{/* with $.Values.clpConfig.archive_output.storage */}}
 
   presto-worker-config-config.properties: |
-    discovery.uri=http://{{ include "clp.fullname" $ }}-presto-coordinator:8080
-    http-server.http.port=8080
+    discovery.uri=http://{{ include "clp.fullname" $ }}-presto-coordinator:8889
+    http-server.http.port=8889
     query-memory-gb={{ .worker.query_memory_gb }}
     shutdown-onset-sec=1
     system-memory-gb={{ .worker.system_memory_gb }}
@@ -414,5 +418,23 @@ data:
 
   presto-worker-config-velox.properties: |
     mutable-config=true
+
+  presto-worker-setup-configs.sh: |
+    #!/bin/sh
+    set -e
+
+    CONFIG_TEMPLATE_DIR="/etc/presto-config"
+    PRESTO_CONFIG_DIR="/opt/presto-server/etc"
+
+    cp "${CONFIG_TEMPLATE_DIR}/presto-worker-config-config.properties" "${PRESTO_CONFIG_DIR}/config.properties"
+    cp "${CONFIG_TEMPLATE_DIR}/presto-worker-config-node.properties" "${PRESTO_CONFIG_DIR}/node.properties"
+
+    # Query coordinator for version and append to config
+    DISCOVERY_URI="http://{{ include "clp.fullname" $ }}-presto-coordinator:8889"
+    PRESTO_VERSION=$(wget -q -O - "${DISCOVERY_URI}/v1/info" | jq -r '.version')
+    echo "presto.version=${PRESTO_VERSION}" >> "${PRESTO_CONFIG_DIR}/config.properties"
+
+    # Set node identity from hostname
+    echo "node.internal-address=$(hostname -i)" >> "${PRESTO_CONFIG_DIR}/node.properties"
+    echo "node.id=$(hostname)" >> "${PRESTO_CONFIG_DIR}/node.properties"
 {{- end }}{{/* with .Values.clpConfig.presto */}}
-{{- end }}{{/* if eq .Values.clpConfig.package.query_engine "presto" */}}