docs: Add k8s setup instructions.

yscope-k8s/.gitignore

@@ -0,0 +1 @@
+demo-assets/clp-config.yml

yscope-k8s/.helmignore

@@ -0,0 +1,27 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+
+# Demo assets
+/demo-assets/

yscope-k8s/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: presto-velox
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

yscope-k8s/README.md

@@ -0,0 +1,107 @@
+# Setup local K8s cluster for presto + clp
+
+## Install docker
+
+Follow the guide here: [docker]
+
+## Install kubectl
+
+`kubectl` is the command-line tool for interacting with Kubernetes clusters. You will use it to
+manage and inspect your k3d cluster.
+
+Follow the guide here: [kubectl]
+
+## Install k3d
+
+k3d is a lightweight wrapper to run k3s (Rancher Lab's minimal Kubernetes distribution) in docker.
+
+Follow the guide here: [k3d]
+
+## Install Helm
+
+Helm is the package manager for Kubernetes.
+
+Follow the guide here: [helm]
+
+# Launch clp-package
+1. Find the clp-package for test on our official website [clp-json-v0.4.0]. We also put the dataset for demo here: `mongod-256MB-presto-clp.log.tar.gz`.
+
+2. Untar it.
+
+3. Replace the content of `/path/to/clp-json-package/etc/clp-config.yml` with the output of `demo-assets/init.sh <ip_addr>` where the `<ip_addr>` is the IP address of the host that you are running the clp-package.
+
+4. Launch:
+```bash
+# You probably want to run in a 3.11 python environment
+sbin/start-clp.sh
+```
+
+5. Compress:
+```bash
+# You can also use your own dataset
+sbin/compress.sh --timestamp-key 't.dollar_sign_date' datasets/mongod-256MB-processed.log
+```
+
+6. Use the following command to update the CLP metadata database so that the worker can find the archives in right place:
+```bash
+# Install mysql-client if necessary
+sudo apt update && sudo apt install -y mysql-client
+# Find the user and password in /path/to/clp-json-package/etc/credential.yml
+mysql -h ${REPLACE_IP} -P 6001 -u ${REPLACE_USER} -p'${REPLACE_PASSWORD}' clp-db -e "UPDATE clp_datasets SET archive_storage_directory = '/var/data/archives/default';"
+```
+
+# Create k8s Cluster
+Create a local k8s cluster with port forwarding
+```bash
+k3d cluster create yscope --servers 1 --agents 1 -v $(readlink -f /path/to/clp-json-package/var/data/archives):/var/data/archives
+```
+
+# Working with helm chart
+## Install
+In `yscope-k8s/templates/presto/presto-coordinator-config.yaml`:
+1. replace the `${REPLACE_ME}` in `clp.metadata-db-url=jdbc:mysql://${REPLACE_ME}:6001` with the IP address of the host you are running the clp-package (basially match the IP address that you configured in the `etc/clp-config.yml` of the clp-package).
+2. replace the `${REPLACE_ME}` in `clp.metadata-db-user=${REPLACE_ME}` with the user stored in `/path/to/clp-json-package/etc/credential.yml`.
+3. replace the `${REPLACE_ME}` in `clp.metadata-db-password=${REPLACE_ME}` with the password stored in `/path/to/clp-json-package/etc/credential.yml`.
+
+```bash
+cd yscope-k8s
+
+helm template . 
+
+helm install demo .
+```
+
+## Use cli:
+After all containers are in "Running" states (check by `kubectl get pods`):
+```bash
+kubectl port-forward service/presto-coordinator 8080:8080
+```
+
+Then you can further forward the 8080 port to your local laptop, to access the Presto's WebUI by e.g., http://localhost:8080
+
+To use presto-cli:
+```bash
+./presto-cli-0.293-executable.jar --catalog clp --schema default --server localhost:8080
+```
+
+Example query:
+```
+SELECT * FROM default LIMIT 1;
+```
+
+## Uninstall
+```bash
+helm uninstall demo
+```
+
+# Delete k8s Cluster
+```bash
+k3d cluster delete yscope
+```
+
+
+[clp-json-v0.4.0]: https://github.com/y-scope/clp/releases/tag/v0.4.0
+[docker]: https://docs.docker.com/engine/install
+[k3d]: https://k3d.io/stable/#installation
+[kubectl]: https://kubernetes.io/docs/tasks/tools/#kubectl
+[helm]: https://helm.sh/docs/intro/install/

yscope-k8s/demo-assets/clp-config.yml.bak

@@ -0,0 +1,38 @@
+package:
+  storage_engine: "clp-s"
+database:
+  type: "mariadb"
+  host: "${REPLACE_IP}"
+  port: 6001
+  name: "clp-db"
+query_scheduler:
+  host: "${REPLACE_IP}"
+  port: 6002
+  jobs_poll_delay: 0.1
+  num_archives_to_search_per_sub_job: 16
+  logging_level: "INFO"
+queue:
+  host: "${REPLACE_IP}"
+  port: 6003
+redis:
+  host: "${REPLACE_IP}"
+  port: 6004
+  query_backend_database: 0
+  compression_backend_database: 1
+reducer:
+  host: "${REPLACE_IP}"
+  base_port: 6100
+  logging_level: "INFO"
+  upsert_interval: 100
+results_cache:
+  host: "${REPLACE_IP}"
+  port: 6005
+  db_name: "clp-query-results"
+  stream_collection_name: "stream-files"
+webui:
+  host: "localhost"
+  port: 6000
+  logging_level: "INFO"
+log_viewer_webui:
+  host: "localhost"
+  port: 6006

yscope-k8s/demo-assets/init.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <ip-address>"
+    exit 1
+fi
+
+SCRIPT_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+IP="$1"
+FILE="${SCRIPT_PATH}/clp-config.yml"
+
+cp "${FILE}.bak" "$FILE"
+
+sed -i "s|\${REPLACE_IP}|$IP|g" "$FILE"
+
+echo "Replaced \${REPLACE_IP} with $IP in $FILE"

yscope-k8s/templates/presto/presto-coordinator-config.yaml

@@ -0,0 +1,53 @@
+apiVersion: "v1"
+kind: "ConfigMap"
+metadata:
+  name: "presto-coordinator-config"
+data:
+  clp.properties: |
+    connector.name=clp
+    clp.metadata-provider-type=mysql
+    clp.metadata-db-url=jdbc:mysql://${REPLACE_ME}:6001
+    clp.metadata-db-name=clp-db
+    clp.metadata-db-user=${REPLACE_ME}
+    clp.metadata-db-password=${REPLACE_ME}
+    clp.metadata-table-prefix=clp_
+    clp.split-provider-type=mysql
+    clp.metadata-filter-config=/opt/presto-server/etc/metadata-filter.json
+
+  config.properties: |
+    coordinator=true
+    node-scheduler.include-coordinator=false
+    http-server.http.port=8080
+    query.max-memory=1GB
+    query.max-memory-per-node=1GB
+    discovery-server.enabled=true
+    discovery.uri=http://{{ .Values.presto.coordinator.serviceName }}:{{ .Values.presto.coordinator.httpPort }}
+    optimizer.optimize-hash-generation=false
+    regex-library=RE2J
+    use-alternative-function-signatures=true
+    inline-sql-functions=false
+    nested-data-serialization-enabled=false
+    native-execution-enabled=true
+
+  jvm.config: |
+    -server
+    -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
+    -Xmx4G
+    -XX:+UseG1GC
+    -XX:G1HeapRegionSize=32M
+    -XX:+UseGCOverheadLimit
+    -XX:+ExplicitGCInvokesConcurrent
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:+ExitOnOutOfMemoryError
+    -Djdk.attach.allowAttachSelf=true
+
+  log.properties: |
+    com.facebook.presto=DEBUG
+
+  metadata-filter.json: |
+    {
+    }
+
+  node.properties: |
+    node.environment=production
+    node.id={{ .Values.presto.coordinator.serviceName }}

yscope-k8s/templates/presto/presto-coordinator.yaml

@@ -0,0 +1,75 @@
+apiVersion: "v1"
+kind: "Pod"
+metadata:
+  labels:
+    app: "{{ .Values.presto.coordinator.serviceName }}"
+  name: "{{ .Values.presto.coordinator.serviceName }}"
+spec:
+  initContainers:
+    - name: "generate-configs"
+      image: "busybox"
+      command: ["/scripts/generate-configs.sh"]
+      volumeMounts:
+        - name: "presto-coordinator-config-script-volume"
+          mountPath: "/scripts"
+        - name: "presto-coordinator-config-dir-volume"
+          mountPath: "/opt/presto-server/etc"
+        - name: "presto-coordinator-config-volume"
+          mountPath: "/configs"
+  containers:
+    - name: "{{ .Values.presto.coordinator.serviceName }}"
+      image: "{{ .Values.presto.coordinator.image }}"
+      volumeMounts:
+        - name: "presto-coordinator-config-dir-volume"
+          mountPath: "/opt/presto-server/etc"
+        - name: "clp-archives-volume"
+          mountPath: "/var/data/archives"
+  volumes:
+    - name: "presto-coordinator-config-script-volume"
+      configMap:
+        name: "presto-coordinator-config-script"
+        defaultMode: 0755
+    - name: "presto-coordinator-config-dir-volume"
+      emptyDir: {}
+    - name: "presto-coordinator-config-volume"
+      configMap:
+        name: "presto-coordinator-config"
+    - name: "clp-archives-volume"
+      hostPath:
+        path: "/var/data/archives"
+---
+apiVersion: "v1"
+kind: "Service"
+metadata:
+  name: "{{ .Values.presto.coordinator.serviceName }}"
+  labels:
+    app: "{{ .Values.presto.coordinator.serviceName }}"
+spec:
+  type: "ClusterIP"
+  ports:
+    - name: "http"
+      port: {{ .Values.presto.coordinator.httpPort }}
+      targetPort: {{ .Values.presto.coordinator.httpPort }}
+  selector:
+    app: "{{ .Values.presto.coordinator.serviceName }}"
+---
+apiVersion: "v1"
+kind: "ConfigMap"
+metadata:
+  name: "presto-coordinator-config-script"
+data:
+  generate-configs.sh: |
+    #!/bin/sh
+
+    # Exit on error
+    set -e
+
+    PRESTO_CONFIG_DIR="/opt/presto-server/etc"
+
+    # Setup the config directory hierarchy
+    mkdir -p ${PRESTO_CONFIG_DIR}/catalog
+
+    # Copy over files
+    cp /configs/* ${PRESTO_CONFIG_DIR}
+    mv ${PRESTO_CONFIG_DIR}/clp.properties ${PRESTO_CONFIG_DIR}/catalog
+

yscope-k8s/templates/presto/presto-worker-config.yaml

@@ -0,0 +1,24 @@
+apiVersion: "v1"
+kind: "ConfigMap"
+metadata:
+  name: "presto-worker-config"
+data:
+  clp.properties: |
+    connector.name=clp
+
+  config.properties: |
+    discovery.uri=http://{{ .Values.presto.coordinator.serviceName }}.default.svc.cluster.local:{{ .Values.presto.coordinator.httpPort }}
+    presto.version=REPLACE_ME
+    http-server.http.port={{ .Values.presto.worker.httpPort }}
+    shutdown-onset-sec=1
+    register-test-functions=false
+    runtime-metrics-collection-enabled=false
+
+  node.properties: |
+    node.environment=production
+    node.internal-address=REPLACE_ME
+    node.location=testing-location
+    node.id=REPLACE_ME
+
+  velox.properties: |
+    mutable-config=true