diff --git a/CHANGELOG.md b/CHANGELOG.md index e81ef40..fecee0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,11 @@ All notable changes to this project will be documented in this file. - The Stackable scaler now ensures that a `TrinoCluster` has changed to `ready` more than 5 seconds ago before marking it as `ready` ([#68]). +- Support two different proxy modes: `ProxyAllCalls` (default) and `ProxyFirstCall`. + Read the [proxy modes docs](./docs/proxy-modes.md) for details ([#82]). [#68]: https://github.com/stackabletech/trino-lb/pull/68 +[#82]: https://github.com/stackabletech/trino-lb/pull/82 ## [0.5.0] - 2025-03-14 diff --git a/Cargo.lock b/Cargo.lock index 400d802..1537f24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2417,6 +2417,12 @@ dependencies = [ "regex", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem" version = "3.0.5" @@ -2669,8 +2675,8 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "prusto" -version = "0.5.1" -source = "git+https://github.com/sbernauer/prusto.git?branch=feat%2Fderive-name-attribute#8a3ba043f4315f03f44becaf9094effce765643b" +version = "0.5.2" +source = "git+https://github.com/sbernauer/prusto.git?branch=feat%2Fderive-name-attribute-2#5752bc9a591127dab277ea1b41397b775789a565" dependencies = [ "bigdecimal", "chrono", @@ -2681,6 +2687,7 @@ dependencies = [ "iterable", "lazy_static", "log", + "paste", "prusto-macros", "regex", "reqwest 0.11.27", @@ -2694,8 +2701,8 @@ dependencies = [ [[package]] name = "prusto-macros" -version = "0.2.0" -source = "git+https://github.com/sbernauer/prusto.git?branch=feat%2Fderive-name-attribute#8a3ba043f4315f03f44becaf9094effce765643b" +version = "0.2.1" +source = "git+https://github.com/sbernauer/prusto.git?branch=feat%2Fderive-name-attribute-2#5752bc9a591127dab277ea1b41397b775789a565" dependencies = [ "proc-macro2", "quote", @@ -4203,6 +4210,7 @@ dependencies = [ "futures-core", "http 1.3.1", "http-body 1.0.1", + "http-body-util", "mime", "pin-project-lite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 4e065b7..8ecc0ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,7 @@ sqlx = { version = "0.8.2", features = [ strum = { version = "0.27", features = ["derive"] } tokio = "1.39" tower = "0.5" -tower-http = { version = "0.6", features = ["compression-full", "tracing"] } +tower-http = { version = "0.6", features = ["compression-full", "decompression-full", "tracing"] } tracing = "0.1" tracing-opentelemetry = "0.25" tracing-subscriber = { version = "0.3", features = ["env-filter"] } @@ -94,4 +94,4 @@ urlencoding = "2.1" indicatif = "0.17" [patch.crates-io] -prusto = { git = "https://github.com/sbernauer/prusto.git", branch = "feat/derive-name-attribute" } +prusto = { git = "https://github.com/sbernauer/prusto.git", branch = "feat/derive-name-attribute-2" } diff --git a/README.md b/README.md index 8344df8..649cf6b 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A highly available load balancer with support for queueing, routing and auto-sca * [Postgres](./docs/persistence/postgres.md) * [Scaling](./docs/scaling/index.md) * [Stackable](./docs/scaling/stackable.md) +* [Proxy modes](./docs/proxy-modes.md) ## Try it out locally diff --git a/deploy/helm/trino-lb/configs/trino-lb-config.yaml b/deploy/helm/trino-lb/configs/trino-lb-config.yaml index fb5b92e..42d249a 100644 --- a/deploy/helm/trino-lb/configs/trino-lb-config.yaml +++ b/deploy/helm/trino-lb/configs/trino-lb-config.yaml @@ -1,9 +1,9 @@ trinoLb: - externalAddress: https://5.250.179.64:8443 + externalAddress: https://85.215.178.111:8443 tls: enabled: true - certPemFile: /certificates/cert.pem - keyPemFile: /certificates/key.pem + certPemFile: /certificates/tls.crt + keyPemFile: /certificates/tls.key persistence: redis: clusterMode: true @@ -12,6 +12,7 @@ trinoLb: # # helm install postgres bitnami/postgresql --version 13.2.18 --set auth.username=trino-lb,auth.password=trino-lb,auth.database=trino_lb # url: postgres://trino-lb:trino-lb@postgres-postgresql.default.svc.cluster.local/trino_lb # maxConnections: 10 + proxyMode: ProxyFirstCall # ProxyFirstCall refreshQueryCounterInterval: 30s # It is low for testing purpose and to get frequent running queries metrics tracing: enabled: true @@ -101,7 +102,7 @@ routers: routingFallback: m clusterAutoscaler: - reconcileInterval: 1s + # reconcileInterval: 1s stackable: clusters: trino-s-1: diff --git a/deploy/helm/trino-lb/templates/deployment.yaml b/deploy/helm/trino-lb/templates/deployment.yaml index 0672f7d..8eb8ffd 100644 --- a/deploy/helm/trino-lb/templates/deployment.yaml +++ b/deploy/helm/trino-lb/templates/deployment.yaml @@ -22,6 +22,9 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["trino-lb", "--config-file", "/etc/stackable/trino-lb/config/trino-lb-config.yaml"] + # env: + # - name: RUST_LOG + # value: debug ports: - containerPort: 8443 - containerPort: 9090 @@ -36,7 +39,22 @@ spec: - name: config secret: secretName: {{ .Release.Name }}-config + # - name: certificates + # secret: + # secretName: {{ .Release.Name }}-certificates - name: certificates - secret: - secretName: {{ .Release.Name }}-certificates + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,service=trino-lb + secrets.stackable.tech/format: tls-pem + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" serviceAccountName: {{ .Release.Name }} diff --git a/deploy/helm/trino-lb/templates/trinos.yaml b/deploy/helm/trino-lb/templates/trinos.yaml index f4391e0..3ba3311 100644 --- a/deploy/helm/trino-lb/templates/trinos.yaml +++ b/deploy/helm/trino-lb/templates/trinos.yaml @@ -1,10 +1,10 @@ -# 1 small cluster -{{- range untilStep 1 2 1 }} +{{- $trinos := list "trino-s-1" "trino-m-1" "trino-m-2" }} +{{- range $trinos }} --- apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCluster metadata: - name: trino-s-{{ . }} + name: {{ . }} namespace: default spec: image: @@ -21,46 +21,47 @@ spec: config.properties: tracing.enabled: "true" tracing.exporter.endpoint: http://jaeger-collector.default.svc.cluster.local:4317 + event-listener.config-files: /tmp/http-event-listener.properties + podOverrides: &podOverrides + spec: + containers: + - name: trino + volumeMounts: + - name: http-event-listener-config + mountPath: /tmp/http-event-listener.properties + subPath: http-event-listener.properties + volumes: + - name: http-event-listener-config + configMap: + name: {{ . }}-http-event-listener-config roleGroups: default: replicas: 1 workers: configOverrides: *configOverrides + podOverrides: *podOverrides roleGroups: default: replicas: 1 -{{- end }} -# 2 medium clusters -{{- range untilStep 1 3 1 }} --- -apiVersion: trino.stackable.tech/v1alpha1 -kind: TrinoCluster +apiVersion: v1 +kind: ConfigMap metadata: - name: trino-m-{{ . }} + name: {{ . }}-http-event-listener-config namespace: default -spec: - image: - productVersion: "455" - clusterConfig: - catalogLabelSelector: - matchLabels: - trino: trino - listenerClass: external-stable - authentication: - - authenticationClass: trino-users - coordinators: - configOverrides: &configOverrides - config.properties: - tracing.enabled: "true" - tracing.exporter.endpoint: http://jaeger-collector.default.svc.cluster.local:4317 - roleGroups: - default: - replicas: 1 - workers: - configOverrides: *configOverrides - roleGroups: - default: - replicas: 1 +data: + http-event-listener.properties: | + event-listener.name=http + + http-event-listener.connect-ingest-uri=https://trino-lb.trino-lb.svc.cluster.local:8443/v1/trino-event-listener + http-event-listener.connect-retry-count=10 + # http-event-listener.connect-http-headers="Trino-Cluster:{{ . }}" # IS BROKEN! + http-event-listener.http-client.trust-store-path=/stackable/server_tls/truststore.p12 + http-event-listener.http-client.trust-store-password=changeit + + http-event-listener.log-created=true + http-event-listener.log-completed=true + http-event-listener.log-split=false {{- end }} --- apiVersion: v1 diff --git a/docs/design.md b/docs/design.md index fa01f55..155086e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -94,6 +94,12 @@ Currently the following autoscalers are implemented: Read on the [scaling page](./scaling/index.md) for more details. +## 6. Proxy modes + +trino-lb can be configured to either proxy all calls to the underlying Trino clusters or only the initial `POST` request and instruct the client to connect directly to the Trino cluster for the subsequent polling requests. + +Read on the [proxy modes page](./proxy-modes.md) for more details. + ## Monitoring trino-lb emits [OpenTelemetry Metrics](https://opentelemetry.io/docs/concepts/signals/metrics/), which (for now) are only exposed as [Prometheus](https://prometheus.io/) metrics on `http://0.0.0.0:9090/metrics`. @@ -107,6 +113,7 @@ Also, the cluster `trino-s-1` was started on demand, executed the 60 queries and ![Grafana screenshot](./assets/grafana-screenshot.png) ## Tracing + trino-lb emits [OpenTelemetry Traces](https://opentelemetry.io/docs/concepts/signals/traces/) to [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) endpoints such as [Jaeger](https://www.jaegertracing.io/). When proxy-ing requests to Trino we take care of [OpenTelemetry Propagation](https://opentelemetry.io/docs/instrumentation/js/propagation/), so that the Trino spans will show up within the trino-lb spans. This enables nice tracing across trino-lb and trino (as seen in the screenshot below) @@ -118,6 +125,40 @@ This enables nice tracing across trino-lb and trino (as seen in the screenshot b This flowchart represents a Trino client submitting a query. It might be send to a Trino clusters or queued if all clusters are full. +### General flow + +```mermaid +sequenceDiagram + actor client as Trino client + participant lb as trino-lb + participant trino as Trino + + client ->>+ lb: Submit query using
POST /v1/statement + lb ->> lb: Determine if query should be queued
or which Trino cluster it should be routed to.
See below diagram for details + lb ->>- client: # Don't strip space + + loop While queued + client ->>+ lb: Poll + lb ->>- client: # Don't strip space + end + + alt Proxy mode: Proxy all calls + loop While query running + client ->>+ lb: Poll + lb -->+ trino: # Don't strip space + trino ->>- lb: # Don't strip space + lb ->>- client: # Don<>'t strip space + end + else Proxy mode: Proxy first call + loop While query running + client ->>+ trino: Poll + trino ->>- client: # Don't strip space + end + end +``` + +### Detailed initial `POST /v1/statement` + ```mermaid sequenceDiagram actor client as Trino client @@ -141,7 +182,7 @@ sequenceDiagram lb ->> lb: Determine cluster with the fewest running queries - alt All active clusters full + alt All active clusters reached query limit or all clusters unhealthy lb ->>+ persistence: Store queued query persistence ->>- lb: # Don't strip space @@ -156,7 +197,7 @@ sequenceDiagram lb ->>+ persistence: Store query persistence ->>- lb: # Don't strip space - lb ->> lb: Change nextUri to point to trino-lb + lb ->> lb: Change nextUri to point to trino-lb in case all requests should be proxied end lb ->>- client: Response containing nextUri diff --git a/docs/proxy-modes.md b/docs/proxy-modes.md new file mode 100644 index 0000000..e6085fa --- /dev/null +++ b/docs/proxy-modes.md @@ -0,0 +1,105 @@ +# Proxy mode + +trino-lb can be configured to either proxy all calls to the underlying Trino clusters or only the initial `POST` request and instruct the client to connect directly to the Trino cluster for the subsequent polling requests. +It also needs to keep track of all started and finished queries on the Trino clusters, so that it can correctly calculate the number of running queries. + +You can configure the proxy mode using + +```yaml +trinoLb: + proxyMode: ProxyAllCalls # or ProxyFirstCall, defaults to ProxyAllCalls +``` + +## Proxy all calls (default) + +In this mode, the client will make all requests through trino-lb, not only the initial `POST`. + +Benefits: + +- Counting queries can be achieved by inspecting the traffic +- Trino clients do not require network access to coordinator + +Downsides: + +- Increased query run times when a lot of data is transferred from the Trino coordinator to the Trino client due to network delay added by trino-lb (see the [performance research task](https://github.com/stackabletech/trino-lb/issues/72) for details) + +## Proxy first call + +In this mode, the client only sends the initial `POST` request to trino-lb. All following requests will be send to the Trino cluster directly. + +As trino-lb cannot inspect the traffic in the subsequent calls, it would have no knowledge of the started and finished queries. However, an [HTTP event listener](https://trino.io/docs/current/admin/event-listeners-http.html) can be configured in Trino to inform trino-lb about all query starts and completions. + +Benefits: + +- Better performance, as there is no network delay added by trino-lb +- In the future more advanced features can be built based on information from the Trino events + +Downsides: + +- It requires active configuration on the Trino side, namely setting up the HTTP event listener +- Trino clients require network access to the coordinator + +A sample configuration in Trino can look something like the following. +Please have a look at the [kuttl tests](https://github.com/stackabletech/trino-lb/tree/main/tests/templates/kuttl/) for complete examples. + +Please note that you cannot disable the TLS certificate check. In the example below, the secret-operator from the Stackable Data Platform is used to provision valid TLS certificates automatically. + +Please also note that you need to use the FQDN for the configured Trino clusters in your trino-lb config. +This is needed, because we need to determine which Trino cluster send the Trino HTTP event based on the `uri`. +If you get Trino HTTP events from a Trino cluster trino-lb does not know about, you can give Trino clusters a list of alternative names it might be using: + +```yaml +trinoClusterGroups: + default: + maxRunningQueries: 1 + trinoClusters: + - name: my-trino-cluster + endpoint: https://1.2.3.4:8443 + alternativeHostnames: + - my-trino-cluster.internal.corp + - trino-s-1-coordinator-default-0.trino-s-1-coordinator-default.default.svc.cluster.local +``` + +```yaml +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: my-trino +spec: + coordinators: + configOverrides: &configOverrides + config.properties: + event-listener.config-files: /tmp/http-event-listener.properties + podOverrides: &podOverrides + spec: + containers: + - name: trino + volumeMounts: + - name: http-event-listener-config + mountPath: /tmp/http-event-listener.properties + subPath: http-event-listener.properties + volumes: + - name: http-event-listener-config + configMap: + name: trino-http-event-listener-config + workers: + configOverrides: *configOverrides + podOverrides: *podOverrides +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trino-http-event-listener-config +data: + http-event-listener.properties: | + event-listener.name=http + + http-event-listener.connect-ingest-uri=https://trino-lb.trino-lb.svc.cluster.local:8443/v1/trino-event-listener + http-event-listener.connect-retry-count=10 + http-event-listener.http-client.trust-store-path=/stackable/server_tls/truststore.p12 + http-event-listener.http-client.trust-store-password=changeit + + http-event-listener.log-created=true + http-event-listener.log-completed=true + http-event-listener.log-split=false +``` diff --git a/scripts/run-tests b/scripts/run-tests new file mode 100755 index 0000000..4d13827 --- /dev/null +++ b/scripts/run-tests @@ -0,0 +1,409 @@ +#!/usr/bin/env python +# vim: filetype=python syntax=python tabstop=4 expandtab + +# WARNING +# No not modify this file, the source of truth is +# https://github.com/stackabletech/operator-templating/blob/main/template/scripts/run-tests +# WARNING + +import argparse +import collections.abc +import contextlib +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile + +__version__ = "0.0.1" + +DESCRIPTION = """ +Run integration tests. Call this script from the root of the repository. + +Exits with 0 on success, 1 on failure. + +Requires the following commands to be installed: +* beku +* stackablectl +* kubectl +* kubectl-kuttl + +Examples: + +1. Install operators, run all tests and clean up test namespaces: + + ./scripts/run-tests --parallel 4 + +2. Install operators but for Airflow use version "0.0.0-pr123" instead of "0.0.0-dev" and run all tests as above: + + ./scripts/run-tests --operator airflow=0.0.0-pr123 --parallel 4 + +3. Do not install any operators, run the smoke test suite and keep namespace: + + ./scripts/run-tests --skip-release --skip-delete --test-suite smoke-latest + +4. Run the ldap test(s) from the openshift test suite and keep namespace: + + ./scripts/run-tests --skip-release --skip-delete --test-suite openshift --test ldap + +5. Run the smoke test suite in the namespace "smoke". The namespace will be + created if it doesn't exist and will not be deleted when the tests end. + + ./scripts/run-tests --test-suite smoke-latest --namespace smoke +""" + + +class TestRunnerException(Exception): + pass + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse command line args.""" + parser = argparse.ArgumentParser( + description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--version", + help="Display application version", + action="version", + version=f"%(prog)s {__version__}", + ) + + parser.add_argument( + "--skip-delete", + help="Do not delete test namespaces.", + action="store_true", + ) + + parser.add_argument( + "--skip-tests", + help="Do not actually run the tests.", + action="store_true", + ) + + parser.add_argument( + "--skip-release", + help="Do not install operators.", + action="store_true", + ) + + parser.add_argument( + "--parallel", + help="How many tests to run in parallel. Default 2.", + type=int, + required=False, + default=2, + ) + + parser.add_argument( + "--operator", + help="Patch operator version in release.yaml. Format =", + action="append", + type=cli_parse_operator_args, + default=[], + ) + + parser.add_argument( + "--skip-operator", + help="Skip given operator(s) when installing a release.", + action="append", + default=[], + ) + + parser.add_argument( + "--test", + help="Kuttl test to run.", + type=str, + required=False, + ) + + parser.add_argument( + "--test-suite", + help="Name of the test suite to expand. Default: default", + type=str, + required=False, + ) + + parser.add_argument( + "--log-level", + help="Set log level.", + type=cli_log_level, + required=False, + default=logging.INFO, + ) + + parser.add_argument( + "--namespace", + help="Namespace to run the tests in. It will be created if it doesn't already exist.", + type=str, + required=False, + ) + + return parser.parse_args(argv) + + +def cli_parse_operator_args(args: str) -> tuple[str, str]: + if "=" not in args: + raise argparse.ArgumentTypeError( + f"Invalid operator argument: {args}. Must be in format =" + ) + op, version = args.split("=", maxsplit=1) + return op, version + + +def cli_log_level(cli_arg: str) -> int: + match cli_arg: + case "debug": + return logging.DEBUG + case "info": + return logging.INFO + case "error": + return logging.ERROR + case "warning": + return logging.WARNING + case "critical": + return logging.CRITICAL + case _: + raise argparse.ArgumentTypeError("Invalid log level") + + +def have_requirements() -> None: + commands = [ + ("beku", "https://github.com/stackabletech/beku.py"), + ( + "stackablectl", + "https://github.com/stackabletech/stackable-cockpit/blob/main/rust/stackablectl/README.md", + ), + ("kubectl", "https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/"), + ("kubectl-kuttl", "https://kuttl.dev/"), + ] + + err = False + for command, url in commands: + if not shutil.which(command): + logging.error(f'Command "{command}" not found, please install from {url}') + err = True + if err: + raise TestRunnerException() + + +@contextlib.contextmanager +def release_file( + operators: list[tuple[str, str]], skip_ops: list[str] +) -> collections.abc.Generator[str, None, None]: + """Generate a (possibly modified) copy of the release.yaml file. + + Operator versions passed as --operator take precedence over the release.yaml contents. + + Operators passed as --skip-operator are excluded from the resulting release.yaml contents. + + If an invalid operator name is provided (i.e. one that doesn't exist in the + original release file), a TestRunnerException is raised. + + Yields the name of the (potentially patched) release file. This is a temporary + file that will be deleted when the context manager exits. + """ + + def _patch(): + release_file = os.path.join("tests", "release.yaml") + # A marker to validate that all ops were patched + patched_release = [] + with open(release_file, "r") as f: + patched_ops = [] + patch_version = "" + for line in f: + if patch_version: + line = re.sub(":.+$", f": {patch_version}", line) + patch_version = "" + else: + for op, version in operators: + if op in line: + patch_version = version + patched_ops.append(op) + break + patched_release.append(line.rstrip("\n")) + + # Sanity test that cli didn't contain garbage that is silently discarded + ops_not_patched = set([op for op, _ in operators]) - set(patched_ops) + if ops_not_patched: + logging.error( + f"Patched operators [{', '.join(ops_not_patched)}] not found in {release_file}" + ) + raise TestRunnerException() + + # Filter out skip operators + release_contents = [] + skip_lines = 0 + valid_skip_ops = [] + for line in patched_release: + if skip_lines: + skip_lines -= 1 + continue + for op in skip_ops: + if op in line: + # Every product section has 1 line of additional config to skip + skip_lines = 1 + valid_skip_ops.append(op) + break + else: + release_contents.append(line) + # Sanity test that cli didn't contain garbage that is silently discarded + ops_not_skipped = set(skip_ops) - set(valid_skip_ops) + if ops_not_skipped: + logging.error( + f"Skipped operators [{', '.join(ops_not_skipped)}] not found in {release_file}" + ) + raise TestRunnerException() + + with tempfile.NamedTemporaryFile( + mode="w", + delete=False, + prefix="patched", + ) as f: + pcontents = "\n".join(release_contents) + logging.debug(f"Writing patched release to {f.name}: {pcontents}\n") + f.write(pcontents) + return f.name + + release_file = _patch() + try: + yield release_file + except TestRunnerException as e: + logging.error(f"Caught exception: {e}") + raise + finally: + if "patched" in release_file: + try: + logging.debug(f"Removing patched release file : {release_file}") + os.remove(release_file) + except FileNotFoundError | OSError: + logging.error(f"Failed to delete patched release file: {release_file}") + + +def maybe_install_release(skip_release: bool, release_file: str) -> None: + if skip_release: + logging.debug("Skip release installation") + return + stackablectl_err = "" + try: + stackablectl_cmd = [ + "stackablectl", + "release", + "install", + "--release-file", + release_file, + "tests", + ] + logging.debug(f"Running : {stackablectl_cmd}") + + completed_proc = subprocess.run( + stackablectl_cmd, + capture_output=True, + check=True, + ) + # stackablectl doesn't return a non-zero exit code on failure + # so we need to check stderr for errors + stackablectl_err = completed_proc.stderr.decode("utf-8") + if "error" in stackablectl_err.lower(): + logging.error(stackablectl_err) + logging.error("stackablectl failed") + raise TestRunnerException() + + except subprocess.CalledProcessError as e: + # in case stackablectl starts returning non-zero exit codes + logging.error(e.stderr.decode("utf-8")) + logging.error("stackablectl failed") + raise TestRunnerException() + + +def gen_tests(test_suite: str, namespace: str) -> None: + try: + beku_cmd = [ + "beku", + "--test_definition", + os.path.join("tests", "test-definition.yaml"), + "--kuttl_test", + os.path.join("tests", "kuttl-test.yaml.jinja2"), + "--template_dir", + os.path.join("tests", "templates", "kuttl"), + "--output_dir", + os.path.join("tests", "_work"), + ] + if test_suite: + beku_cmd.extend(["--suite", test_suite]) + if namespace: + beku_cmd.extend(["--namespace", namespace]) + + logging.debug(f"Running : {beku_cmd}") + subprocess.run( + beku_cmd, + check=True, + ) + except subprocess.CalledProcessError: + logging.error("beku failed") + raise TestRunnerException() + + +def run_tests(test: str, parallel: int, namespace: str, skip_delete: bool) -> None: + try: + kuttl_cmd = ["kubectl-kuttl", "test"] + if test: + kuttl_cmd.extend(["--test", test]) + if parallel: + kuttl_cmd.extend(["--parallel", str(parallel)]) + if skip_delete: + kuttl_cmd.extend(["--skip-delete"]) + if namespace: + kuttl_cmd.extend(["--namespace", namespace]) + # kuttl doesn't create the namespace so we need to do it ourselves + create_ns_cmd = ["kubectl", "create", "namespace", namespace] + try: + logging.debug(f"Running : {create_ns_cmd}") + subprocess.run( + create_ns_cmd, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as e: + stderr = e.stderr.decode("utf-8") + # If the namespace already exists, this will fail and we ignore the + # error. If it fails for any other reason, we raise an exception. + if "already exists" not in stderr: + logging.error(stderr) + logging.error("namespace creation failed") + raise TestRunnerException() + + logging.debug(f"Running : {kuttl_cmd}") + + subprocess.run( + kuttl_cmd, + cwd="tests/_work", + check=True, + ) + except subprocess.CalledProcessError: + logging.error("kuttl failed") + raise TestRunnerException() + + +def main(argv) -> int: + ret = 0 + try: + opts = parse_args(argv[1:]) + logging.basicConfig(encoding="utf-8", level=opts.log_level) + have_requirements() + gen_tests(opts.test_suite, opts.namespace) + with release_file(opts.operator, opts.skip_operator) as f: + maybe_install_release(opts.skip_release, f) + if opts.skip_tests: + logging.info("Skip running tests.") + else: + run_tests(opts.test, opts.parallel, opts.namespace, opts.skip_delete) + except TestRunnerException: + ret = 1 + return ret + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100755 index 0000000..a31a138 --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./scripts/run-tests "$@" diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..e96b7d6 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +_work/ diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2 new file mode 100644 index 0000000..8a5620c --- /dev/null +++ b/tests/kuttl-test.yaml.jinja2 @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: + {% for testcase in testinput.tests %} + - ./tests/{{ testcase.name }} + {% endfor %} + +startKIND: false +suppress: ["events"] +parallel: 2 + +# The timeout (in seconds) is used when namespaces are created or +# deleted, and, if not overridden, in TestSteps, TestAsserts, and +# Commands. If not set, the timeout is 30 seconds by default. +# +# The deletion of a namespace can take a while until all resources, +# especially PersistentVolumeClaims, are gracefully shut down. If the +# timeout is reached in the meantime, even a successful test case is +# considered a failure. +# +# For instance, the termination grace period of the Vector aggregator in +# the logging tests is set to 60 seconds. If there are logs entries +# which could not be forwarded yet to the external aggregator defined in +# the VECTOR_AGGREGATOR environment variable, then the test aggregator +# uses this period of time by trying to forward the events. In this +# case, deleting a namespace with several Pods takes about 90 seconds. +timeout: 300 diff --git a/tests/release.yaml b/tests/release.yaml new file mode 100644 index 0000000..4093b6e --- /dev/null +++ b/tests/release.yaml @@ -0,0 +1,16 @@ +# Contains all operators required to run the test suite. +--- +releases: + # Do not change the name of the release as it's referenced from run-tests + tests: + releaseDate: 1970-01-01 + description: Integration test + products: + commons: + operatorVersion: 0.0.0-dev + secret: + operatorVersion: 0.0.0-dev + listener: + operatorVersion: 0.0.0-dev + trino: + operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/smoke/00-assert.yaml.j2 b/tests/templates/kuttl/smoke/00-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/smoke/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 new file mode 100644 index 0000000..67185ac --- /dev/null +++ b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-rbac.yaml.j2 b/tests/templates/kuttl/smoke/00-rbac.yaml.j2 new file mode 100644 index 0000000..4abbf4f --- /dev/null +++ b/tests/templates/kuttl/smoke/00-rbac.yaml.j2 @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: trino-lb +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: trino-lb +subjects: + - kind: ServiceAccount + name: trino-lb +roleRef: + kind: Role + name: trino-lb + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: trino-lb + labels: + app.kubernetes.io/name: trino-lb +rules: + - apiGroups: + - trino.stackable.tech + resources: + - trinoclusters + verbs: + - get + - list + - watch + - patch + - apiGroups: + - trino.stackable.tech + resources: + - trinoclusters/status + verbs: + - get diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml new file mode 100644 index 0000000..7ebc7d9 --- /dev/null +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-s-1 --timeout 719s + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-m-1 --timeout 719s diff --git a/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 new file mode 100644 index 0000000..00aa754 --- /dev/null +++ b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: tpch + labels: + trino: trino +spec: + connector: + tpch: {} diff --git a/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 new file mode 100644 index 0000000..5db28ec --- /dev/null +++ b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 @@ -0,0 +1,107 @@ +{% for name in ['trino-s-1', 'trino-m-1'] %} +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: {{ name }} +spec: + image: +{% if test_scenario['values']['trino'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino + authentication: + - authenticationClass: trino-users-auth +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + resources: + cpu: + min: 250m + max: "1" + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: &configOverrides + config.properties: + event-listener.config-files: /tmp/http-event-listener.properties + # http-server.process-forwarded: "true" + podOverrides: &podOverrides + spec: + containers: + - name: trino + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: http-event-listener-config + mountPath: /tmp/http-event-listener.properties + subPath: http-event-listener.properties + volumes: + - name: http-event-listener-config + configMap: + name: trino-http-event-listener-config + roleGroups: + default: + replicas: 1 + workers: + config: + resources: + cpu: + min: 250m + max: "1" + gracefulShutdownTimeout: 60s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: *configOverrides + podOverrides: *podOverrides + roleGroups: + default: + replicas: 1 +{% endfor %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: trino-http-event-listener-config +data: + http-event-listener.properties: | + event-listener.name=http + + http-event-listener.connect-ingest-uri=https://trino-lb.${ENV:NAMESPACE}.svc.cluster.local:8443/v1/trino-event-listener + http-event-listener.connect-retry-count=10 + http-event-listener.http-client.trust-store-path=/stackable/server_tls/truststore.p12 + http-event-listener.http-client.trust-store-password=changeit + + http-event-listener.log-created=true + http-event-listener.log-completed=true + http-event-listener.log-split=false +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: trino-users-auth +spec: + provider: + static: + userCredentialsSecret: + name: trino-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: trino-users +stringData: + admin: adminadmin + alice: alicealice + bob: bobbob diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml new file mode 100644 index 0000000..4729c98 --- /dev/null +++ b/tests/templates/kuttl/smoke/20-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trino-lb +status: + replicas: 1 + readyReplicas: 1 diff --git a/tests/templates/kuttl/smoke/20-install-trino-lb.j2 b/tests/templates/kuttl/smoke/20-install-trino-lb.j2 new file mode 100644 index 0000000..b43a77e --- /dev/null +++ b/tests/templates/kuttl/smoke/20-install-trino-lb.j2 @@ -0,0 +1,89 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=20_trino-lb-config.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trino-lb +spec: + # I'm a bit surprised that 3 replicas work in combination with inMemory persistence :) + # Maybe Kubernetes is sticky for some reason, maybe trino-clients will retry call until they get + # to the correct trino-lb instance... + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: trino-lb + template: + metadata: + labels: + app.kubernetes.io/name: trino-lb + spec: + containers: + - name: trino-lb + image: {{ test_scenario['values']['trino-lb'] }} + imagePullPolicy: IfNotPresent + command: ["trino-lb", "--config-file", "/etc/stackable/trino-lb/config/trino-lb-config.yaml"] + # env: + # - name: RUST_LOG + # value: debug + ports: + - containerPort: 8080 + - containerPort: 8443 + - containerPort: 9090 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - mountPath: /etc/stackable/trino-lb/config/ + name: config + - mountPath: /certificates/ + name: certificates + volumes: + - name: config + secret: + secretName: trino-lb-config + - name: certificates + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,service=trino-lb + secrets.stackable.tech/format: tls-pem + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + serviceAccountName: trino-lb +--- +apiVersion: v1 +kind: Service +metadata: + name: trino-lb +spec: + type: NodePort # to access it for debugging purposes + selector: + app.kubernetes.io/name: trino-lb + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + name: http + - protocol: TCP + port: 8443 + targetPort: 8443 + name: https + - protocol: TCP + port: 9090 + targetPort: 9090 + name: metrics diff --git a/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 new file mode 100644 index 0000000..35d7de8 --- /dev/null +++ b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 @@ -0,0 +1,22 @@ +trinoLb: + externalAddress: https://trino-lb:8443 + tls: + enabled: true + certPemFile: /certificates/tls.crt + keyPemFile: /certificates/tls.key + persistence: + inMemory: {} +trinoClusterGroups: + default: + maxRunningQueries: 1 + trinoClusters: + - name: trino-s-1 + endpoint: https://trino-s-1-coordinator:8443 + credentials: + username: admin + password: adminadmin +trinoClusterGroupsIgnoreCert: true + +# Route all queries to the "default" cluster group +routers: [] +routingFallback: default diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml new file mode 100644 index 0000000..7f2cc57 --- /dev/null +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-queries +status: + succeeded: 1 diff --git a/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 new file mode 100644 index 0000000..f230d15 --- /dev/null +++ b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 @@ -0,0 +1,54 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-queries +spec: + template: + spec: + containers: + - name: test-queries + image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + - | + # Query Trinos and trino-lb + COORDINATORS=( + "https://trino-s-1-coordinator:8443" + "https://trino-m-1-coordinator:8443" + "https://trino-lb:8443" + ) + + export TRINO_USER="alice" + export TRINO_PASSWORD="alicealice" + QUERY="select count(*) from tpch.sf5.customer" + + for COORDINATOR in "${COORDINATORS[@]}"; do + echo "$QUERY" | java -jar trino-cli-executable.jar --server $COORDINATOR --insecure --user $TRINO_USER --password + done + + # Send multiple queries in parallel to trino-lb + NUM_REQUESTS=30 + TRINO_LB_ADDRESS="https://trino-lb:8443" + + pids=() + + for ((i = 1; i <= NUM_REQUESTS; i++)); do + echo "$QUERY" | java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user $TRINO_USER --password & + pids+=("$!") + done + + # Wait for all processes to complete and check exit codes + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + echo "One of the requests failed with a non-zero exit code." + exit 1 + fi + done + + echo "All queries completed successfully." + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/smoke/40-change-routing.j2 b/tests/templates/kuttl/smoke/40-change-routing.j2 new file mode 100644 index 0000000..5d08ea8 --- /dev/null +++ b/tests/templates/kuttl/smoke/40-change-routing.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE delete secret trino-lb-config || true + - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=40_trino-lb-config.yaml diff --git a/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 new file mode 100644 index 0000000..8914f77 --- /dev/null +++ b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 @@ -0,0 +1,52 @@ +trinoLb: + externalAddress: https://trino-lb:8443 + tls: + enabled: true + certPemFile: /certificates/tls.crt + keyPemFile: /certificates/tls.key + persistence: + inMemory: {} +trinoClusterGroups: + s: + maxRunningQueries: 1 + trinoClusters: + - name: trino-s-1 + endpoint: https://trino-s-1-coordinator:8443 + credentials: + username: admin + password: adminadmin + m: + maxRunningQueries: 1 + trinoClusters: + - name: trino-m-1 + endpoint: https://trino-m-1-coordinator:8443 + credentials: + username: admin + password: adminadmin +trinoClusterGroupsIgnoreCert: true + +routers: + - trinoRoutingGroupHeader: + headerName: X-Trino-Routing-Group + - clientTags: + oneOf: ["s"] + trinoClusterGroup: s + - clientTags: + oneOf: ["m"] + trinoClusterGroup: m + - pythonScript: + script: | + from typing import Optional + + def targetClusterGroup(query: str, headers: dict[str, str]) -> Optional[str]: + user = get_user(headers) + if user == "alice": + return "s" + elif user == "bob": + return "m" + else: + return None + + def get_user(headers: dict[str, str]) -> Optional[str]: + return headers.get("x-trino-user") +routingFallback: s diff --git a/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml new file mode 100644 index 0000000..9fc9de3 --- /dev/null +++ b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE rollout restart deployment trino-lb + - script: kubectl -n $NAMESPACE rollout status deployment trino-lb --watch diff --git a/tests/templates/kuttl/smoke/50-assert.yaml b/tests/templates/kuttl/smoke/50-assert.yaml new file mode 100644 index 0000000..32690b9 --- /dev/null +++ b/tests/templates/kuttl/smoke/50-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-routing +status: + succeeded: 1 diff --git a/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 new file mode 100644 index 0000000..deb69b1 --- /dev/null +++ b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-routing +spec: + template: + spec: + containers: + - name: test-routing + image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + - | + TRINO_LB_ADDRESS="https://trino-lb:8443" + COORDINATOR_NAME_QUERY="select regexp_extract(url_extract_host(http_uri), '[a-z0-9-]+') from "system".runtime.nodes where coordinator = true" + + # admin lands in s + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1 + # alice lands in s + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user alice --password | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1 + # bob lands in m + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=bobbob java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user bob --password | grep -qx '"trino-m-1-coordinator-default-0"' || exit 1 + + # We can also set client tags explicitly + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password --client-tags=s | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1 + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password --client-tags=m | grep -qx '"trino-m-1-coordinator-default-0"' || exit 1 + + echo "All queries completed successfully." + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/smoke/60-change-proxy-mode.j2 b/tests/templates/kuttl/smoke/60-change-proxy-mode.j2 new file mode 100644 index 0000000..0d9623a --- /dev/null +++ b/tests/templates/kuttl/smoke/60-change-proxy-mode.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE delete secret trino-lb-config || true + - script: envsubst '$NAMESPACE' < 60_trino-lb-config.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/smoke/60_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/60_trino-lb-config.yaml.j2 new file mode 100644 index 0000000..3870498 --- /dev/null +++ b/tests/templates/kuttl/smoke/60_trino-lb-config.yaml.j2 @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Secret +metadata: + name: trino-lb-config +stringData: + trino-lb-config.yaml: | + trinoLb: + externalAddress: https://trino-lb:8443 + tls: + enabled: true + certPemFile: /certificates/tls.crt + keyPemFile: /certificates/tls.key + persistence: + inMemory: {} + proxyMode: ProxyFirstCall + trinoClusterGroups: + default: + maxRunningQueries: 1 + trinoClusters: + - name: trino-s-1 + endpoint: https://trino-s-1-coordinator-default-0.trino-s-1-coordinator-default.${NAMESPACE}.svc.cluster.local:8443 + credentials: + username: admin + password: adminadmin + trinoClusterGroupsIgnoreCert: true + + # Route all queries to the "default" cluster group + routers: [] + routingFallback: default diff --git a/tests/templates/kuttl/smoke/61-restart-trino-lb.yaml b/tests/templates/kuttl/smoke/61-restart-trino-lb.yaml new file mode 100644 index 0000000..9fc9de3 --- /dev/null +++ b/tests/templates/kuttl/smoke/61-restart-trino-lb.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE rollout restart deployment trino-lb + - script: kubectl -n $NAMESPACE rollout status deployment trino-lb --watch diff --git a/tests/templates/kuttl/smoke/70-assert.yaml b/tests/templates/kuttl/smoke/70-assert.yaml new file mode 100644 index 0000000..0c2a8ec --- /dev/null +++ b/tests/templates/kuttl/smoke/70-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-proxy-mode +status: + succeeded: 1 diff --git a/tests/templates/kuttl/smoke/70-test-proxy-mode.yaml.j2 b/tests/templates/kuttl/smoke/70-test-proxy-mode.yaml.j2 new file mode 100644 index 0000000..c2341dd --- /dev/null +++ b/tests/templates/kuttl/smoke/70-test-proxy-mode.yaml.j2 @@ -0,0 +1,54 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-proxy-mode +spec: + template: + spec: + containers: + - name: test-proxy-mode + image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + - | + # Query Trinos and trino-lb + COORDINATORS=( + "https://trino-s-1-coordinator:8443" + "https://trino-m-1-coordinator:8443" + "https://trino-lb:8443" + ) + + export TRINO_USER="alice" + export TRINO_PASSWORD="alicealice" + QUERY="select count(*) from tpch.sf5.customer" + + for COORDINATOR in "${COORDINATORS[@]}"; do + echo "$QUERY" | java -jar trino-cli-executable.jar --server $COORDINATOR --insecure --user $TRINO_USER --password + done + + # Send multiple queries in parallel to trino-lb + NUM_REQUESTS=30 + TRINO_LB_ADDRESS="https://trino-lb:8443" + + pids=() + + for ((i = 1; i <= NUM_REQUESTS; i++)); do + echo "$QUERY" | java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user $TRINO_USER --password & + pids+=("$!") + done + + # Wait for all processes to complete and check exit codes + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + echo "One of the requests failed with a non-zero exit code." + exit 1 + fi + done + + echo "All queries completed successfully." + restartPolicy: OnFailure diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml new file mode 100644 index 0000000..bf556ca --- /dev/null +++ b/tests/test-definition.yaml @@ -0,0 +1,24 @@ +--- +dimensions: + - name: trino-lb + values: + - oci.stackable.tech/stackable/trino-lb:0.5.0 + # docker build -f docker/Dockerfile . -t oci.stackable.tech/sandbox/trino-lb:2025-04-08 + # - oci.stackable.tech/sandbox/trino-lb:2025-04-08 + - name: trino + values: + - "451" + - "455" + - "470" + # To use a custom image, add a comma and the full name after the product version + # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev + # However, watch out, you need to tweak the trino-cli image +tests: + - name: smoke + dimensions: + - trino-lb + - trino + +# TODOS +# 1. Test storage backend redis and postgres +# a. Also restart trino-lb deployment to make sure persistence is kept diff --git a/trino-lb-core/src/config.rs b/trino-lb-core/src/config.rs index 40ba782..58d1099 100644 --- a/trino-lb-core/src/config.rs +++ b/trino-lb-core/src/config.rs @@ -56,6 +56,9 @@ pub struct TrinoLbConfig { #[serde(default)] pub tls: TrinoLbTlsConfig, + #[serde(default)] + pub proxy_mode: TrinoLbProxyMode, + #[serde( default = "default_refresh_query_counter_interval", with = "humantime_serde" @@ -82,6 +85,14 @@ pub struct TrinoLbTlsConfig { pub key_pem_file: Option, } +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "PascalCase")] +pub enum TrinoLbProxyMode { + #[default] + ProxyAllCalls, + ProxyFirstCall, +} + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct TrinoLbTracingConfig { @@ -182,7 +193,12 @@ pub struct TrinoClusterGroupConfig { #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct TrinoClusterConfig { pub name: String, + + #[serde(default)] + pub alternative_hostnames: Vec, + pub endpoint: Url, + pub credentials: TrinoClusterCredentialsConfig, } diff --git a/trino-lb-core/src/trino_api/mod.rs b/trino-lb-core/src/trino_api/mod.rs new file mode 100644 index 0000000..18ea3f1 --- /dev/null +++ b/trino-lb-core/src/trino_api/mod.rs @@ -0,0 +1,3 @@ +pub mod queries; +pub mod stats; +pub mod trino_events; diff --git a/trino-lb-core/src/trino_api.rs b/trino-lb-core/src/trino_api/queries.rs similarity index 83% rename from trino-lb-core/src/trino_api.rs rename to trino-lb-core/src/trino_api/queries.rs index d03dcc5..22b5395 100644 --- a/trino-lb-core/src/trino_api.rs +++ b/trino-lb-core/src/trino_api/queries.rs @@ -7,11 +7,13 @@ use prusto::{QueryError, Warning}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use snafu::{ResultExt, Snafu}; -use tracing::instrument; +use tracing::{debug, instrument, warn}; use url::Url; use crate::{TrinoQueryId, trino_query::QueuedQuery}; +use super::stats::Stat; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("Failed to join API path onto trino-lb url {trino_lb_addr}"))] @@ -59,32 +61,6 @@ pub struct TrinoQueryApiResponse { pub update_count: Option, } -/// Copied from [`prusto::Stat`], but with `root_stage` -#[derive(Deserialize, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Stat { - pub completed_splits: u32, - pub cpu_time_millis: u64, - pub elapsed_time_millis: u64, - pub nodes: u32, - pub peak_memory_bytes: u64, - pub physical_input_bytes: u64, - pub processed_bytes: u64, - pub processed_rows: u64, - pub progress_percentage: Option, - pub queued_splits: u32, - pub queued_time_millis: u64, - pub queued: bool, - pub root_stage: Option, - pub running_percentage: Option, - pub running_splits: u32, - pub scheduled: bool, - pub spilled_bytes: u64, - pub state: String, - pub total_splits: u32, - pub wall_time_millis: u64, -} - impl TrinoQueryApiResponse { #[instrument( skip(query), @@ -156,22 +132,34 @@ impl TrinoQueryApiResponse { }) } - #[instrument( - skip(self), - fields(trino_lb_addr = %trino_lb_addr), - )] + #[instrument(skip(self, trino_lb_addr))] pub fn change_next_uri_to_trino_lb(&mut self, trino_lb_addr: &Url) -> Result<(), Error> { if let Some(next_uri) = &self.next_uri { let next_uri = Url::parse(next_uri).context(ParseNextUriFromTrinoSnafu)?; - self.next_uri = Some(change_next_uri_to_trino_lb(&next_uri, trino_lb_addr).to_string()); + self.next_uri = + Some(change_next_uri_to_base_addr(&next_uri, trino_lb_addr).to_string()); + + debug!(self.next_uri, "Changed nextUri to trino-lb"); + } + + Ok(()) + } + + #[instrument(skip(self, trino_addr))] + pub fn change_next_uri_to_trino(&mut self, trino_addr: &Url) -> Result<(), Error> { + if let Some(next_uri) = &self.next_uri { + let next_uri = Url::parse(next_uri).context(ParseNextUriFromTrinoSnafu)?; + self.next_uri = Some(change_next_uri_to_base_addr(&next_uri, trino_addr).to_string()); + + debug!(self.next_uri, "Changed nextUri to Trino"); } Ok(()) } } -fn change_next_uri_to_trino_lb(next_uri: &Url, trino_lb_addr: &Url) -> Url { - let mut result = trino_lb_addr.clone(); +fn change_next_uri_to_base_addr(next_uri: &Url, base_addr: &Url) -> Url { + let mut result = base_addr.clone(); result.set_path(next_uri.path()); result } @@ -207,7 +195,7 @@ mod tests { ) { let next_uri = Url::parse(&next_uri).unwrap(); let trino_lb_addr = Url::parse(&trino_lb_addr).unwrap(); - let result = change_next_uri_to_trino_lb(&next_uri, &trino_lb_addr); + let result = change_next_uri_to_base_addr(&next_uri, &trino_lb_addr); assert_eq!(result.to_string(), expected); } } diff --git a/trino-lb-core/src/trino_api/stats.rs b/trino-lb-core/src/trino_api/stats.rs new file mode 100644 index 0000000..1c97ca7 --- /dev/null +++ b/trino-lb-core/src/trino_api/stats.rs @@ -0,0 +1,27 @@ +use serde::{Deserialize, Serialize}; + +/// Copied from [`prusto::Stat`], but with `root_stage` +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Stat { + pub completed_splits: u32, + pub cpu_time_millis: u64, + pub elapsed_time_millis: u64, + pub nodes: u32, + pub peak_memory_bytes: u64, + pub physical_input_bytes: u64, + pub processed_bytes: u64, + pub processed_rows: u64, + pub progress_percentage: Option, + pub queued_splits: u32, + pub queued_time_millis: u64, + pub queued: bool, + pub root_stage: Option, + pub running_percentage: Option, + pub running_splits: u32, + pub scheduled: bool, + pub spilled_bytes: u64, + pub state: String, + pub total_splits: u32, + pub wall_time_millis: u64, +} diff --git a/trino-lb-core/src/trino_api/trino_events.rs b/trino-lb-core/src/trino_api/trino_events.rs new file mode 100644 index 0000000..f2c3803 --- /dev/null +++ b/trino-lb-core/src/trino_api/trino_events.rs @@ -0,0 +1,88 @@ +use std::fmt::Display; + +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::TrinoQueryId; + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrinoEvent { + pub metadata: TrinoEventMetadata, + pub context: TrinoEventContext, +} + +impl TrinoEvent { + pub fn query_state(&self) -> &TrinoQueryState { + &self.metadata.query_state + } + + pub fn query_id(&self) -> &TrinoQueryId { + &self.metadata.query_id + } + + pub fn uri(&self) -> &Url { + &self.metadata.uri + } + + pub fn server_address(&self) -> &str { + &self.context.server_address + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrinoEventMetadata { + pub uri: Url, + pub query_id: TrinoQueryId, + pub query_state: TrinoQueryState, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrinoEventContext { + /// We can not use [`url::Url`] or [`url::Host`] here, as we otherwise get + /// `Error("relative URL without a base: \"trino-m-1-coordinator-default-0.trino-m-1-coordinator-default.default.svc.cluster.local\"", line: 24, column: 110)` + pub server_address: String, + pub environment: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum TrinoQueryState { + Queued, + Executing, + Finished, +} + +impl Display for TrinoQueryState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Sticking to Trino casing for consistency + match self { + TrinoQueryState::Queued => write!(f, "QUEUED"), + TrinoQueryState::Executing => write!(f, "EXECUTING"), + TrinoQueryState::Finished => write!(f, "FINISHED"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deserialization() { + let query_crated: TrinoEvent = serde_json::from_str(include_str!( + "../../tests/sample_trino_events/query_created.json" + )) + .expect("Failed to deserialize query created event"); + assert_eq!(query_crated.query_id(), "20250328_101456_00000_gt85c"); + assert_eq!(query_crated.query_state(), &TrinoQueryState::Queued); + + let query_finished: TrinoEvent = serde_json::from_str(include_str!( + "../../tests/sample_trino_events/query_finished.json" + )) + .expect("Failed to deserialize query finished event"); + assert_eq!(query_finished.query_state(), &TrinoQueryState::Finished); + } +} diff --git a/trino-lb-core/tests/sample_trino_events/query_created.json b/trino-lb-core/tests/sample_trino_events/query_created.json new file mode 100644 index 0000000..d316244 --- /dev/null +++ b/trino-lb-core/tests/sample_trino_events/query_created.json @@ -0,0 +1,39 @@ +{ + "createTime": "2025-03-28T10:14:56.382Z", + "context": { + "user": "admin", + "originalUser": "admin", + "principal": "admin", + "enabledRoles": [], + "groups": [], + "remoteClientAddress": "100.103.68.12", + "userAgent": "StatementClientV1/447", + "clientTags": [], + "clientCapabilities": [ + "PATH", + "PARAMETRIC_DATETIME", + "SESSION_AUTHORIZATION" + ], + "source": "trino-cli", + "timezone": "Europe/Berlin", + "resourceGroupId": [ + "global" + ], + "sessionProperties": {}, + "resourceEstimates": {}, + "serverAddress": "trino-m-1-coordinator-default-0.trino-m-1-coordinator-default.default.svc.cluster.local", + "serverVersion": "455", + "environment": "trino_m_1", + "queryType": "SELECT", + "retryPolicy": "NONE" + }, + "metadata": { + "queryId": "20250328_101456_00000_gt85c", + "transactionId": "68e63b73-2b3a-4b21-be2d-b5750cad22ac", + "query": "select count(*) from tpch.sf1.customer", + "queryState": "QUEUED", + "tables": [], + "routines": [], + "uri": "http://trino-m-1-coordinator-default-0.trino-m-1-coordinator-default.default.svc.cluster.local:8080/v1/query/20250328_101456_00000_gt85c" + } +} diff --git a/trino-lb-core/tests/sample_trino_events/query_finished.json b/trino-lb-core/tests/sample_trino_events/query_finished.json new file mode 100644 index 0000000..2f22556 --- /dev/null +++ b/trino-lb-core/tests/sample_trino_events/query_finished.json @@ -0,0 +1,945 @@ +{ + "metadata": { + "queryId": "20250328_101456_00000_gt85c", + "transactionId": "68e63b73-2b3a-4b21-be2d-b5750cad22ac", + "query": "select count(*) from tpch.sf1.customer", + "queryState": "FINISHED", + "tables": [ + { + "catalog": "tpch", + "schema": "sf1", + "table": "customer", + "authorization": "admin", + "filters": [], + "columns": [], + "directlyReferenced": true, + "referenceChain": [] + } + ], + "routines": [ + { + "routine": "count", + "authorization": "admin" + } + ], + "uri": "http://trino-m-1-coordinator-default-0.trino-m-1-coordinator-default.default.svc.cluster.local:8080/v1/query/20250328_101456_00000_gt85c", + "plan": "Trino version: 455\nQueued: 67.94ms, Analysis: 294.95ms, Planning: 261.59ms, Execution: 711.31ms\nFragment 0 [SINGLE]\n CPU: 1.93ms, Scheduled: 2.01ms, Blocked 215.78ms (Input: 172.71ms, Output: 0.00ns), Input: 2 rows (18B); per task: avg.: 2.00 std.dev.: 0.00, Output: 1 row (9B)\n Output layout: [count]\n Output partitioning: SINGLE []\n Output[columnNames = [_col0]]\n │ Layout: [count:bigint]\n │ Estimates: {rows: 1 (9B), cpu: 0, memory: 0B, network: 0B}\n │ CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Blocked: 0.00ns (0.00%), Output: 1 row (9B)\n │ Input avg.: 1.00 rows, Input std.dev.: 0.00%\n │ _col0 := count\n └─ Aggregate[type = FINAL]\n │ Layout: [count:bigint]\n │ Estimates: {rows: 1 (9B), cpu: ?, memory: 9B, network: 0B}\n │ CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Blocked: 0.00ns (0.00%), Output: 1 row (9B)\n │ Input avg.: 2.00 rows, Input std.dev.: 0.00%\n │ count := count(count_0)\n └─ LocalExchange[partitioning = SINGLE]\n │ Layout: [count_0:bigint]\n │ Estimates: {rows: ? (?), cpu: 0, memory: 0B, network: 0B}\n │ CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Blocked: 43.00ms (19.91%), Output: 2 rows (18B)\n │ Input avg.: 0.50 rows, Input std.dev.: 100.00%\n └─ RemoteSource[sourceFragmentIds = [1]]\n Layout: [count_0:bigint]\n CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Blocked: 173.00ms (80.09%), Output: 2 rows (18B)\n Input avg.: 0.50 rows, Input std.dev.: 100.00%\n\nFragment 1 [SOURCE]\n CPU: 76.73ms, Scheduled: 76.79ms, Blocked 0.00ns (Input: 0.00ns, Output: 0.00ns), Input: 150000 rows (0B); per task: avg.: 150000.00 std.dev.: 0.00, Output: 2 rows (18B)\n Output layout: [count_0]\n Output partitioning: SINGLE []\n Aggregate[type = PARTIAL]\n │ Layout: [count_0:bigint]\n │ CPU: 1.00ms (1.30%), Scheduled: 1.00ms (1.30%), Blocked: 0.00ns (0.00%), Output: 2 rows (18B)\n │ Input avg.: 75000.00 rows, Input std.dev.: 0.00%\n │ count_0 := count(*)\n └─ TableScan[table = tpch:sf1:customer]\n Layout: []\n Estimates: {rows: ? (0B), cpu: 0, memory: 0B, network: 0B}\n CPU: 76.00ms (98.70%), Scheduled: 76.00ms (98.70%), Blocked: 0.00ns (0.00%), Output: 150000 rows (0B)\n Input avg.: 75000.00 rows, Input std.dev.: 0.00%\n Input: 150000 rows (0B)\n\n", + "jsonPlan": "{\n \"0\" : {\n \"id\" : \"9\",\n \"name\" : \"Output\",\n \"descriptor\" : {\n \"columnNames\" : \"[_col0]\"\n },\n \"outputs\" : [ {\n \"type\" : \"bigint\",\n \"name\" : \"count\"\n } ],\n \"details\" : [ \"_col0 := count\" ],\n \"estimates\" : [ {\n \"outputRowCount\" : 1.0,\n \"outputSizeInBytes\" : 9.0,\n \"cpuCost\" : 0.0,\n \"memoryCost\" : 0.0,\n \"networkCost\" : 0.0\n } ],\n \"children\" : [ {\n \"id\" : \"4\",\n \"name\" : \"Aggregate\",\n \"descriptor\" : {\n \"type\" : \"FINAL\",\n \"keys\" : \"\",\n \"hash\" : \"[]\"\n },\n \"outputs\" : [ {\n \"type\" : \"bigint\",\n \"name\" : \"count\"\n } ],\n \"details\" : [ \"count := count(count_0)\" ],\n \"estimates\" : [ {\n \"outputRowCount\" : 1.0,\n \"outputSizeInBytes\" : 9.0,\n \"cpuCost\" : \"NaN\",\n \"memoryCost\" : 9.0,\n \"networkCost\" : 0.0\n } ],\n \"children\" : [ {\n \"id\" : \"183\",\n \"name\" : \"LocalExchange\",\n \"descriptor\" : {\n \"partitioning\" : \"SINGLE\",\n \"isReplicateNullsAndAny\" : \"\",\n \"hashColumn\" : \"[]\",\n \"arguments\" : \"[]\"\n },\n \"outputs\" : [ {\n \"type\" : \"bigint\",\n \"name\" : \"count_0\"\n } ],\n \"details\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"outputSizeInBytes\" : \"NaN\",\n \"cpuCost\" : 0.0,\n \"memoryCost\" : 0.0,\n \"networkCost\" : 0.0\n } ],\n \"children\" : [ {\n \"id\" : \"189\",\n \"name\" : \"RemoteSource\",\n \"descriptor\" : {\n \"sourceFragmentIds\" : \"[1]\"\n },\n \"outputs\" : [ {\n \"type\" : \"bigint\",\n \"name\" : \"count_0\"\n } ],\n \"details\" : [ ],\n \"estimates\" : [ ],\n \"children\" : [ ]\n } ]\n } ]\n } ]\n },\n \"1\" : {\n \"id\" : \"187\",\n \"name\" : \"Aggregate\",\n \"descriptor\" : {\n \"type\" : \"PARTIAL\",\n \"keys\" : \"\",\n \"hash\" : \"[]\"\n },\n \"outputs\" : [ {\n \"type\" : \"bigint\",\n \"name\" : \"count_0\"\n } ],\n \"details\" : [ \"count_0 := count(*)\" ],\n \"estimates\" : [ ],\n \"children\" : [ {\n \"id\" : \"0\",\n \"name\" : \"TableScan\",\n \"descriptor\" : {\n \"table\" : \"tpch:sf1:customer\"\n },\n \"outputs\" : [ ],\n \"details\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"outputSizeInBytes\" : 0.0,\n \"cpuCost\" : 0.0,\n \"memoryCost\" : 0.0,\n \"networkCost\" : 0.0\n } ],\n \"children\" : [ ]\n } ]\n }\n}", + "payload": "{\"stageId\":\"20250328_101456_00000_gt85c.0\",\"state\":\"FINISHED\",\"plan\":{\"id\":\"0\",\"root\":{\"@type\":\"output\",\"id\":\"9\",\"source\":{\"@type\":\"aggregation\",\"id\":\"4\",\"source\":{\"@type\":\"exchange\",\"id\":\"183\",\"type\":\"GATHER\",\"scope\":\"LOCAL\",\"partitioningScheme\":{\"partitioning\":{\"handle\":{\"connectorHandle\":{\"@type\":\"system:io.trino.sql.planner.SystemPartitioningHandle\",\"partitioning\":\"SINGLE\",\"function\":\"SINGLE\"},\"scaleWriters\":false},\"arguments\":[]},\"outputLayout\":[{\"type\":\"bigint\",\"name\":\"count_0\"}],\"replicateNullsAndAny\":false},\"sources\":[{\"@type\":\"remoteSource\",\"id\":\"189\",\"sourceFragmentIds\":[\"1\"],\"outputs\":[{\"type\":\"bigint\",\"name\":\"count_0\"}],\"exchangeType\":\"GATHER\",\"retryPolicy\":\"NONE\"}],\"inputs\":[[{\"type\":\"bigint\",\"name\":\"count_0\"}]]},\"aggregations\":{\"Y291bnQ=:YmlnaW50\":{\"resolvedFunction\":{\"signature\":{\"name\":{\"catalogName\":\"system\",\"schemaName\":\"builtin\",\"functionName\":\"count\"},\"returnType\":\"bigint\",\"argumentTypes\":[]},\"catalogHandle\":\"system:normal:system\",\"functionId\":\"count():bigint\",\"functionKind\":\"AGGREGATE\",\"deterministic\":true,\"functionNullability\":{\"returnNullable\":true,\"argumentNullable\":[]},\"typeDependencies\":{},\"functionDependencies\":[]},\"arguments\":[{\"@type\":\"reference\",\"type\":\"bigint\",\"name\":\"count_0\"}],\"distinct\":false}},\"groupingSets\":{\"groupingKeys\":[],\"groupingSetCount\":1,\"globalGroupingSets\":[0]},\"preGroupedSymbols\":[],\"step\":\"FINAL\"},\"columns\":[\"_col0\"],\"outputs\":[{\"type\":\"bigint\",\"name\":\"count\"}]},\"symbols\":[{\"type\":\"bigint\",\"name\":\"count\"},{\"type\":\"bigint\",\"name\":\"count_0\"}],\"partitioning\":{\"connectorHandle\":{\"@type\":\"system:io.trino.sql.planner.SystemPartitioningHandle\",\"partitioning\":\"SINGLE\",\"function\":\"SINGLE\"},\"scaleWriters\":false},\"partitionedSources\":[],\"outputPartitioningScheme\":{\"partitioning\":{\"handle\":{\"connectorHandle\":{\"@type\":\"system:io.trino.sql.planner.SystemPartitioningHandle\",\"partitioning\":\"SINGLE\",\"function\":\"SINGLE\"},\"scaleWriters\":false},\"arguments\":[]},\"outputLayout\":[{\"type\":\"bigint\",\"name\":\"count\"}],\"replicateNullsAndAny\":false},\"statsAndCosts\":{\"stats\":{\"9\":{\"outputRowCount\":1.0,\"symbolStatistics\":{}},\"4\":{\"outputRowCount\":1.0,\"symbolStatistics\":{}},\"183\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}},\"189\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}}},\"costs\":{\"9\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":9.0,\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":0.0}},\"4\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":9.0,\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":\"NaN\",\"maxMemory\":9.0,\"networkCost\":0.0}},\"183\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":\"NaN\",\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":0.0}},\"189\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":\"NaN\",\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":\"NaN\"}}}},\"activeCatalogs\":[],\"languageFunctions\":{},\"jsonRepresentation\":\"{\\n \\\"id\\\" : \\\"9\\\",\\n \\\"name\\\" : \\\"Output\\\",\\n \\\"descriptor\\\" : {\\n \\\"columnNames\\\" : \\\"[_col0]\\\"\\n },\\n \\\"outputs\\\" : [ {\\n \\\"type\\\" : \\\"bigint\\\",\\n \\\"name\\\" : \\\"count\\\"\\n } ],\\n \\\"details\\\" : [ \\\"_col0 := count\\\" ],\\n \\\"estimates\\\" : [ ],\\n \\\"children\\\" : [ {\\n \\\"id\\\" : \\\"4\\\",\\n \\\"name\\\" : \\\"Aggregate\\\",\\n \\\"descriptor\\\" : {\\n \\\"type\\\" : \\\"FINAL\\\",\\n \\\"keys\\\" : \\\"\\\",\\n \\\"hash\\\" : \\\"[]\\\"\\n },\\n \\\"outputs\\\" : [ {\\n \\\"type\\\" : \\\"bigint\\\",\\n \\\"name\\\" : \\\"count\\\"\\n } ],\\n \\\"details\\\" : [ \\\"count := count(count_0)\\\" ],\\n \\\"estimates\\\" : [ ],\\n \\\"children\\\" : [ {\\n \\\"id\\\" : \\\"183\\\",\\n \\\"name\\\" : \\\"LocalExchange\\\",\\n \\\"descriptor\\\" : {\\n \\\"partitioning\\\" : \\\"SINGLE\\\",\\n \\\"isReplicateNullsAndAny\\\" : \\\"\\\",\\n \\\"hashColumn\\\" : \\\"[]\\\",\\n \\\"arguments\\\" : \\\"[]\\\"\\n },\\n \\\"outputs\\\" : [ {\\n \\\"type\\\" : \\\"bigint\\\",\\n \\\"name\\\" : \\\"count_0\\\"\\n } ],\\n \\\"details\\\" : [ ],\\n \\\"estimates\\\" : [ ],\\n \\\"children\\\" : [ {\\n \\\"id\\\" : \\\"189\\\",\\n \\\"name\\\" : \\\"RemoteSource\\\",\\n \\\"descriptor\\\" : {\\n \\\"sourceFragmentIds\\\" : \\\"[1]\\\"\\n },\\n \\\"outputs\\\" : [ {\\n \\\"type\\\" : \\\"bigint\\\",\\n \\\"name\\\" : \\\"count_0\\\"\\n } ],\\n \\\"details\\\" : [ ],\\n \\\"estimates\\\" : [ ],\\n \\\"children\\\" : [ ]\\n } ]\\n } ]\\n } ]\\n}\"},\"coordinatorOnly\":false,\"types\":[\"bigint\"],\"stageStats\":{\"schedulingComplete\":\"2025-03-28T10:14:58.029Z\",\"getSplitDistribution\":{\"count\":0.0,\"total\":0.0,\"p01\":\"NaN\",\"p05\":\"NaN\",\"p10\":\"NaN\",\"p25\":\"NaN\",\"p50\":\"NaN\",\"p75\":\"NaN\",\"p90\":\"NaN\",\"p95\":\"NaN\",\"p99\":\"NaN\",\"min\":\"NaN\",\"max\":\"NaN\",\"avg\":\"NaN\"},\"totalTasks\":1,\"runningTasks\":0,\"completedTasks\":1,\"failedTasks\":0,\"totalDrivers\":5,\"queuedDrivers\":0,\"runningDrivers\":0,\"blockedDrivers\":0,\"completedDrivers\":5,\"cumulativeUserMemory\":10072.344439999999,\"failedCumulativeUserMemory\":0.0,\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"totalMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"112B\",\"peakRevocableMemoryReservation\":\"0B\",\"totalScheduledTime\":\"2.01ms\",\"failedScheduledTime\":\"0.00s\",\"totalCpuTime\":\"1.93ms\",\"failedCpuTime\":\"0.00s\",\"totalBlockedTime\":\"215.78ms\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"failedPhysicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"failedPhysicalInputPositions\":0,\"physicalInputReadTime\":\"0.00s\",\"failedPhysicalInputReadTime\":\"0.00s\",\"internalNetworkInputDataSize\":\"88B\",\"failedInternalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":2,\"failedInternalNetworkInputPositions\":0,\"rawInputDataSize\":\"88B\",\"failedRawInputDataSize\":\"0B\",\"rawInputPositions\":2,\"failedRawInputPositions\":0,\"processedInputDataSize\":\"18B\",\"failedProcessedInputDataSize\":\"0B\",\"processedInputPositions\":2,\"failedProcessedInputPositions\":0,\"inputBlockedTime\":\"172.71ms\",\"failedInputBlockedTime\":\"0.00s\",\"bufferedDataSize\":\"0B\",\"outputBufferUtilization\":{\"digest\":\"AAAAAAAAAAAAAAAAAAAAyj4AAAAAAABZQAAAADA853pBBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADKPgAAAAAAAMo+AAAAwGPIYEEAAAAA9IJNQQAAAAAJVUJBAAAAYBUQaUE=\",\"min\":0.0,\"max\":3.0994415283203125E-6,\"p25\":0.0,\"p50\":3.0994415283203125E-6,\"p75\":3.0994415283203125E-6,\"p90\":3.0994415283203125E-6,\"p95\":3.0994415283203125E-6,\"p99\":3.0994415283203125E-6,\"total\":28210115,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"outputDataSize\":\"9B\",\"failedOutputDataSize\":\"0B\",\"outputPositions\":1,\"failedOutputPositions\":0,\"outputBlockedTime\":\"0.00s\",\"failedOutputBlockedTime\":\"0.00s\",\"physicalWrittenDataSize\":\"0B\",\"failedPhysicalWrittenDataSize\":\"0B\",\"gcInfo\":{\"stageId\":0,\"tasks\":1,\"fullGcTasks\":0,\"minFullGcSec\":0,\"maxFullGcSec\":0,\"totalFullGcSec\":0,\"averageFullGcSec\":0},\"operatorSummaries\":[{\"stageId\":0,\"pipelineId\":0,\"operatorId\":0,\"planNodeId\":\"189\",\"operatorType\":\"ExchangeOperator\",\"totalDrivers\":4,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"88B\",\"internalNetworkInputPositions\":2,\"rawInputDataSize\":\"88B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":2,\"getOutputWall\":\"207.91us\",\"getOutputCpu\":\"202.83us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.25256E-4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.04305554699999999,\"max\":0.043296167999999996,\"p01\":0.04305554699999999,\"p05\":0.04305554699999999,\"p10\":0.04305554699999999,\"p25\":0.04305554699999999,\"p50\":0.04305554699999999,\"p75\":0.04305554699999999,\"p90\":0.04305554699999999,\"p95\":0.04305554699999999,\"p99\":0.04305554699999999},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.2262999999999998E-4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"172.71ms\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"56B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"56B\",\"spilledDataSize\":\"0B\",\"info\":{\"@type\":\"directExchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":100,\"averageBytesPerRequest\":27,\"successfulRequestsCount\":12,\"bufferedPages\":0,\"spilledPages\":0,\"spilledBytes\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[],\"requestDuration\":{\"digest\":\"AAAAAAAAAPA/AAAAAAAAQ0AAAAAAAABZQAAAAAAAAAhAAwAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAABDQAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"min\":1.0,\"max\":38.0,\"p25\":1.0,\"p50\":1.0,\"p75\":38.0,\"p90\":38.0,\"p95\":38.0,\"p99\":38.0,\"total\":3,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0}}},{\"stageId\":0,\"pipelineId\":1,\"operatorId\":0,\"planNodeId\":\"183\",\"operatorType\":\"LocalExchangeSourceOperator\",\"totalDrivers\":1,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":4.0,\"getOutputCalls\":2,\"getOutputWall\":\"50.27us\",\"getOutputCpu\":\"44.53us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":5.027399999999999E-5,\"max\":5.027399999999999E-5,\"p01\":5.027399999999999E-5,\"p05\":5.027399999999999E-5,\"p10\":5.027399999999999E-5,\"p25\":5.027399999999999E-5,\"p50\":5.027399999999999E-5,\"p75\":5.027399999999999E-5,\"p90\":5.027399999999999E-5,\"p95\":5.027399999999999E-5,\"p99\":5.027399999999999E-5},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":0.043062728999999994,\"max\":0.043062728999999994,\"p01\":0.043062728999999994,\"p05\":0.043062728999999994,\"p10\":0.043062728999999994,\"p25\":0.043062728999999994,\"p50\":0.043062728999999994,\"p75\":0.043062728999999994,\"p90\":0.043062728999999994,\"p95\":0.043062728999999994,\"p99\":0.043062728999999994},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":4.453399999999999E-5,\"max\":4.453399999999999E-5,\"p01\":4.453399999999999E-5,\"p05\":4.453399999999999E-5,\"p10\":4.453399999999999E-5,\"p25\":4.453399999999999E-5,\"p50\":4.453399999999999E-5,\"p75\":4.453399999999999E-5,\"p90\":4.453399999999999E-5,\"p95\":4.453399999999999E-5,\"p99\":4.453399999999999E-5},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.0,\"max\":2.0,\"p01\":2.0,\"p05\":2.0,\"p10\":2.0,\"p25\":2.0,\"p50\":2.0,\"p75\":2.0,\"p90\":2.0,\"p95\":2.0,\"p99\":2.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"43.06ms\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"},{\"stageId\":0,\"pipelineId\":1,\"operatorId\":2,\"planNodeId\":\"9\",\"operatorType\":\"TaskOutputOperator\",\"totalDrivers\":1,\"addInputCalls\":1,\"addInputWall\":\"187.13us\",\"addInputCpu\":\"186.74us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"9B\",\"inputPositions\":1,\"sumSquaredInputPositions\":1.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":1.8942599999999997E-4,\"max\":1.8942599999999997E-4,\"p01\":1.8942599999999997E-4,\"p05\":1.8942599999999997E-4,\"p10\":1.8942599999999997E-4,\"p25\":1.8942599999999997E-4,\"p50\":1.8942599999999997E-4,\"p75\":1.8942599999999997E-4,\"p90\":1.8942599999999997E-4,\"p95\":1.8942599999999997E-4,\"p99\":1.8942599999999997E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":1.8957499999999996E-4,\"max\":1.8957499999999996E-4,\"p01\":1.8957499999999996E-4,\"p05\":1.8957499999999996E-4,\"p10\":1.8957499999999996E-4,\"p25\":1.8957499999999996E-4,\"p50\":1.8957499999999996E-4,\"p75\":1.8957499999999996E-4,\"p90\":1.8957499999999996E-4,\"p95\":1.8957499999999996E-4,\"p99\":1.8957499999999996E-4},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":1.0,\"max\":1.0,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0,\"p25\":1.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":1,\"finishWall\":\"2.29us\",\"finishCpu\":\"2.84us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"64B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"64B\",\"spilledDataSize\":\"0B\"},{\"stageId\":0,\"pipelineId\":0,\"operatorId\":1,\"planNodeId\":\"183\",\"operatorType\":\"LocalExchangeSinkOperator\",\"totalDrivers\":4,\"addInputCalls\":2,\"addInputWall\":\"71.90us\",\"addInputCpu\":\"72.57us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":3.602699999999999E-5,\"max\":6.705499999999999E-5,\"p01\":3.602699999999999E-5,\"p05\":3.602699999999999E-5,\"p10\":3.602699999999999E-5,\"p25\":3.602699999999999E-5,\"p50\":3.602699999999999E-5,\"p75\":3.602699999999999E-5,\"p90\":3.602699999999999E-5,\"p95\":3.602699999999999E-5,\"p99\":3.602699999999999E-5},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":3.2099999999999994E-5,\"max\":6.4351E-5,\"p01\":3.2099999999999994E-5,\"p05\":3.2099999999999994E-5,\"p10\":3.2099999999999994E-5,\"p25\":3.2099999999999994E-5,\"p50\":3.2099999999999994E-5,\"p75\":3.2099999999999994E-5,\"p90\":3.2099999999999994E-5,\"p95\":3.2099999999999994E-5,\"p99\":3.2099999999999994E-5},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":4,\"finishWall\":\"151.01us\",\"finishCpu\":\"140.01us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"},{\"stageId\":0,\"pipelineId\":1,\"operatorId\":1,\"planNodeId\":\"4\",\"operatorType\":\"AggregationOperator\",\"totalDrivers\":1,\"addInputCalls\":2,\"addInputWall\":\"154.96us\",\"addInputCpu\":\"136.16us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":4.0,\"getOutputCalls\":6,\"getOutputWall\":\"84.45us\",\"getOutputCpu\":\"81.76us\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.5230399999999996E-4,\"max\":2.5230399999999996E-4,\"p01\":2.5230399999999996E-4,\"p05\":2.5230399999999996E-4,\"p10\":2.5230399999999996E-4,\"p25\":2.5230399999999996E-4,\"p50\":2.5230399999999996E-4,\"p75\":2.5230399999999996E-4,\"p90\":2.5230399999999996E-4,\"p95\":2.5230399999999996E-4,\"p99\":2.5230399999999996E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.2891999999999997E-4,\"max\":2.2891999999999997E-4,\"p01\":2.2891999999999997E-4,\"p05\":2.2891999999999997E-4,\"p10\":2.2891999999999997E-4,\"p25\":2.2891999999999997E-4,\"p50\":2.2891999999999997E-4,\"p75\":2.2891999999999997E-4,\"p90\":2.2891999999999997E-4,\"p95\":2.2891999999999997E-4,\"p99\":2.2891999999999997E-4},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.0,\"max\":2.0,\"p01\":2.0,\"p05\":2.0,\"p10\":2.0,\"p25\":2.0,\"p50\":2.0,\"p75\":2.0,\"p90\":2.0,\"p95\":2.0,\"p99\":2.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":1,\"finishWall\":\"12.89us\",\"finishCpu\":\"11.01us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"24B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"24B\",\"spilledDataSize\":\"0B\"}]},\"tasks\":[{\"taskStatus\":{\"taskId\":\"20250328_101456_00000_gt85c.0.0.0\",\"taskInstanceId\":\"8a40e424-c24d-47ed-b070-320daf3b6d4e\",\"version\":2,\"state\":\"FINISHED\",\"self\":\"http://trino-m-1-worker-default-0.trino-m-1-worker-default.default.svc.cluster.local:8080/v1/task/20250328_101456_00000_gt85c.0.0.0\",\"nodeId\":\"b81fc056-0921-4e5a-9773-2aea9121d298\",\"speculative\":false,\"failures\":[],\"queuedPartitionedDrivers\":0,\"runningPartitionedDrivers\":0,\"outputBufferStatus\":{\"outputBuffersVersion\":2,\"overutilized\":false,\"exchangeSinkInstanceHandleUpdateRequired\":false},\"outputDataSize\":\"9B\",\"writerInputDataSize\":\"0B\",\"physicalWrittenDataSize\":\"0B\",\"memoryReservation\":\"0B\",\"peakMemoryReservation\":\"288B\",\"revocableMemoryReservation\":\"0B\",\"fullGcCount\":0,\"fullGcTime\":\"0.00ns\",\"dynamicFiltersVersion\":0,\"queuedPartitionedSplitsWeight\":0,\"runningPartitionedSplitsWeight\":0},\"lastHeartbeat\":\"2025-03-28T10:14:58.293Z\",\"outputBuffers\":{\"type\":\"PARTITIONED\",\"state\":\"FINISHED\",\"canAddBuffers\":false,\"canAddPages\":false,\"totalBufferedBytes\":0,\"totalBufferedPages\":0,\"totalRowsSent\":1,\"totalPagesSent\":1,\"utilization\":{\"digest\":\"AAAAAAAAAAAAAAAAAAAAyj4AAAAAAABZQAAAADA853pBBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADKPgAAAAAAAMo+AAAAwGPIYEEAAAAA9IJNQQAAAAAJVUJBAAAAYBUQaUE=\",\"min\":0.0,\"max\":3.0994415283203125E-6,\"p25\":0.0,\"p50\":3.0994415283203125E-6,\"p75\":3.0994415283203125E-6,\"p90\":3.0994415283203125E-6,\"p95\":3.0994415283203125E-6,\"p99\":3.0994415283203125E-6,\"total\":28210115,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0}},\"noMoreSplits\":[\"189\"],\"stats\":{\"createTime\":\"2025-03-28T10:14:58.034Z\",\"firstStartTime\":\"2025-03-28T10:14:58.230Z\",\"lastStartTime\":\"2025-03-28T10:14:58.231Z\",\"lastEndTime\":\"2025-03-28T10:14:58.275Z\",\"endTime\":\"2025-03-28T10:14:58.299Z\",\"elapsedTime\":\"70.50ms\",\"queuedTime\":\"1.79ms\",\"totalDrivers\":5,\"queuedDrivers\":0,\"queuedPartitionedDrivers\":0,\"queuedPartitionedSplitsWeight\":0,\"runningDrivers\":0,\"runningPartitionedDrivers\":0,\"runningPartitionedSplitsWeight\":0,\"blockedDrivers\":0,\"completedDrivers\":5,\"cumulativeUserMemory\":10072.344439999999,\"userMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"288B\",\"revocableMemoryReservation\":\"0B\",\"totalScheduledTime\":\"2.01ms\",\"totalCpuTime\":\"1.93ms\",\"totalBlockedTime\":\"215.78ms\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"88B\",\"internalNetworkInputPositions\":2,\"rawInputDataSize\":\"88B\",\"rawInputPositions\":2,\"processedInputDataSize\":\"18B\",\"processedInputPositions\":2,\"inputBlockedTime\":\"172.71ms\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"outputBlockedTime\":\"0.00ns\",\"writerInputDataSize\":\"0B\",\"physicalWrittenDataSize\":\"0B\",\"fullGcCount\":0,\"fullGcTime\":\"0.00ns\",\"pipelines\":[{\"pipelineId\":0,\"firstStartTime\":\"2025-03-28T10:14:58.230Z\",\"lastStartTime\":\"2025-03-28T10:14:58.231Z\",\"lastEndTime\":\"2025-03-28T10:14:58.274Z\",\"inputPipeline\":true,\"outputPipeline\":false,\"totalDrivers\":4,\"queuedDrivers\":0,\"queuedPartitionedDrivers\":0,\"queuedPartitionedSplitsWeight\":0,\"runningDrivers\":0,\"runningPartitionedDrivers\":0,\"runningPartitionedSplitsWeight\":0,\"blockedDrivers\":0,\"completedDrivers\":4,\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"queuedTime\":{\"count\":4.0,\"total\":6188053.0,\"p01\":1284813.0,\"p05\":1284813.0,\"p10\":1284813.0,\"p25\":1423865.0,\"p50\":1618249.0,\"p75\":1861126.0,\"p90\":1861126.0,\"p95\":1861126.0,\"p99\":1861126.0,\"min\":1284813.0,\"max\":1861126.0,\"avg\":1547013.25},\"elapsedTime\":{\"count\":4.0,\"total\":1.81457524E8,\"p01\":4.5324548E7,\"p05\":4.5324548E7,\"p10\":4.5324548E7,\"p25\":4.532534E7,\"p50\":4.5374563E7,\"p75\":4.5433073E7,\"p90\":4.5433073E7,\"p95\":4.5433073E7,\"p99\":4.5433073E7,\"min\":4.5324548E7,\"max\":4.5433073E7,\"avg\":4.5364381E7},\"totalScheduledTime\":\"1.33ms\",\"totalCpuTime\":\"1.27ms\",\"totalBlockedTime\":\"172.70ms\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"88B\",\"internalNetworkInputPositions\":2,\"rawInputDataSize\":\"88B\",\"rawInputPositions\":2,\"processedInputDataSize\":\"18B\",\"processedInputPositions\":2,\"inputBlockedTime\":\"172.71ms\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"outputBlockedTime\":\"0.00ns\",\"physicalWrittenDataSize\":\"0B\",\"operatorSummaries\":[{\"stageId\":0,\"pipelineId\":0,\"operatorId\":0,\"planNodeId\":\"189\",\"operatorType\":\"ExchangeOperator\",\"totalDrivers\":4,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"88B\",\"internalNetworkInputPositions\":2,\"rawInputDataSize\":\"88B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":2,\"getOutputWall\":\"207.91us\",\"getOutputCpu\":\"202.83us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAZ73l2ORqID8AAAAAAABZQAAAAAAAABBABAAAAAAAAAAAAAAAAAAAAAAAAAC7E2AW4qoVP2e95djkaiA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":0.0,\"max\":1.25256E-4,\"p25\":0.0,\"p50\":8.265499999999999E-5,\"p75\":1.25256E-4,\"p90\":1.25256E-4,\"p95\":1.25256E-4,\"p99\":1.25256E-4,\"total\":4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"ABuDjWxgC6Y/TEk0U+oqpj8AAAAAAABZQAAAAAAAABBABAAAABuDjWxgC6Y/8l4x+jwMpj+QK9nX2CqmP0xJNFPqKqY/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":0.04305554699999999,\"max\":0.043296167999999996,\"p25\":0.043062119999999995,\"p50\":0.04329564699999999,\"p75\":0.043296167999999996,\"p90\":0.043296167999999996,\"p95\":0.043296167999999996,\"p99\":0.043296167999999996,\"total\":4,\"p01\":0.04305554699999999,\"p05\":0.04305554699999999,\"p10\":0.04305554699999999},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAA8D8AAAAAAABZQAAAAAAAABBABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":0.0,\"max\":1.0,\"p25\":0.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0,\"total\":4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAANA7UrccSID8AAAAAAABZQAAAAAAAABBABAAAAAAAAAAAAAAAAAAAAAAAAABHxzCCIQYVPzQO1K3HEiA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":0.0,\"max\":1.2262999999999998E-4,\"p25\":0.0,\"p50\":8.019999999999998E-5,\"p75\":1.2262999999999998E-4,\"p90\":1.2262999999999998E-4,\"p95\":1.2262999999999998E-4,\"p99\":1.2262999999999998E-4,\"total\":4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"172.71ms\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"56B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"56B\",\"spilledDataSize\":\"0B\",\"info\":{\"@type\":\"directExchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":100,\"averageBytesPerRequest\":27,\"successfulRequestsCount\":12,\"bufferedPages\":0,\"spilledPages\":0,\"spilledBytes\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[],\"requestDuration\":{\"digest\":\"AAAAAAAAAPA/AAAAAAAAQ0AAAAAAAABZQAAAAAAAAAhAAwAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAABDQAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"min\":1.0,\"max\":38.0,\"p25\":1.0,\"p50\":1.0,\"p75\":38.0,\"p90\":38.0,\"p95\":38.0,\"p99\":38.0,\"total\":3,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0}}},{\"stageId\":0,\"pipelineId\":0,\"operatorId\":1,\"planNodeId\":\"183\",\"operatorType\":\"LocalExchangeSinkOperator\",\"totalDrivers\":4,\"addInputCalls\":2,\"addInputWall\":\"71.90us\",\"addInputCpu\":\"72.57us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AG+jT0t24wI/B1XNIPyTET8AAAAAAABZQAAAAAAAABBABAAAAG+jT0t24wI/MyGAsGDWDD9CWvF6If4QPwdVzSD8kxE/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":3.602699999999999E-5,\"max\":6.705499999999999E-5,\"p25\":5.5002999999999995E-5,\"p50\":6.4822E-5,\"p75\":6.705499999999999E-5,\"p90\":6.705499999999999E-5,\"p95\":6.705499999999999E-5,\"p99\":6.705499999999999E-5,\"total\":4,\"p01\":3.602699999999999E-5,\"p05\":3.602699999999999E-5,\"p10\":3.602699999999999E-5},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABZQAAAAAAAABBABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":0.0,\"max\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0,\"total\":4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAA8D8AAAAAAABZQAAAAAAAABBABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":0.0,\"max\":1.0,\"p25\":0.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0,\"total\":4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AORLA5pj1AA/zToJw4XeED8AAAAAAABZQAAAAAAAABBABAAAAORLA5pj1AA/2vVWTaXZCz80P55gdoQQP806CcOF3hA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8=\",\"min\":3.2099999999999994E-5,\"max\":6.4351E-5,\"p25\":5.311999999999999E-5,\"p50\":6.300899999999999E-5,\"p75\":6.4351E-5,\"p90\":6.4351E-5,\"p95\":6.4351E-5,\"p99\":6.4351E-5,\"total\":4,\"p01\":3.2099999999999994E-5,\"p05\":3.2099999999999994E-5,\"p10\":3.2099999999999994E-5}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":4,\"finishWall\":\"151.01us\",\"finishCpu\":\"140.01us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"}],\"drivers\":[]},{\"pipelineId\":1,\"firstStartTime\":\"2025-03-28T10:14:58.231Z\",\"lastStartTime\":\"2025-03-28T10:14:58.231Z\",\"lastEndTime\":\"2025-03-28T10:14:58.275Z\",\"inputPipeline\":false,\"outputPipeline\":true,\"totalDrivers\":1,\"queuedDrivers\":0,\"queuedPartitionedDrivers\":0,\"queuedPartitionedSplitsWeight\":0,\"runningDrivers\":0,\"runningPartitionedDrivers\":0,\"runningPartitionedSplitsWeight\":0,\"blockedDrivers\":0,\"completedDrivers\":1,\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"queuedTime\":{\"count\":1.0,\"total\":1658957.0,\"p01\":1658957.0,\"p05\":1658957.0,\"p10\":1658957.0,\"p25\":1658957.0,\"p50\":1658957.0,\"p75\":1658957.0,\"p90\":1658957.0,\"p95\":1658957.0,\"p99\":1658957.0,\"min\":1658957.0,\"max\":1658957.0,\"avg\":1658957.0},\"elapsedTime\":{\"count\":1.0,\"total\":4.5776818E7,\"p01\":4.5776818E7,\"p05\":4.5776818E7,\"p10\":4.5776818E7,\"p25\":4.5776818E7,\"p50\":4.5776818E7,\"p75\":4.5776818E7,\"p90\":4.5776818E7,\"p95\":4.5776818E7,\"p99\":4.5776818E7,\"min\":4.5776818E7,\"max\":4.5776818E7,\"avg\":4.5776818E7},\"totalScheduledTime\":\"684.34us\",\"totalCpuTime\":\"654.60us\",\"totalBlockedTime\":\"43.07ms\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"rawInputPositions\":2,\"processedInputDataSize\":\"18B\",\"processedInputPositions\":2,\"inputBlockedTime\":\"43.06ms\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"outputBlockedTime\":\"0.00ns\",\"physicalWrittenDataSize\":\"0B\",\"operatorSummaries\":[{\"stageId\":0,\"pipelineId\":1,\"operatorId\":0,\"planNodeId\":\"183\",\"operatorType\":\"LocalExchangeSourceOperator\",\"totalDrivers\":1,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":4.0,\"getOutputCalls\":2,\"getOutputWall\":\"50.27us\",\"getOutputCpu\":\"44.53us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AE82mXypWwo/TzaZfKlbCj8AAAAAAABZQAAAAAAAAPA/AQAAAE82mXypWwo/AAAAAAAA8D8=\",\"min\":5.027399999999999E-5,\"max\":5.027399999999999E-5,\"p25\":5.027399999999999E-5,\"p50\":5.027399999999999E-5,\"p75\":5.027399999999999E-5,\"p90\":5.027399999999999E-5,\"p95\":5.027399999999999E-5,\"p99\":5.027399999999999E-5,\"total\":1,\"p01\":5.027399999999999E-5,\"p05\":5.027399999999999E-5,\"p10\":5.027399999999999E-5},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AJSIdmlRDKY/lIh2aVEMpj8AAAAAAABZQAAAAAAAAPA/AQAAAJSIdmlRDKY/AAAAAAAA8D8=\",\"min\":0.043062728999999994,\"max\":0.043062728999999994,\"p25\":0.043062728999999994,\"p50\":0.043062728999999994,\"p75\":0.043062728999999994,\"p90\":0.043062728999999994,\"p95\":0.043062728999999994,\"p99\":0.043062728999999994,\"total\":1,\"p01\":0.043062728999999994,\"p05\":0.043062728999999994,\"p10\":0.043062728999999994},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAABAAAAAAAAAAEAAAAAAAABZQAAAAAAAAPA/AQAAAAAAAAAAAABAAAAAAAAA8D8=\",\"min\":2.0,\"max\":2.0,\"p25\":2.0,\"p50\":2.0,\"p75\":2.0,\"p90\":2.0,\"p95\":2.0,\"p99\":2.0,\"total\":1,\"p01\":2.0,\"p05\":2.0,\"p10\":2.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AKioppZAWQc/qKimlkBZBz8AAAAAAABZQAAAAAAAAPA/AQAAAKioppZAWQc/AAAAAAAA8D8=\",\"min\":4.453399999999999E-5,\"max\":4.453399999999999E-5,\"p25\":4.453399999999999E-5,\"p50\":4.453399999999999E-5,\"p75\":4.453399999999999E-5,\"p90\":4.453399999999999E-5,\"p95\":4.453399999999999E-5,\"p99\":4.453399999999999E-5,\"total\":1,\"p01\":4.453399999999999E-5,\"p05\":4.453399999999999E-5,\"p10\":4.453399999999999E-5}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"43.06ms\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"},{\"stageId\":0,\"pipelineId\":1,\"operatorId\":1,\"planNodeId\":\"4\",\"operatorType\":\"AggregationOperator\",\"totalDrivers\":1,\"addInputCalls\":2,\"addInputWall\":\"154.96us\",\"addInputCpu\":\"136.16us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":4.0,\"getOutputCalls\":6,\"getOutputWall\":\"84.45us\",\"getOutputCpu\":\"81.76us\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AF4BvG31iDA/XgG8bfWIMD8AAAAAAABZQAAAAAAAAPA/AQAAAF4BvG31iDA/AAAAAAAA8D8=\",\"min\":2.5230399999999996E-4,\"max\":2.5230399999999996E-4,\"p25\":2.5230399999999996E-4,\"p50\":2.5230399999999996E-4,\"p75\":2.5230399999999996E-4,\"p90\":2.5230399999999996E-4,\"p95\":2.5230399999999996E-4,\"p99\":2.5230399999999996E-4,\"total\":1,\"p01\":2.5230399999999996E-4,\"p05\":2.5230399999999996E-4,\"p10\":2.5230399999999996E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABZQAAAAAAAAPA/AQAAAAAAAAAAAAAAAAAAAAAA8D8=\",\"min\":0.0,\"max\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0,\"total\":1,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAABAAAAAAAAAAEAAAAAAAABZQAAAAAAAAPA/AQAAAAAAAAAAAABAAAAAAAAA8D8=\",\"min\":2.0,\"max\":2.0,\"p25\":2.0,\"p50\":2.0,\"p75\":2.0,\"p90\":2.0,\"p95\":2.0,\"p99\":2.0,\"total\":1,\"p01\":2.0,\"p05\":2.0,\"p10\":2.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AOw0qdNHAS4/7DSp00cBLj8AAAAAAABZQAAAAAAAAPA/AQAAAOw0qdNHAS4/AAAAAAAA8D8=\",\"min\":2.2891999999999997E-4,\"max\":2.2891999999999997E-4,\"p25\":2.2891999999999997E-4,\"p50\":2.2891999999999997E-4,\"p75\":2.2891999999999997E-4,\"p90\":2.2891999999999997E-4,\"p95\":2.2891999999999997E-4,\"p99\":2.2891999999999997E-4,\"total\":1,\"p01\":2.2891999999999997E-4,\"p05\":2.2891999999999997E-4,\"p10\":2.2891999999999997E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":1,\"finishWall\":\"12.89us\",\"finishCpu\":\"11.01us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"24B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"24B\",\"spilledDataSize\":\"0B\"},{\"stageId\":0,\"pipelineId\":1,\"operatorId\":2,\"planNodeId\":\"9\",\"operatorType\":\"TaskOutputOperator\",\"totalDrivers\":1,\"addInputCalls\":1,\"addInputWall\":\"187.13us\",\"addInputCpu\":\"186.74us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"9B\",\"inputPositions\":1,\"sumSquaredInputPositions\":1.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"ABLJNPMU1Cg/Esk08xTUKD8AAAAAAABZQAAAAAAAAPA/AQAAABLJNPMU1Cg/AAAAAAAA8D8=\",\"min\":1.8942599999999997E-4,\"max\":1.8942599999999997E-4,\"p25\":1.8942599999999997E-4,\"p50\":1.8942599999999997E-4,\"p75\":1.8942599999999997E-4,\"p90\":1.8942599999999997E-4,\"p95\":1.8942599999999997E-4,\"p99\":1.8942599999999997E-4,\"total\":1,\"p01\":1.8942599999999997E-4,\"p05\":1.8942599999999997E-4,\"p10\":1.8942599999999997E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABZQAAAAAAAAPA/AQAAAAAAAAAAAAAAAAAAAAAA8D8=\",\"min\":0.0,\"max\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0,\"total\":1,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAPA/AAAAAAAA8D8AAAAAAABZQAAAAAAAAPA/AQAAAAAAAAAAAPA/AAAAAAAA8D8=\",\"min\":1.0,\"max\":1.0,\"p25\":1.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0,\"total\":1,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AF/Yq9kU2Sg/X9ir2RTZKD8AAAAAAABZQAAAAAAAAPA/AQAAAF/Yq9kU2Sg/AAAAAAAA8D8=\",\"min\":1.8957499999999996E-4,\"max\":1.8957499999999996E-4,\"p25\":1.8957499999999996E-4,\"p50\":1.8957499999999996E-4,\"p75\":1.8957499999999996E-4,\"p90\":1.8957499999999996E-4,\"p95\":1.8957499999999996E-4,\"p99\":1.8957499999999996E-4,\"total\":1,\"p01\":1.8957499999999996E-4,\"p05\":1.8957499999999996E-4,\"p10\":1.8957499999999996E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":1,\"finishWall\":\"2.29us\",\"finishCpu\":\"2.84us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"64B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"64B\",\"spilledDataSize\":\"0B\"}],\"drivers\":[]}]},\"needsPlan\":false}],\"subStages\":[{\"stageId\":\"20250328_101456_00000_gt85c.1\",\"state\":\"FINISHED\",\"plan\":{\"id\":\"1\",\"root\":{\"@type\":\"aggregation\",\"id\":\"187\",\"source\":{\"@type\":\"tableScan\",\"id\":\"0\",\"table\":{\"catalogHandle\":\"tpch:normal:default\",\"connectorHandle\":{\"@type\":\"tpch:io.trino.plugin.tpch.TpchTableHandle\",\"schemaName\":\"sf1\",\"tableName\":\"customer\",\"scaleFactor\":1.0,\"constraint\":{\"columnDomains\":[]}},\"transaction\":[\"tpch:io.trino.plugin.tpch.TpchTransactionHandle\",\"INSTANCE\"]},\"outputSymbols\":[],\"assignments\":{},\"updateTarget\":false,\"useConnectorNodePartitioning\":false},\"aggregations\":{\"Y291bnRfMA==:YmlnaW50\":{\"resolvedFunction\":{\"signature\":{\"name\":{\"catalogName\":\"system\",\"schemaName\":\"builtin\",\"functionName\":\"count\"},\"returnType\":\"bigint\",\"argumentTypes\":[]},\"catalogHandle\":\"system:normal:system\",\"functionId\":\"count():bigint\",\"functionKind\":\"AGGREGATE\",\"deterministic\":true,\"functionNullability\":{\"returnNullable\":true,\"argumentNullable\":[]},\"typeDependencies\":{},\"functionDependencies\":[]},\"arguments\":[],\"distinct\":false}},\"groupingSets\":{\"groupingKeys\":[],\"groupingSetCount\":1,\"globalGroupingSets\":[0]},\"preGroupedSymbols\":[],\"step\":\"PARTIAL\"},\"symbols\":[{\"type\":\"bigint\",\"name\":\"count_0\"}],\"partitioning\":{\"connectorHandle\":{\"@type\":\"system:io.trino.sql.planner.SystemPartitioningHandle\",\"partitioning\":\"SOURCE\",\"function\":\"UNKNOWN\"},\"scaleWriters\":false},\"partitionedSources\":[\"0\"],\"outputPartitioningScheme\":{\"partitioning\":{\"handle\":{\"connectorHandle\":{\"@type\":\"system:io.trino.sql.planner.SystemPartitioningHandle\",\"partitioning\":\"SINGLE\",\"function\":\"SINGLE\"},\"scaleWriters\":false},\"arguments\":[]},\"outputLayout\":[{\"type\":\"bigint\",\"name\":\"count_0\"}],\"replicateNullsAndAny\":false},\"statsAndCosts\":{\"stats\":{\"187\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}},\"0\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}}},\"costs\":{\"187\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":\"NaN\",\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"networkCost\":\"NaN\"}},\"0\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"maxMemoryWhenOutputting\":0.0,\"networkCost\":0.0,\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":0.0}}}},\"activeCatalogs\":[],\"languageFunctions\":{},\"jsonRepresentation\":\"{\\n \\\"id\\\" : \\\"187\\\",\\n \\\"name\\\" : \\\"Aggregate\\\",\\n \\\"descriptor\\\" : {\\n \\\"type\\\" : \\\"PARTIAL\\\",\\n \\\"keys\\\" : \\\"\\\",\\n \\\"hash\\\" : \\\"[]\\\"\\n },\\n \\\"outputs\\\" : [ {\\n \\\"type\\\" : \\\"bigint\\\",\\n \\\"name\\\" : \\\"count_0\\\"\\n } ],\\n \\\"details\\\" : [ \\\"count_0 := count(*)\\\" ],\\n \\\"estimates\\\" : [ ],\\n \\\"children\\\" : [ {\\n \\\"id\\\" : \\\"0\\\",\\n \\\"name\\\" : \\\"TableScan\\\",\\n \\\"descriptor\\\" : {\\n \\\"table\\\" : \\\"tpch:sf1:customer\\\"\\n },\\n \\\"outputs\\\" : [ ],\\n \\\"details\\\" : [ ],\\n \\\"estimates\\\" : [ ],\\n \\\"children\\\" : [ ]\\n } ]\\n}\"},\"coordinatorOnly\":false,\"types\":[\"bigint\"],\"stageStats\":{\"schedulingComplete\":\"2025-03-28T10:14:58.023Z\",\"getSplitDistribution\":{\"count\":1.0,\"total\":238218.0,\"p01\":238218.0,\"p05\":238218.0,\"p10\":238218.0,\"p25\":238218.0,\"p50\":238218.0,\"p75\":238218.0,\"p90\":238218.0,\"p95\":238218.0,\"p99\":238218.0,\"min\":238218.0,\"max\":238218.0,\"avg\":238218.0},\"totalTasks\":1,\"runningTasks\":0,\"completedTasks\":1,\"failedTasks\":0,\"totalDrivers\":2,\"queuedDrivers\":0,\"runningDrivers\":0,\"blockedDrivers\":0,\"completedDrivers\":2,\"cumulativeUserMemory\":2810.842816,\"failedCumulativeUserMemory\":0.0,\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"totalMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"208B\",\"peakRevocableMemoryReservation\":\"0B\",\"totalScheduledTime\":\"76.79ms\",\"failedScheduledTime\":\"0.00s\",\"totalCpuTime\":\"76.73ms\",\"failedCpuTime\":\"0.00s\",\"totalBlockedTime\":\"0.00s\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"failedPhysicalInputDataSize\":\"0B\",\"physicalInputPositions\":150000,\"failedPhysicalInputPositions\":0,\"physicalInputReadTime\":\"0.00s\",\"failedPhysicalInputReadTime\":\"0.00s\",\"internalNetworkInputDataSize\":\"0B\",\"failedInternalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"failedInternalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"failedRawInputDataSize\":\"0B\",\"rawInputPositions\":150000,\"failedRawInputPositions\":0,\"processedInputDataSize\":\"0B\",\"failedProcessedInputDataSize\":\"0B\",\"processedInputPositions\":150000,\"failedProcessedInputPositions\":0,\"inputBlockedTime\":\"0.00s\",\"failedInputBlockedTime\":\"0.00s\",\"bufferedDataSize\":\"0B\",\"outputBufferUtilization\":{\"digest\":\"AAAAAAAAAAAAAAAAAAAA2j4AAAAAAABZQAAAAFCvXXhBCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADKPgAAAAAAAMo+AAAAAAAA2j4AAAAA2e41QQAAAACYTg5BAAAAgKF8SEEAAAAA4Lv4QAAAAIBdmG9BAAAAAPatJUEAAAAAh309QQAAAADahTRB\",\"min\":0.0,\"max\":6.198883056640625E-6,\"p25\":0.0,\"p50\":0.0,\"p75\":2.1110405757015712E-6,\"p90\":3.0994415283203125E-6,\"p95\":5.054705620511771E-6,\"p99\":6.198883056640625E-6,\"total\":25549557,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"outputDataSize\":\"18B\",\"failedOutputDataSize\":\"0B\",\"outputPositions\":2,\"failedOutputPositions\":0,\"outputBlockedTime\":\"0.00s\",\"failedOutputBlockedTime\":\"0.00s\",\"physicalWrittenDataSize\":\"0B\",\"failedPhysicalWrittenDataSize\":\"0B\",\"gcInfo\":{\"stageId\":1,\"tasks\":1,\"fullGcTasks\":0,\"minFullGcSec\":0,\"maxFullGcSec\":0,\"totalFullGcSec\":0,\"averageFullGcSec\":0},\"operatorSummaries\":[{\"stageId\":1,\"pipelineId\":0,\"operatorId\":0,\"planNodeId\":\"0\",\"operatorType\":\"TableScanOperator\",\"totalDrivers\":2,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":150000,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"0B\",\"inputPositions\":150000,\"sumSquaredInputPositions\":1.125E10,\"getOutputCalls\":38,\"getOutputWall\":\"75.57ms\",\"getOutputCpu\":\"75.54ms\",\"outputDataSize\":\"0B\",\"outputPositions\":150000,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.03748956999999999,\"max\":0.038077997999999995,\"p01\":0.03748956999999999,\"p05\":0.03748956999999999,\"p10\":0.03748956999999999,\"p25\":0.03748956999999999,\"p50\":0.03748956999999999,\"p75\":0.03748956999999999,\"p90\":0.03748956999999999,\"p95\":0.03748956999999999,\"p99\":0.03748956999999999},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.037475165,\"max\":0.038066303999999995,\"p01\":0.037475165,\"p05\":0.037475165,\"p10\":0.037475165,\"p25\":0.037475165,\"p50\":0.037475165,\"p75\":0.037475165,\"p90\":0.037475165,\"p95\":0.037475165,\"p99\":0.037475165},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":75000.0,\"max\":75000.0,\"p01\":75000.0,\"p05\":75000.0,\"p10\":75000.0,\"p25\":75000.0,\"p50\":75000.0,\"p75\":75000.0,\"p90\":75000.0,\"p95\":75000.0,\"p99\":75000.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"},{\"stageId\":1,\"pipelineId\":0,\"operatorId\":2,\"planNodeId\":\"187\",\"operatorType\":\"TaskOutputOperator\",\"totalDrivers\":2,\"addInputCalls\":2,\"addInputWall\":\"648.52us\",\"addInputCpu\":\"594.65us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.33581E-4,\"max\":5.209589999999999E-4,\"p01\":1.33581E-4,\"p05\":1.33581E-4,\"p10\":1.33581E-4,\"p25\":1.33581E-4,\"p50\":1.33581E-4,\"p75\":1.33581E-4,\"p90\":1.33581E-4,\"p95\":1.33581E-4,\"p99\":1.33581E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.33901E-4,\"max\":4.6756599999999993E-4,\"p01\":1.33901E-4,\"p05\":1.33901E-4,\"p10\":1.33901E-4,\"p25\":1.33901E-4,\"p50\":1.33901E-4,\"p75\":1.33901E-4,\"p90\":1.33901E-4,\"p95\":1.33901E-4,\"p99\":1.33901E-4},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.0,\"max\":1.0,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0,\"p25\":1.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":2,\"finishWall\":\"6.02us\",\"finishCpu\":\"6.81us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"64B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"64B\",\"spilledDataSize\":\"0B\"},{\"stageId\":1,\"pipelineId\":0,\"operatorId\":1,\"planNodeId\":\"187\",\"operatorType\":\"AggregationOperator\",\"totalDrivers\":2,\"addInputCalls\":2,\"addInputWall\":\"169.01us\",\"addInputCpu\":\"169.52us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"0B\",\"inputPositions\":150000,\"sumSquaredInputPositions\":1.125E10,\"getOutputCalls\":38,\"getOutputWall\":\"214.28us\",\"getOutputCpu\":\"233.72us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.8643999999999998E-4,\"max\":2.0529899999999996E-4,\"p01\":1.8643999999999998E-4,\"p05\":1.8643999999999998E-4,\"p10\":1.8643999999999998E-4,\"p25\":1.8643999999999998E-4,\"p50\":1.8643999999999998E-4,\"p75\":1.8643999999999998E-4,\"p90\":1.8643999999999998E-4,\"p95\":1.8643999999999998E-4,\"p99\":1.8643999999999998E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.9570699999999996E-4,\"max\":2.1664699999999996E-4,\"p01\":1.9570699999999996E-4,\"p05\":1.9570699999999996E-4,\"p10\":1.9570699999999996E-4,\"p25\":1.9570699999999996E-4,\"p50\":1.9570699999999996E-4,\"p75\":1.9570699999999996E-4,\"p90\":1.9570699999999996E-4,\"p95\":1.9570699999999996E-4,\"p99\":1.9570699999999996E-4},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":75000.0,\"max\":75000.0,\"p01\":75000.0,\"p05\":75000.0,\"p10\":75000.0,\"p25\":75000.0,\"p50\":75000.0,\"p75\":75000.0,\"p90\":75000.0,\"p95\":75000.0,\"p99\":75000.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":2,\"finishWall\":\"8.46us\",\"finishCpu\":\"9.12us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"24B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"24B\",\"spilledDataSize\":\"0B\"}]},\"tasks\":[{\"taskStatus\":{\"taskId\":\"20250328_101456_00000_gt85c.1.0.0\",\"taskInstanceId\":\"8653a123-0f64-4a1a-b752-37be2e41db29\",\"version\":2,\"state\":\"FINISHED\",\"self\":\"http://trino-m-1-worker-default-0.trino-m-1-worker-default.default.svc.cluster.local:8080/v1/task/20250328_101456_00000_gt85c.1.0.0\",\"nodeId\":\"b81fc056-0921-4e5a-9773-2aea9121d298\",\"speculative\":false,\"failures\":[],\"queuedPartitionedDrivers\":0,\"runningPartitionedDrivers\":0,\"outputBufferStatus\":{\"outputBuffersVersion\":2,\"overutilized\":false,\"exchangeSinkInstanceHandleUpdateRequired\":false},\"outputDataSize\":\"18B\",\"writerInputDataSize\":\"0B\",\"physicalWrittenDataSize\":\"0B\",\"memoryReservation\":\"0B\",\"peakMemoryReservation\":\"208B\",\"revocableMemoryReservation\":\"0B\",\"fullGcCount\":0,\"fullGcTime\":\"0.00ns\",\"dynamicFiltersVersion\":0,\"queuedPartitionedSplitsWeight\":0,\"runningPartitionedSplitsWeight\":0},\"lastHeartbeat\":\"2025-03-28T10:14:58.292Z\",\"outputBuffers\":{\"type\":\"PARTITIONED\",\"state\":\"FINISHED\",\"canAddBuffers\":false,\"canAddPages\":false,\"totalBufferedBytes\":0,\"totalBufferedPages\":0,\"totalRowsSent\":2,\"totalPagesSent\":2,\"utilization\":{\"digest\":\"AAAAAAAAAAAAAAAAAAAA2j4AAAAAAABZQAAAAFCvXXhBCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADKPgAAAAAAAMo+AAAAAAAA2j4AAAAA2e41QQAAAACYTg5BAAAAgKF8SEEAAAAA4Lv4QAAAAIBdmG9BAAAAAPatJUEAAAAAh309QQAAAADahTRB\",\"min\":0.0,\"max\":6.198883056640625E-6,\"p25\":0.0,\"p50\":0.0,\"p75\":2.1110405757015712E-6,\"p90\":3.0994415283203125E-6,\"p95\":5.054705620511771E-6,\"p99\":6.198883056640625E-6,\"total\":25549557,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0}},\"noMoreSplits\":[\"0\"],\"stats\":{\"createTime\":\"2025-03-28T10:14:58.028Z\",\"firstStartTime\":\"2025-03-28T10:14:58.230Z\",\"lastStartTime\":\"2025-03-28T10:14:58.230Z\",\"lastEndTime\":\"2025-03-28T10:14:58.269Z\",\"endTime\":\"2025-03-28T10:14:58.274Z\",\"elapsedTime\":\"45.10ms\",\"queuedTime\":\"1.73ms\",\"totalDrivers\":2,\"queuedDrivers\":0,\"queuedPartitionedDrivers\":0,\"queuedPartitionedSplitsWeight\":0,\"runningDrivers\":0,\"runningPartitionedDrivers\":0,\"runningPartitionedSplitsWeight\":0,\"blockedDrivers\":0,\"completedDrivers\":2,\"cumulativeUserMemory\":2810.842816,\"userMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"208B\",\"revocableMemoryReservation\":\"0B\",\"totalScheduledTime\":\"76.79ms\",\"totalCpuTime\":\"76.73ms\",\"totalBlockedTime\":\"0.00ns\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":150000,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"rawInputPositions\":150000,\"processedInputDataSize\":\"0B\",\"processedInputPositions\":150000,\"inputBlockedTime\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"outputBlockedTime\":\"0.00ns\",\"writerInputDataSize\":\"0B\",\"physicalWrittenDataSize\":\"0B\",\"fullGcCount\":0,\"fullGcTime\":\"0.00ns\",\"pipelines\":[{\"pipelineId\":0,\"firstStartTime\":\"2025-03-28T10:14:58.230Z\",\"lastStartTime\":\"2025-03-28T10:14:58.230Z\",\"lastEndTime\":\"2025-03-28T10:14:58.269Z\",\"inputPipeline\":true,\"outputPipeline\":true,\"totalDrivers\":2,\"queuedDrivers\":0,\"queuedPartitionedDrivers\":0,\"queuedPartitionedSplitsWeight\":0,\"runningDrivers\":0,\"runningPartitionedDrivers\":0,\"runningPartitionedSplitsWeight\":0,\"blockedDrivers\":0,\"completedDrivers\":2,\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"queuedTime\":{\"count\":2.0,\"total\":2613880.0,\"p01\":1239629.0,\"p05\":1239629.0,\"p10\":1239629.0,\"p25\":1239629.0,\"p50\":1374251.0,\"p75\":1374251.0,\"p90\":1374251.0,\"p95\":1374251.0,\"p99\":1374251.0,\"min\":1239629.0,\"max\":1374251.0,\"avg\":1306940.0},\"elapsedTime\":{\"count\":2.0,\"total\":7.940031E7,\"p01\":3.954803E7,\"p05\":3.954803E7,\"p10\":3.954803E7,\"p25\":3.954803E7,\"p50\":3.985228E7,\"p75\":3.985228E7,\"p90\":3.985228E7,\"p95\":3.985228E7,\"p99\":3.985228E7,\"min\":3.954803E7,\"max\":3.985228E7,\"avg\":3.9700155E7},\"totalScheduledTime\":\"76.79ms\",\"totalCpuTime\":\"76.73ms\",\"totalBlockedTime\":\"0.00ns\",\"fullyBlocked\":false,\"blockedReasons\":[],\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":150000,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"rawInputPositions\":150000,\"processedInputDataSize\":\"0B\",\"processedInputPositions\":150000,\"inputBlockedTime\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"outputBlockedTime\":\"0.00ns\",\"physicalWrittenDataSize\":\"0B\",\"operatorSummaries\":[{\"stageId\":1,\"pipelineId\":0,\"operatorId\":0,\"planNodeId\":\"0\",\"operatorType\":\"TableScanOperator\",\"totalDrivers\":2,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":150000,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"0B\",\"inputPositions\":150000,\"sumSquaredInputPositions\":1.125E10,\"getOutputCalls\":38,\"getOutputWall\":\"75.57ms\",\"getOutputCpu\":\"75.54ms\",\"outputDataSize\":\"0B\",\"outputPositions\":150000,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAOlLjrVMaM/z9w2mPV+oz8AAAAAAABZQAAAAAAAAABAAgAAAAOlLjrVMaM/z9w2mPV+oz8AAAAAAADwPwAAAAAAAPA/\",\"min\":0.03748956999999999,\"max\":0.038077997999999995,\"p25\":0.03748956999999999,\"p50\":0.038077997999999995,\"p75\":0.038077997999999995,\"p90\":0.038077997999999995,\"p95\":0.038077997999999995,\"p99\":0.038077997999999995,\"total\":2,\"p01\":0.03748956999999999,\"p05\":0.03748956999999999,\"p10\":0.03748956999999999},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABZQAAAAAAAAABAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/\",\"min\":0.0,\"max\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0,\"total\":2,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAACAT/JAAAAAAIBP8kAAAAAAAABZQAAAAAAAAABAAgAAAAAAAACAT/JAAAAAAIBP8kAAAAAAAADwPwAAAAAAAPA/\",\"min\":75000.0,\"max\":75000.0,\"p25\":75000.0,\"p50\":75000.0,\"p75\":75000.0,\"p90\":75000.0,\"p95\":75000.0,\"p99\":75000.0,\"total\":2,\"p01\":75000.0,\"p05\":75000.0,\"p10\":75000.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"APulLODxL6M/gOmENW19oz8AAAAAAABZQAAAAAAAAABAAgAAAPulLODxL6M/gOmENW19oz8AAAAAAADwPwAAAAAAAPA/\",\"min\":0.037475165,\"max\":0.038066303999999995,\"p25\":0.037475165,\"p50\":0.038066303999999995,\"p75\":0.038066303999999995,\"p90\":0.038066303999999995,\"p95\":0.038066303999999995,\"p99\":0.038066303999999995,\"total\":2,\"p01\":0.037475165,\"p05\":0.037475165,\"p10\":0.037475165}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"},{\"stageId\":1,\"pipelineId\":0,\"operatorId\":1,\"planNodeId\":\"187\",\"operatorType\":\"AggregationOperator\",\"totalDrivers\":2,\"addInputCalls\":2,\"addInputWall\":\"169.01us\",\"addInputCpu\":\"169.52us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"0B\",\"inputPositions\":150000,\"sumSquaredInputPositions\":1.125E10,\"getOutputCalls\":38,\"getOutputWall\":\"214.28us\",\"getOutputCpu\":\"233.72us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AJHew2fjbyg/IXFX+7DoKj8AAAAAAABZQAAAAAAAAABAAgAAAJHew2fjbyg/IXFX+7DoKj8AAAAAAADwPwAAAAAAAPA/\",\"min\":1.8643999999999998E-4,\"max\":2.0529899999999996E-4,\"p25\":1.8643999999999998E-4,\"p50\":2.0529899999999996E-4,\"p75\":2.0529899999999996E-4,\"p90\":2.0529899999999996E-4,\"p95\":2.0529899999999996E-4,\"p99\":2.0529899999999996E-4,\"total\":2,\"p01\":1.8643999999999998E-4,\"p05\":1.8643999999999998E-4,\"p10\":1.8643999999999998E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABZQAAAAAAAAABAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/\",\"min\":0.0,\"max\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0,\"total\":2,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAACAT/JAAAAAAIBP8kAAAAAAAABZQAAAAAAAAABAAgAAAAAAAACAT/JAAAAAAIBP8kAAAAAAAADwPwAAAAAAAPA/\",\"min\":75000.0,\"max\":75000.0,\"p25\":75000.0,\"p50\":75000.0,\"p75\":75000.0,\"p90\":75000.0,\"p95\":75000.0,\"p99\":75000.0,\"total\":2,\"p01\":75000.0,\"p05\":75000.0,\"p10\":75000.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AIU5RlTWpik/8d0+j3dlLD8AAAAAAABZQAAAAAAAAABAAgAAAIU5RlTWpik/8d0+j3dlLD8AAAAAAADwPwAAAAAAAPA/\",\"min\":1.9570699999999996E-4,\"max\":2.1664699999999996E-4,\"p25\":1.9570699999999996E-4,\"p50\":2.1664699999999996E-4,\"p75\":2.1664699999999996E-4,\"p90\":2.1664699999999996E-4,\"p95\":2.1664699999999996E-4,\"p99\":2.1664699999999996E-4,\"total\":2,\"p01\":1.9570699999999996E-4,\"p05\":1.9570699999999996E-4,\"p10\":1.9570699999999996E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":2,\"finishWall\":\"8.46us\",\"finishCpu\":\"9.12us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"24B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"24B\",\"spilledDataSize\":\"0B\"},{\"stageId\":1,\"pipelineId\":0,\"operatorId\":2,\"planNodeId\":\"187\",\"operatorType\":\"TaskOutputOperator\",\"totalDrivers\":2,\"addInputCalls\":2,\"addInputWall\":\"648.52us\",\"addInputCpu\":\"594.65us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AOf4fw08giE/choM7x4SQT8AAAAAAABZQAAAAAAAAABAAgAAAOf4fw08giE/choM7x4SQT8AAAAAAADwPwAAAAAAAPA/\",\"min\":1.33581E-4,\"max\":5.209589999999999E-4,\"p25\":1.33581E-4,\"p50\":5.209589999999999E-4,\"p75\":5.209589999999999E-4,\"p90\":5.209589999999999E-4,\"p95\":5.209589999999999E-4,\"p99\":5.209589999999999E-4,\"total\":2,\"p01\":1.33581E-4,\"p05\":1.33581E-4,\"p10\":1.33581E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABZQAAAAAAAAABAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/\",\"min\":0.0,\"max\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0,\"total\":2,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AAAAAAAAAPA/AAAAAAAA8D8AAAAAAABZQAAAAAAAAABAAgAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPwAAAAAAAPA/\",\"min\":1.0,\"max\":1.0,\"p25\":1.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0,\"total\":2,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.plugin.base.metrics.TDigestHistogram\",\"digest\":\"AC0R8dT4jCE/z7HArXSkPj8AAAAAAABZQAAAAAAAAABAAgAAAC0R8dT4jCE/z7HArXSkPj8AAAAAAADwPwAAAAAAAPA/\",\"min\":1.33901E-4,\"max\":4.6756599999999993E-4,\"p25\":1.33901E-4,\"p50\":4.6756599999999993E-4,\"p75\":4.6756599999999993E-4,\"p90\":4.6756599999999993E-4,\"p95\":4.6756599999999993E-4,\"p99\":4.6756599999999993E-4,\"total\":2,\"p01\":1.33901E-4,\"p05\":1.33901E-4,\"p10\":1.33901E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":2,\"finishWall\":\"6.02us\",\"finishCpu\":\"6.81us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"64B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"64B\",\"spilledDataSize\":\"0B\"}],\"drivers\":[]}]},\"needsPlan\":false}],\"subStages\":[],\"tables\":{\"0\":{\"connectorName\":\"tpch\",\"tableName\":\"tpch.sf1.customer\",\"predicate\":{\"columnDomains\":[]}}}}],\"tables\":{}}" + }, + "statistics": { + "cpuTime": 0.079000000, + "failedCpuTime": 0.0, + "wallTime": 2.025000000, + "queuedTime": 0.068000000, + "scheduledTime": 0.079000000, + "failedScheduledTime": 0.0, + "resourceWaitingTime": 1.245000000, + "analysisTime": 0.295000000, + "planningTime": 0.262000000, + "planningCpuTime": 0.214000000, + "startingTime": 0.071000000, + "executionTime": 0.711000000, + "inputBlockedTime": 0.173000000, + "failedInputBlockedTime": 0.0, + "outputBlockedTime": 0.0, + "failedOutputBlockedTime": 0.0, + "physicalInputReadTime": 0.0, + "peakUserMemoryBytes": 320, + "peakTaskUserMemory": 208, + "peakTaskTotalMemory": 208, + "physicalInputBytes": 0, + "physicalInputRows": 150000, + "processedInputBytes": 0, + "processedInputRows": 150000, + "internalNetworkBytes": 88, + "internalNetworkRows": 2, + "totalBytes": 0, + "totalRows": 150000, + "outputBytes": 9, + "outputRows": 1, + "writtenBytes": 0, + "writtenRows": 0, + "spilledBytes": 0, + "cumulativeMemory": 12883.187256, + "failedCumulativeMemory": 0.0, + "stageGcStatistics": [ + { + "stageId": 0, + "tasks": 1, + "fullGcTasks": 0, + "minFullGcSec": 0, + "maxFullGcSec": 0, + "totalFullGcSec": 0, + "averageFullGcSec": 0 + }, + { + "stageId": 1, + "tasks": 1, + "fullGcTasks": 0, + "minFullGcSec": 0, + "maxFullGcSec": 0, + "totalFullGcSec": 0, + "averageFullGcSec": 0 + } + ], + "completedSplits": 7, + "complete": true, + "cpuTimeDistribution": [ + { + "stageId": 0, + "tasks": 1, + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + { + "stageId": 1, + "tasks": 1, + "p25": 77, + "p50": 77, + "p75": 77, + "p90": 77, + "p95": 77, + "p99": 77, + "min": 77, + "max": 77, + "total": 77, + "average": 77.0 + } + ], + "outputBufferUtilization": [ + { + "stageId": 0, + "tasks": 1, + "p01": 0.0, + "p05": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 0.00030994415283203125, + "p75": 0.00030994415283203125, + "p90": 0.00030994415283203125, + "p95": 0.00030994415283203125, + "p99": 0.00030994415283203125, + "min": 0.0, + "max": 0.00030994415283203125, + "duration": 0.028210115 + }, + { + "stageId": 1, + "tasks": 1, + "p01": 0.0, + "p05": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 0.0, + "p75": 0.00021110405757015713, + "p90": 0.00030994415283203125, + "p95": 0.0005054705620511771, + "p99": 0.0006198883056640625, + "min": 0.0, + "max": 0.0006198883056640625, + "duration": 0.025549557 + } + ], + "taskStatistics": [ + { + "stageId": 0, + "tasks": 1, + "cpuTimeDistribution": { + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + "scheduledTimeDistribution": { + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + "peakMemoryReservationDistribution": { + "p25": 288, + "p50": 288, + "p75": 288, + "p90": 288, + "p95": 288, + "p99": 288, + "min": 288, + "max": 288, + "total": 288, + "average": 288.0 + }, + "estimatedMemoryReservationDistribution": { + "p25": 0, + "p50": 0, + "p75": 0, + "p90": 0, + "p95": 0, + "p99": 0, + "min": 0, + "max": 0, + "total": 0, + "average": 0.0 + }, + "rawInputDataSizeDistribution": { + "p25": 88, + "p50": 88, + "p75": 88, + "p90": 88, + "p95": 88, + "p99": 88, + "min": 88, + "max": 88, + "total": 88, + "average": 88.0 + }, + "rawInputPositionsDistribution": { + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + "processedInputDataSizeDistribution": { + "p25": 18, + "p50": 18, + "p75": 18, + "p90": 18, + "p95": 18, + "p99": 18, + "min": 18, + "max": 18, + "total": 18, + "average": 18.0 + }, + "processedInputPositionsDistribution": { + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + "outputDataSizeDistribution": { + "p25": 9, + "p50": 9, + "p75": 9, + "p90": 9, + "p95": 9, + "p99": 9, + "min": 9, + "max": 9, + "total": 9, + "average": 9.0 + }, + "outputPositionsDistribution": { + "p25": 1, + "p50": 1, + "p75": 1, + "p90": 1, + "p95": 1, + "p99": 1, + "min": 1, + "max": 1, + "total": 1, + "average": 1.0 + }, + "totalDriversDistribution": { + "p25": 5, + "p50": 5, + "p75": 5, + "p90": 5, + "p95": 5, + "p99": 5, + "min": 5, + "max": 5, + "total": 5, + "average": 5.0 + }, + "createTimeMillisDistribution": { + "p01": 1652, + "p05": 1652, + "p10": 1652, + "p25": 1652, + "p50": 1652, + "p75": 1652, + "p90": 1652, + "p95": 1652, + "p99": 1652, + "min": 1652, + "max": 1652, + "total": 1652, + "average": 1652.0 + }, + "firstStartTimeMillisDistribution": { + "p01": 1848, + "p05": 1848, + "p10": 1848, + "p25": 1848, + "p50": 1848, + "p75": 1848, + "p90": 1848, + "p95": 1848, + "p99": 1848, + "min": 1848, + "max": 1848, + "total": 1848, + "average": 1848.0 + }, + "lastStartTimeMillisDistribution": { + "p01": 1849, + "p05": 1849, + "p10": 1849, + "p25": 1849, + "p50": 1849, + "p75": 1849, + "p90": 1849, + "p95": 1849, + "p99": 1849, + "min": 1849, + "max": 1849, + "total": 1849, + "average": 1849.0 + }, + "terminatingStartTimeMillisDistribution": { + "p01": 0, + "p05": 0, + "p10": 0, + "p25": 0, + "p50": 0, + "p75": 0, + "p90": 0, + "p95": 0, + "p99": 0, + "min": 0, + "max": 0, + "total": 0, + "average": 0.0 + }, + "lastEndTimeMillisDistribution": { + "p01": 1893, + "p05": 1893, + "p10": 1893, + "p25": 1893, + "p50": 1893, + "p75": 1893, + "p90": 1893, + "p95": 1893, + "p99": 1893, + "min": 1893, + "max": 1893, + "total": 1893, + "average": 1893.0 + }, + "endTimeMillisDistribution": { + "p01": 1917, + "p05": 1917, + "p10": 1917, + "p25": 1917, + "p50": 1917, + "p75": 1917, + "p90": 1917, + "p95": 1917, + "p99": 1917, + "min": 1917, + "max": 1917, + "total": 1917, + "average": 1917.0 + }, + "createTimeScaledDistribution": { + "p01": 0.8162055335968379, + "p05": 0.8162055335968379, + "p10": 0.8162055335968379, + "p25": 0.8162055335968379, + "p50": 0.8162055335968379, + "p75": 0.8162055335968379, + "p90": 0.8162055335968379, + "p95": 0.8162055335968379, + "p99": 0.8162055335968379, + "min": 0.8162055335968379, + "max": 0.8162055335968379, + "total": 1652.0, + "average": 0.8162055335968379 + }, + "firstStartTimeScaledDistribution": { + "p01": 0.9130434782608695, + "p05": 0.9130434782608695, + "p10": 0.9130434782608695, + "p25": 0.9130434782608695, + "p50": 0.9130434782608695, + "p75": 0.9130434782608695, + "p90": 0.9130434782608695, + "p95": 0.9130434782608695, + "p99": 0.9130434782608695, + "min": 0.9130434782608695, + "max": 0.9130434782608695, + "total": 1848.0, + "average": 0.9130434782608695 + }, + "lastStartTimeScaledDistribution": { + "p01": 0.9135375494071146, + "p05": 0.9135375494071146, + "p10": 0.9135375494071146, + "p25": 0.9135375494071146, + "p50": 0.9135375494071146, + "p75": 0.9135375494071146, + "p90": 0.9135375494071146, + "p95": 0.9135375494071146, + "p99": 0.9135375494071146, + "min": 0.9135375494071146, + "max": 0.9135375494071146, + "total": 1849.0, + "average": 0.9135375494071146 + }, + "terminatingStartTimeScaledDistribution": { + "p01": 0.0, + "p05": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0, + "min": 0.0, + "max": 0.0, + "total": 0.0, + "average": 0.0 + }, + "lastEndTimeScaledDistribution": { + "p01": 0.9352766798418972, + "p05": 0.9352766798418972, + "p10": 0.9352766798418972, + "p25": 0.9352766798418972, + "p50": 0.9352766798418972, + "p75": 0.9352766798418972, + "p90": 0.9352766798418972, + "p95": 0.9352766798418972, + "p99": 0.9352766798418972, + "min": 0.9352766798418972, + "max": 0.9352766798418972, + "total": 1893.0, + "average": 0.9352766798418972 + }, + "endTimeScaledDistribution": { + "p01": 0.9471343873517787, + "p05": 0.9471343873517787, + "p10": 0.9471343873517787, + "p25": 0.9471343873517787, + "p50": 0.9471343873517787, + "p75": 0.9471343873517787, + "p90": 0.9471343873517787, + "p95": 0.9471343873517787, + "p99": 0.9471343873517787, + "min": 0.9471343873517787, + "max": 0.9471343873517787, + "total": 1917.0, + "average": 0.9471343873517787 + }, + "getSplitDistribution": { + "p01": "NaN", + "p05": "NaN", + "p10": "NaN", + "p25": "NaN", + "p50": "NaN", + "p75": "NaN", + "p90": "NaN", + "p95": "NaN", + "p99": "NaN", + "min": "NaN", + "max": "NaN", + "total": 0.0, + "average": 0.0 + } + }, + { + "stageId": 1, + "tasks": 1, + "cpuTimeDistribution": { + "p25": 77, + "p50": 77, + "p75": 77, + "p90": 77, + "p95": 77, + "p99": 77, + "min": 77, + "max": 77, + "total": 77, + "average": 77.0 + }, + "scheduledTimeDistribution": { + "p25": 77, + "p50": 77, + "p75": 77, + "p90": 77, + "p95": 77, + "p99": 77, + "min": 77, + "max": 77, + "total": 77, + "average": 77.0 + }, + "peakMemoryReservationDistribution": { + "p25": 208, + "p50": 208, + "p75": 208, + "p90": 208, + "p95": 208, + "p99": 208, + "min": 208, + "max": 208, + "total": 208, + "average": 208.0 + }, + "estimatedMemoryReservationDistribution": { + "p25": 0, + "p50": 0, + "p75": 0, + "p90": 0, + "p95": 0, + "p99": 0, + "min": 0, + "max": 0, + "total": 0, + "average": 0.0 + }, + "rawInputDataSizeDistribution": { + "p25": 0, + "p50": 0, + "p75": 0, + "p90": 0, + "p95": 0, + "p99": 0, + "min": 0, + "max": 0, + "total": 0, + "average": 0.0 + }, + "rawInputPositionsDistribution": { + "p25": 150000, + "p50": 150000, + "p75": 150000, + "p90": 150000, + "p95": 150000, + "p99": 150000, + "min": 150000, + "max": 150000, + "total": 150000, + "average": 150000.0 + }, + "processedInputDataSizeDistribution": { + "p25": 0, + "p50": 0, + "p75": 0, + "p90": 0, + "p95": 0, + "p99": 0, + "min": 0, + "max": 0, + "total": 0, + "average": 0.0 + }, + "processedInputPositionsDistribution": { + "p25": 150000, + "p50": 150000, + "p75": 150000, + "p90": 150000, + "p95": 150000, + "p99": 150000, + "min": 150000, + "max": 150000, + "total": 150000, + "average": 150000.0 + }, + "outputDataSizeDistribution": { + "p25": 18, + "p50": 18, + "p75": 18, + "p90": 18, + "p95": 18, + "p99": 18, + "min": 18, + "max": 18, + "total": 18, + "average": 18.0 + }, + "outputPositionsDistribution": { + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + "totalDriversDistribution": { + "p25": 2, + "p50": 2, + "p75": 2, + "p90": 2, + "p95": 2, + "p99": 2, + "min": 2, + "max": 2, + "total": 2, + "average": 2.0 + }, + "createTimeMillisDistribution": { + "p01": 1646, + "p05": 1646, + "p10": 1646, + "p25": 1646, + "p50": 1646, + "p75": 1646, + "p90": 1646, + "p95": 1646, + "p99": 1646, + "min": 1646, + "max": 1646, + "total": 1646, + "average": 1646.0 + }, + "firstStartTimeMillisDistribution": { + "p01": 1848, + "p05": 1848, + "p10": 1848, + "p25": 1848, + "p50": 1848, + "p75": 1848, + "p90": 1848, + "p95": 1848, + "p99": 1848, + "min": 1848, + "max": 1848, + "total": 1848, + "average": 1848.0 + }, + "lastStartTimeMillisDistribution": { + "p01": 1848, + "p05": 1848, + "p10": 1848, + "p25": 1848, + "p50": 1848, + "p75": 1848, + "p90": 1848, + "p95": 1848, + "p99": 1848, + "min": 1848, + "max": 1848, + "total": 1848, + "average": 1848.0 + }, + "terminatingStartTimeMillisDistribution": { + "p01": 0, + "p05": 0, + "p10": 0, + "p25": 0, + "p50": 0, + "p75": 0, + "p90": 0, + "p95": 0, + "p99": 0, + "min": 0, + "max": 0, + "total": 0, + "average": 0.0 + }, + "lastEndTimeMillisDistribution": { + "p01": 1887, + "p05": 1887, + "p10": 1887, + "p25": 1887, + "p50": 1887, + "p75": 1887, + "p90": 1887, + "p95": 1887, + "p99": 1887, + "min": 1887, + "max": 1887, + "total": 1887, + "average": 1887.0 + }, + "endTimeMillisDistribution": { + "p01": 1892, + "p05": 1892, + "p10": 1892, + "p25": 1892, + "p50": 1892, + "p75": 1892, + "p90": 1892, + "p95": 1892, + "p99": 1892, + "min": 1892, + "max": 1892, + "total": 1892, + "average": 1892.0 + }, + "createTimeScaledDistribution": { + "p01": 0.8132411067193676, + "p05": 0.8132411067193676, + "p10": 0.8132411067193676, + "p25": 0.8132411067193676, + "p50": 0.8132411067193676, + "p75": 0.8132411067193676, + "p90": 0.8132411067193676, + "p95": 0.8132411067193676, + "p99": 0.8132411067193676, + "min": 0.8132411067193676, + "max": 0.8132411067193676, + "total": 1646.0, + "average": 0.8132411067193676 + }, + "firstStartTimeScaledDistribution": { + "p01": 0.9130434782608695, + "p05": 0.9130434782608695, + "p10": 0.9130434782608695, + "p25": 0.9130434782608695, + "p50": 0.9130434782608695, + "p75": 0.9130434782608695, + "p90": 0.9130434782608695, + "p95": 0.9130434782608695, + "p99": 0.9130434782608695, + "min": 0.9130434782608695, + "max": 0.9130434782608695, + "total": 1848.0, + "average": 0.9130434782608695 + }, + "lastStartTimeScaledDistribution": { + "p01": 0.9130434782608695, + "p05": 0.9130434782608695, + "p10": 0.9130434782608695, + "p25": 0.9130434782608695, + "p50": 0.9130434782608695, + "p75": 0.9130434782608695, + "p90": 0.9130434782608695, + "p95": 0.9130434782608695, + "p99": 0.9130434782608695, + "min": 0.9130434782608695, + "max": 0.9130434782608695, + "total": 1848.0, + "average": 0.9130434782608695 + }, + "terminatingStartTimeScaledDistribution": { + "p01": 0.0, + "p05": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 0.0, + "p75": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0, + "min": 0.0, + "max": 0.0, + "total": 0.0, + "average": 0.0 + }, + "lastEndTimeScaledDistribution": { + "p01": 0.9323122529644269, + "p05": 0.9323122529644269, + "p10": 0.9323122529644269, + "p25": 0.9323122529644269, + "p50": 0.9323122529644269, + "p75": 0.9323122529644269, + "p90": 0.9323122529644269, + "p95": 0.9323122529644269, + "p99": 0.9323122529644269, + "min": 0.9323122529644269, + "max": 0.9323122529644269, + "total": 1887.0, + "average": 0.9323122529644269 + }, + "endTimeScaledDistribution": { + "p01": 0.9347826086956522, + "p05": 0.9347826086956522, + "p10": 0.9347826086956522, + "p25": 0.9347826086956522, + "p50": 0.9347826086956522, + "p75": 0.9347826086956522, + "p90": 0.9347826086956522, + "p95": 0.9347826086956522, + "p99": 0.9347826086956522, + "min": 0.9347826086956522, + "max": 0.9347826086956522, + "total": 1892.0, + "average": 0.9347826086956522 + }, + "getSplitDistribution": { + "p01": 238218.0, + "p05": 238218.0, + "p10": 238218.0, + "p25": 238218.0, + "p50": 238218.0, + "p75": 238218.0, + "p90": 238218.0, + "p95": 238218.0, + "p99": 238218.0, + "min": 238218.0, + "max": 238218.0, + "total": 238218.0, + "average": 1.0 + } + } + ], + "operatorSummaries": [ + "{\"stageId\":0,\"pipelineId\":0,\"operatorId\":0,\"planNodeId\":\"189\",\"operatorType\":\"ExchangeOperator\",\"totalDrivers\":4,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"88B\",\"internalNetworkInputPositions\":2,\"rawInputDataSize\":\"88B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":2,\"getOutputWall\":\"207.91us\",\"getOutputCpu\":\"202.83us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.25256E-4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.04305554699999999,\"max\":0.043296167999999996,\"p01\":0.04305554699999999,\"p05\":0.04305554699999999,\"p10\":0.04305554699999999,\"p25\":0.04305554699999999,\"p50\":0.04305554699999999,\"p75\":0.04305554699999999,\"p90\":0.04305554699999999,\"p95\":0.04305554699999999,\"p99\":0.04305554699999999},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.2262999999999998E-4,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"172.71ms\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"56B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"56B\",\"spilledDataSize\":\"0B\",\"info\":{\"@type\":\"directExchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":100,\"averageBytesPerRequest\":27,\"successfulRequestsCount\":12,\"bufferedPages\":0,\"spilledPages\":0,\"spilledBytes\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[],\"requestDuration\":{\"digest\":\"AAAAAAAAAPA/AAAAAAAAQ0AAAAAAAABZQAAAAAAAAAhAAwAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAABDQAAAAAAAAPA/AAAAAAAA8D8AAAAAAADwPw==\",\"min\":1.0,\"max\":38.0,\"p25\":1.0,\"p50\":1.0,\"p75\":38.0,\"p90\":38.0,\"p95\":38.0,\"p99\":38.0,\"total\":3,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0}}}", + "{\"stageId\":0,\"pipelineId\":1,\"operatorId\":0,\"planNodeId\":\"183\",\"operatorType\":\"LocalExchangeSourceOperator\",\"totalDrivers\":1,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":4.0,\"getOutputCalls\":2,\"getOutputWall\":\"50.27us\",\"getOutputCpu\":\"44.53us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":5.027399999999999E-5,\"max\":5.027399999999999E-5,\"p01\":5.027399999999999E-5,\"p05\":5.027399999999999E-5,\"p10\":5.027399999999999E-5,\"p25\":5.027399999999999E-5,\"p50\":5.027399999999999E-5,\"p75\":5.027399999999999E-5,\"p90\":5.027399999999999E-5,\"p95\":5.027399999999999E-5,\"p99\":5.027399999999999E-5},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":0.043062728999999994,\"max\":0.043062728999999994,\"p01\":0.043062728999999994,\"p05\":0.043062728999999994,\"p10\":0.043062728999999994,\"p25\":0.043062728999999994,\"p50\":0.043062728999999994,\"p75\":0.043062728999999994,\"p90\":0.043062728999999994,\"p95\":0.043062728999999994,\"p99\":0.043062728999999994},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.0,\"max\":2.0,\"p01\":2.0,\"p05\":2.0,\"p10\":2.0,\"p25\":2.0,\"p50\":2.0,\"p75\":2.0,\"p90\":2.0,\"p95\":2.0,\"p99\":2.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":4.453399999999999E-5,\"max\":4.453399999999999E-5,\"p01\":4.453399999999999E-5,\"p05\":4.453399999999999E-5,\"p10\":4.453399999999999E-5,\"p25\":4.453399999999999E-5,\"p50\":4.453399999999999E-5,\"p75\":4.453399999999999E-5,\"p90\":4.453399999999999E-5,\"p95\":4.453399999999999E-5,\"p99\":4.453399999999999E-5}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"43.06ms\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"}", + "{\"stageId\":0,\"pipelineId\":1,\"operatorId\":2,\"planNodeId\":\"9\",\"operatorType\":\"TaskOutputOperator\",\"totalDrivers\":1,\"addInputCalls\":1,\"addInputWall\":\"187.13us\",\"addInputCpu\":\"186.74us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"9B\",\"inputPositions\":1,\"sumSquaredInputPositions\":1.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":1.8942599999999997E-4,\"max\":1.8942599999999997E-4,\"p01\":1.8942599999999997E-4,\"p05\":1.8942599999999997E-4,\"p10\":1.8942599999999997E-4,\"p25\":1.8942599999999997E-4,\"p50\":1.8942599999999997E-4,\"p75\":1.8942599999999997E-4,\"p90\":1.8942599999999997E-4,\"p95\":1.8942599999999997E-4,\"p99\":1.8942599999999997E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":1.0,\"max\":1.0,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0,\"p25\":1.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":1.8957499999999996E-4,\"max\":1.8957499999999996E-4,\"p01\":1.8957499999999996E-4,\"p05\":1.8957499999999996E-4,\"p10\":1.8957499999999996E-4,\"p25\":1.8957499999999996E-4,\"p50\":1.8957499999999996E-4,\"p75\":1.8957499999999996E-4,\"p90\":1.8957499999999996E-4,\"p95\":1.8957499999999996E-4,\"p99\":1.8957499999999996E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":1,\"finishWall\":\"2.29us\",\"finishCpu\":\"2.84us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"64B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"64B\",\"spilledDataSize\":\"0B\"}", + "{\"stageId\":0,\"pipelineId\":0,\"operatorId\":1,\"planNodeId\":\"183\",\"operatorType\":\"LocalExchangeSinkOperator\",\"totalDrivers\":4,\"addInputCalls\":2,\"addInputWall\":\"71.90us\",\"addInputCpu\":\"72.57us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":3.602699999999999E-5,\"max\":6.705499999999999E-5,\"p01\":3.602699999999999E-5,\"p05\":3.602699999999999E-5,\"p10\":3.602699999999999E-5,\"p25\":3.602699999999999E-5,\"p50\":3.602699999999999E-5,\"p75\":3.602699999999999E-5,\"p90\":3.602699999999999E-5,\"p95\":3.602699999999999E-5,\"p99\":3.602699999999999E-5},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":0.0,\"max\":1.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":4,\"min\":3.2099999999999994E-5,\"max\":6.4351E-5,\"p01\":3.2099999999999994E-5,\"p05\":3.2099999999999994E-5,\"p10\":3.2099999999999994E-5,\"p25\":3.2099999999999994E-5,\"p50\":3.2099999999999994E-5,\"p75\":3.2099999999999994E-5,\"p90\":3.2099999999999994E-5,\"p95\":3.2099999999999994E-5,\"p99\":3.2099999999999994E-5}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":4,\"finishWall\":\"151.01us\",\"finishCpu\":\"140.01us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"}", + "{\"stageId\":0,\"pipelineId\":1,\"operatorId\":1,\"planNodeId\":\"4\",\"operatorType\":\"AggregationOperator\",\"totalDrivers\":1,\"addInputCalls\":2,\"addInputWall\":\"154.96us\",\"addInputCpu\":\"136.16us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":4.0,\"getOutputCalls\":6,\"getOutputWall\":\"84.45us\",\"getOutputCpu\":\"81.76us\",\"outputDataSize\":\"9B\",\"outputPositions\":1,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.5230399999999996E-4,\"max\":2.5230399999999996E-4,\"p01\":2.5230399999999996E-4,\"p05\":2.5230399999999996E-4,\"p10\":2.5230399999999996E-4,\"p25\":2.5230399999999996E-4,\"p50\":2.5230399999999996E-4,\"p75\":2.5230399999999996E-4,\"p90\":2.5230399999999996E-4,\"p95\":2.5230399999999996E-4,\"p99\":2.5230399999999996E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.0,\"max\":2.0,\"p01\":2.0,\"p05\":2.0,\"p10\":2.0,\"p25\":2.0,\"p50\":2.0,\"p75\":2.0,\"p90\":2.0,\"p95\":2.0,\"p99\":2.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":1,\"min\":2.2891999999999997E-4,\"max\":2.2891999999999997E-4,\"p01\":2.2891999999999997E-4,\"p05\":2.2891999999999997E-4,\"p10\":2.2891999999999997E-4,\"p25\":2.2891999999999997E-4,\"p50\":2.2891999999999997E-4,\"p75\":2.2891999999999997E-4,\"p90\":2.2891999999999997E-4,\"p95\":2.2891999999999997E-4,\"p99\":2.2891999999999997E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":1,\"finishWall\":\"12.89us\",\"finishCpu\":\"11.01us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"24B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"24B\",\"spilledDataSize\":\"0B\"}", + "{\"stageId\":1,\"pipelineId\":0,\"operatorId\":0,\"planNodeId\":\"0\",\"operatorType\":\"TableScanOperator\",\"totalDrivers\":2,\"addInputCalls\":0,\"addInputWall\":\"0.00ns\",\"addInputCpu\":\"0.00ns\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":150000,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"0B\",\"inputPositions\":150000,\"sumSquaredInputPositions\":1.125E10,\"getOutputCalls\":38,\"getOutputWall\":\"75.57ms\",\"getOutputCpu\":\"75.54ms\",\"outputDataSize\":\"0B\",\"outputPositions\":150000,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.03748956999999999,\"max\":0.038077997999999995,\"p01\":0.03748956999999999,\"p05\":0.03748956999999999,\"p10\":0.03748956999999999,\"p25\":0.03748956999999999,\"p50\":0.03748956999999999,\"p75\":0.03748956999999999,\"p90\":0.03748956999999999,\"p95\":0.03748956999999999,\"p99\":0.03748956999999999},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":75000.0,\"max\":75000.0,\"p01\":75000.0,\"p05\":75000.0,\"p10\":75000.0,\"p25\":75000.0,\"p50\":75000.0,\"p75\":75000.0,\"p90\":75000.0,\"p95\":75000.0,\"p99\":75000.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.037475165,\"max\":0.038066303999999995,\"p01\":0.037475165,\"p05\":0.037475165,\"p10\":0.037475165,\"p25\":0.037475165,\"p50\":0.037475165,\"p75\":0.037475165,\"p90\":0.037475165,\"p95\":0.037475165,\"p99\":0.037475165}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":0,\"finishWall\":\"0.00ns\",\"finishCpu\":\"0.00ns\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"0B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"0B\",\"spilledDataSize\":\"0B\"}", + "{\"stageId\":1,\"pipelineId\":0,\"operatorId\":2,\"planNodeId\":\"187\",\"operatorType\":\"TaskOutputOperator\",\"totalDrivers\":2,\"addInputCalls\":2,\"addInputWall\":\"648.52us\",\"addInputCpu\":\"594.65us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"18B\",\"inputPositions\":2,\"sumSquaredInputPositions\":2.0,\"getOutputCalls\":0,\"getOutputWall\":\"0.00ns\",\"getOutputCpu\":\"0.00ns\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.33581E-4,\"max\":5.209589999999999E-4,\"p01\":1.33581E-4,\"p05\":1.33581E-4,\"p10\":1.33581E-4,\"p25\":1.33581E-4,\"p50\":1.33581E-4,\"p75\":1.33581E-4,\"p90\":1.33581E-4,\"p95\":1.33581E-4,\"p99\":1.33581E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.0,\"max\":1.0,\"p01\":1.0,\"p05\":1.0,\"p10\":1.0,\"p25\":1.0,\"p50\":1.0,\"p75\":1.0,\"p90\":1.0,\"p95\":1.0,\"p99\":1.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.33901E-4,\"max\":4.6756599999999993E-4,\"p01\":1.33901E-4,\"p05\":1.33901E-4,\"p10\":1.33901E-4,\"p25\":1.33901E-4,\"p50\":1.33901E-4,\"p75\":1.33901E-4,\"p90\":1.33901E-4,\"p95\":1.33901E-4,\"p99\":1.33901E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":2,\"finishWall\":\"6.02us\",\"finishCpu\":\"6.81us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"64B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"64B\",\"spilledDataSize\":\"0B\"}", + "{\"stageId\":1,\"pipelineId\":0,\"operatorId\":1,\"planNodeId\":\"187\",\"operatorType\":\"AggregationOperator\",\"totalDrivers\":2,\"addInputCalls\":2,\"addInputWall\":\"169.01us\",\"addInputCpu\":\"169.52us\",\"physicalInputDataSize\":\"0B\",\"physicalInputPositions\":0,\"physicalInputReadTime\":\"0.00ns\",\"internalNetworkInputDataSize\":\"0B\",\"internalNetworkInputPositions\":0,\"rawInputDataSize\":\"0B\",\"inputDataSize\":\"0B\",\"inputPositions\":150000,\"sumSquaredInputPositions\":1.125E10,\"getOutputCalls\":38,\"getOutputWall\":\"214.28us\",\"getOutputCpu\":\"233.72us\",\"outputDataSize\":\"18B\",\"outputPositions\":2,\"dynamicFilterSplitsProcessed\":0,\"metrics\":{\"Scheduled time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.8643999999999998E-4,\"max\":2.0529899999999996E-4,\"p01\":1.8643999999999998E-4,\"p05\":1.8643999999999998E-4,\"p10\":1.8643999999999998E-4,\"p25\":1.8643999999999998E-4,\"p50\":1.8643999999999998E-4,\"p75\":1.8643999999999998E-4,\"p90\":1.8643999999999998E-4,\"p95\":1.8643999999999998E-4,\"p99\":1.8643999999999998E-4},\"Blocked time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":0.0,\"max\":0.0,\"p01\":0.0,\"p05\":0.0,\"p10\":0.0,\"p25\":0.0,\"p50\":0.0,\"p75\":0.0,\"p90\":0.0,\"p95\":0.0,\"p99\":0.0},\"Input rows distribution\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":75000.0,\"max\":75000.0,\"p01\":75000.0,\"p05\":75000.0,\"p10\":75000.0,\"p25\":75000.0,\"p50\":75000.0,\"p75\":75000.0,\"p90\":75000.0,\"p95\":75000.0,\"p99\":75000.0},\"CPU time distribution (s)\":{\"@class\":\"io.trino.execution.DistributionSnapshot\",\"total\":2,\"min\":1.9570699999999996E-4,\"max\":2.1664699999999996E-4,\"p01\":1.9570699999999996E-4,\"p05\":1.9570699999999996E-4,\"p10\":1.9570699999999996E-4,\"p25\":1.9570699999999996E-4,\"p50\":1.9570699999999996E-4,\"p75\":1.9570699999999996E-4,\"p90\":1.9570699999999996E-4,\"p95\":1.9570699999999996E-4,\"p99\":1.9570699999999996E-4}},\"connectorMetrics\":{},\"pipelineMetrics\":{},\"physicalWrittenDataSize\":\"0B\",\"blockedWall\":\"0.00ns\",\"finishCalls\":2,\"finishWall\":\"8.46us\",\"finishCpu\":\"9.12us\",\"userMemoryReservation\":\"0B\",\"revocableMemoryReservation\":\"0B\",\"peakUserMemoryReservation\":\"24B\",\"peakRevocableMemoryReservation\":\"0B\",\"peakTotalMemoryReservation\":\"24B\",\"spilledDataSize\":\"0B\"}" + ], + "optimizerRulesSummaries": [ + { + "rule": "io.trino.sql.planner.optimizations.PredicatePushDown", + "invocations": 7, + "applied": 7, + "totalTime": 18612026, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.optimizations.AddExchanges", + "invocations": 1, + "applied": 1, + "totalTime": 16965364, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.PushPartialAggregationThroughExchange", + "invocations": 6, + "applied": 3, + "totalTime": 15761523, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.ExpressionRewriteRuleSet.ProjectExpressionRewrite", + "invocations": 86, + "applied": 0, + "totalTime": 8120470, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.PruneCountAggregationOverScalar", + "invocations": 3, + "applied": 0, + "totalTime": 3069724, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.PushAggregationIntoTableScan", + "invocations": 3, + "applied": 0, + "totalTime": 2300790, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.PruneProjectColumns", + "invocations": 6, + "applied": 3, + "totalTime": 1877015, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.PruneTableScanColumns", + "invocations": 6, + "applied": 1, + "totalTime": 1592621, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.DetermineTableScanNodePartitioning", + "invocations": 3, + "applied": 1, + "totalTime": 1580199, + "failures": 0 + }, + { + "rule": "io.trino.sql.planner.iterative.rule.PushDownDereferenceThroughProject", + "invocations": 6, + "applied": 0, + "totalTime": 1419236, + "failures": 0 + } + ], + "planNodeStatsAndCosts": "{\"stats\":{\"9\":{\"outputRowCount\":1.0,\"symbolStatistics\":{}},\"4\":{\"outputRowCount\":1.0,\"symbolStatistics\":{}},\"183\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}},\"189\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}},\"187\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}},\"0\":{\"outputRowCount\":\"NaN\",\"symbolStatistics\":{}}},\"costs\":{\"9\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":9.0,\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":0.0}},\"4\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":9.0,\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":\"NaN\",\"maxMemory\":9.0,\"networkCost\":0.0}},\"183\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":\"NaN\",\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":0.0}},\"189\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":\"NaN\",\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":\"NaN\"}},\"187\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"maxMemoryWhenOutputting\":\"NaN\",\"networkCost\":\"NaN\",\"rootNodeLocalCostEstimate\":{\"cpuCost\":\"NaN\",\"maxMemory\":\"NaN\",\"networkCost\":\"NaN\"}},\"0\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"maxMemoryWhenOutputting\":0.0,\"networkCost\":0.0,\"rootNodeLocalCostEstimate\":{\"cpuCost\":0.0,\"maxMemory\":0.0,\"networkCost\":0.0}}}}" + }, + "context": { + "user": "admin", + "originalUser": "admin", + "principal": "admin", + "enabledRoles": [], + "groups": [], + "remoteClientAddress": "100.103.68.12", + "userAgent": "StatementClientV1/447", + "clientTags": [], + "clientCapabilities": [ + "PATH", + "PARAMETRIC_DATETIME", + "SESSION_AUTHORIZATION" + ], + "source": "trino-cli", + "timezone": "Europe/Berlin", + "resourceGroupId": [ + "global" + ], + "sessionProperties": {}, + "resourceEstimates": {}, + "serverAddress": "trino-m-1-coordinator-default-0.trino-m-1-coordinator-default.default.svc.cluster.local", + "serverVersion": "455", + "environment": "trino_m_1", + "queryType": "SELECT", + "retryPolicy": "NONE" + }, + "ioMetadata": { + "inputs": [ + { + "catalogName": "tpch", + "catalogVersion": "default", + "schema": "sf1", + "table": "customer", + "columns": [], + "connectorMetrics": {}, + "physicalInputBytes": 0, + "physicalInputRows": 150000 + } + ] + }, + "warnings": [], + "createTime": "2025-03-28T10:14:56.382Z", + "executionStartTime": "2025-03-28T10:14:57.695Z", + "endTime": "2025-03-28T10:14:58.406Z" +} diff --git a/trino-lb-persistence/src/in_memory.rs b/trino-lb-persistence/src/in_memory.rs index 6f01139..c3111cd 100644 --- a/trino-lb-persistence/src/in_memory.rs +++ b/trino-lb-persistence/src/in_memory.rs @@ -29,9 +29,6 @@ pub enum Error { #[snafu(display("Queued query with id {queued_query_id:?} not found"))] QueuedQueryNotFound { queued_query_id: TrinoLbQueryId }, - #[snafu(display("Query with id {query_id:?} not found"))] - QueryNotFound { query_id: TrinoQueryId }, - #[snafu(display("Failed to determined elapsed time since last queryCountFetcher update"))] DetermineElapsedTimeSinceLastUpdate { source: SystemTimeError }, @@ -93,12 +90,12 @@ impl Persistence for InMemoryPersistence { } #[instrument(skip(self))] - async fn load_query(&self, query_id: &TrinoQueryId) -> Result { + async fn load_query( + &self, + query_id: &TrinoQueryId, + ) -> Result, super::Error> { let queries = self.queries.read().await; - Ok(queries - .get(query_id) - .context(QueryNotFoundSnafu { query_id })? - .clone()) + Ok(queries.get(query_id).cloned()) } #[instrument(skip(self))] diff --git a/trino-lb-persistence/src/lib.rs b/trino-lb-persistence/src/lib.rs index 0187d96..abeec7e 100644 --- a/trino-lb-persistence/src/lib.rs +++ b/trino-lb-persistence/src/lib.rs @@ -37,7 +37,7 @@ pub trait Persistence { async fn remove_queued_query(&self, query: &QueuedQuery) -> Result<(), Error>; async fn store_query(&self, query: TrinoQuery) -> Result<(), Error>; - async fn load_query(&self, query_id: &TrinoQueryId) -> Result; + async fn load_query(&self, query_id: &TrinoQueryId) -> Result, Error>; async fn remove_query(&self, query_id: &TrinoQueryId) -> Result<(), Error>; /// `max_allowed_count` is the (inclusive) maximum count that is allowed *after* the increment. diff --git a/trino-lb-persistence/src/postgres/mod.rs b/trino-lb-persistence/src/postgres/mod.rs index 710eb31..b92c568 100644 --- a/trino-lb-persistence/src/postgres/mod.rs +++ b/trino-lb-persistence/src/postgres/mod.rs @@ -220,17 +220,24 @@ impl Persistence for PostgresPersistence { } #[instrument(skip(self))] - async fn load_query(&self, query_id: &TrinoQueryId) -> Result { + async fn load_query( + &self, + query_id: &TrinoQueryId, + ) -> Result, super::Error> { let result = query!( r#"SELECT id, trino_cluster, trino_endpoint, creation_time, delivered_time FROM queries WHERE id = $1"#, query_id, ) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await .context(LoadQuerySnafu)?; + let Some(result) = result else { + return Ok(None); + }; + let query = TrinoQuery { id: result.id, trino_cluster: result.trino_cluster, @@ -240,7 +247,7 @@ impl Persistence for PostgresPersistence { delivered_time: result.delivered_time.into(), }; - Ok(query) + Ok(Some(query)) } #[instrument(skip(self))] diff --git a/trino-lb-persistence/src/redis/mod.rs b/trino-lb-persistence/src/redis/mod.rs index d0c6de1..a1a7f8f 100644 --- a/trino-lb-persistence/src/redis/mod.rs +++ b/trino-lb-persistence/src/redis/mod.rs @@ -251,17 +251,26 @@ where } #[instrument(skip(self))] - async fn load_query(&self, query_id: &TrinoQueryId) -> Result { + async fn load_query( + &self, + query_id: &TrinoQueryId, + ) -> Result, super::Error> { let key = query_key(query_id); - let value: Vec = self + + let value: Option> = self .connection() .get(key) .await .context(ReadFromRedisSnafu)?; - - Ok(bincode::serde::decode_from_slice(&value, BINCODE_CONFIG) - .context(DeserializeFromBinarySnafu)? - .0) + let Some(value) = value else { + return Ok(None); + }; + + Ok(Some( + bincode::serde::decode_from_slice(&value, BINCODE_CONFIG) + .context(DeserializeFromBinarySnafu)? + .0, + )) } #[instrument(skip(self))] diff --git a/trino-lb/src/cluster_group_manager.rs b/trino-lb/src/cluster_group_manager.rs index 4bd0272..8c0abf2 100644 --- a/trino-lb/src/cluster_group_manager.rs +++ b/trino-lb/src/cluster_group_manager.rs @@ -12,7 +12,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{Instrument, debug, info_span, instrument}; use tracing_opentelemetry::OpenTelemetrySpanExt; use trino_lb_core::{ - config::Config, sanitization::Sanitize, trino_api::TrinoQueryApiResponse, + config::Config, sanitization::Sanitize, trino_api::queries::TrinoQueryApiResponse, trino_cluster::ClusterState, trino_query::TrinoQuery, }; use trino_lb_persistence::{Persistence, PersistenceImplementation}; @@ -247,7 +247,7 @@ impl ClusterGroupManager { } #[instrument( - skip(self), + skip(self, query, requested_path), fields(request_headers = ?request_headers.sanitize()) )] pub async fn cancel_query_on_trino( diff --git a/trino-lb/src/http_server/mod.rs b/trino-lb/src/http_server/mod.rs index d157b04..f458718 100644 --- a/trino-lb/src/http_server/mod.rs +++ b/trino-lb/src/http_server/mod.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, fmt::Debug, net::{Ipv6Addr, SocketAddr}, path::PathBuf, @@ -15,9 +16,13 @@ use axum_server::{Handle, tls_rustls::RustlsConfig}; use futures::FutureExt; use snafu::{OptionExt, ResultExt, Snafu}; use tokio::time::sleep; -use tower_http::{compression::CompressionLayer, trace::TraceLayer}; +use tower_http::{ + compression::CompressionLayer, decompression::RequestDecompressionLayer, trace::TraceLayer, +}; use tracing::info; +use trino_lb_core::config::TrinoClusterConfig; use trino_lb_persistence::PersistenceImplementation; +use url::Url; use crate::{ cluster_group_manager::ClusterGroupManager, config::Config, metrics::Metrics, routing, @@ -29,6 +34,19 @@ mod v1; #[derive(Snafu, Debug)] pub enum Error { + #[snafu(display("Failed to extract Trino host from given Trino endpoint {endpoint}"))] + ExtractTrinoHostFromEndpoint { endpoint: Url }, + + #[snafu(display( + "The clusters \"{first_cluster}\" and \"{second_cluster}\" share the same host \"{host}\". \ + This is bad, because we don't know from which cluster a certain Trino HTTP even was send from" + ))] + DuplicateTrinoClusterHost { + first_cluster: String, + second_cluster: String, + host: String, + }, + #[snafu(display( "Failed configure HTTP server PEM cert at {cert_pem_file:?} and PEM key at {key_pem_file:?}" ))] @@ -52,6 +70,8 @@ pub struct AppState { persistence: Arc, cluster_group_manager: ClusterGroupManager, router: routing::Router, + /// Maps from the Cluster host to the name of the cluster + cluster_name_for_host: HashMap, metrics: Arc, } @@ -64,11 +84,45 @@ pub async fn start_http_server( ) -> Result<(), Error> { let tls_config = config.trino_lb.tls.clone(); let ports_config = config.trino_lb.ports.clone(); + + let mut cluster_name_for_host = HashMap::new(); + let clusters = config + .trino_cluster_groups + .values() + .flat_map(|group_config| &group_config.trino_clusters); + for TrinoClusterConfig { + name, + alternative_hostnames, + endpoint, + .. + } in clusters + { + let host = endpoint + .host_str() + .with_context(|| ExtractTrinoHostFromEndpointSnafu { + endpoint: endpoint.clone(), + })?; + + for name in std::iter::once(name).chain(alternative_hostnames.iter()) { + if let Some(first_cluster) = + cluster_name_for_host.insert(host.to_owned(), name.to_owned()) + { + DuplicateTrinoClusterHostSnafu { + first_cluster, + second_cluster: name, + host, + } + .fail()?; + } + } + } + let app_state = Arc::new(AppState { config, persistence, cluster_group_manager, router, + cluster_name_for_host, metrics, }); @@ -121,9 +175,24 @@ pub async fn start_http_server( "/v1/statement/executing/{query_id}/{slug}/{token}", delete(v1::statement::delete_trino_executing_statement), ) + .route( + "/v1/trino-event-listener", + post(v1::trino_event_listener::post_trino_event_listener), + ) .route("/ui/index.html", get(ui::index::get_ui_index)) .route("/ui/query.html", get(ui::query::get_ui_query)) .layer(TraceLayer::new_for_http()) + // Transparently decompress request bodies based on the + // Content-Encoding header. + // + // The Trino HTTP events (received at `/v1/trino-event-listener`) are + // compressed by default, so we need to be able to accept compressed + // content. + .layer(RequestDecompressionLayer::new()) + // Compress response bodies if the associated request had an + // Accept-Encoding header. + // + // Trino clients can ask for compressed data, so we should support compressing the response .layer(CompressionLayer::new()) .with_state(app_state); diff --git a/trino-lb/src/http_server/v1/mod.rs b/trino-lb/src/http_server/v1/mod.rs index 085be5b..98c4ec5 100644 --- a/trino-lb/src/http_server/v1/mod.rs +++ b/trino-lb/src/http_server/v1/mod.rs @@ -1 +1,2 @@ pub mod statement; +pub mod trino_event_listener; diff --git a/trino-lb/src/http_server/v1/statement.rs b/trino-lb/src/http_server/v1/statement.rs index ef62ff2..ccaca06 100644 --- a/trino-lb/src/http_server/v1/statement.rs +++ b/trino-lb/src/http_server/v1/statement.rs @@ -14,13 +14,14 @@ use axum::{ use futures::TryFutureExt; use http::{HeaderMap, StatusCode, Uri}; use opentelemetry::KeyValue; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use tokio::time::Instant; use tracing::{Instrument, debug, info, info_span, instrument, warn}; use trino_lb_core::{ TrinoLbQueryId, TrinoQueryId, + config::TrinoLbProxyMode, sanitization::Sanitize, - trino_api::TrinoQueryApiResponse, + trino_api::queries::TrinoQueryApiResponse, trino_query::{QueuedQuery, TrinoQuery}, }; use trino_lb_persistence::Persistence; @@ -36,12 +37,12 @@ use crate::{ pub enum Error { #[snafu(display("Failed to modify nextUri trino send us to point tu trino-lb"))] ModifyNextUri { - source: trino_lb_core::trino_api::Error, + source: trino_lb_core::trino_api::queries::Error, }, #[snafu(display("Failed to convert queued query to trino query"))] ConvertQueuedQueryToTrinoQuery { - source: trino_lb_core::trino_api::Error, + source: trino_lb_core::trino_api::queries::Error, }, #[snafu(display("Failed to store queued query in persistence"))] @@ -59,12 +60,21 @@ pub enum Error { query_id: TrinoLbQueryId, }, + #[snafu(display("Query with id {query_id:?} not found"))] + QueryNotFound { query_id: TrinoQueryId }, + #[snafu(display("Failed to store query in persistence"))] StoreQueryInPersistence { source: trino_lb_persistence::Error, query_id: TrinoQueryId, }, + #[snafu(display("Failed to delete query from persistence"))] + DeleteQueryFromPersistence { + source: trino_lb_persistence::Error, + query_id: TrinoQueryId, + }, + #[snafu(display("Failed to load query with id {query_id:?} from persistence"))] LoadQueryFromPersistence { source: trino_lb_persistence::Error, @@ -136,11 +146,7 @@ impl IntoResponse for Error { } /// This function gets a new query and decided wether to queue it or to send it to a Trino cluster directly. -#[instrument( - name = "POST /v1/statement", - skip(state), - fields(headers = ?headers.sanitize()), -)] +#[instrument(name = "POST /v1/statement", skip(state, headers, query))] pub async fn post_statement( headers: HeaderMap, State(state): State>, @@ -167,7 +173,7 @@ pub async fn post_statement( /// It either replies with "please hold the line" or forwards the query to an Trino cluster. #[instrument( name = "GET /v1/statement/queued_in_trino_lb/{queryId}/{sequenceNumber}", - skip(state) + skip(state, sequence_number) )] pub async fn get_trino_lb_statement( State(state): State>, @@ -195,8 +201,7 @@ pub async fn get_trino_lb_statement( /// In case the nextUri is null, the query will be stopped and removed from trino-lb. #[instrument( name = "GET /v1/statement/queued/{queryId}/{slug}/{token}", - skip(state), - fields(headers = ?headers.sanitize()), + skip(state, headers) )] pub async fn get_trino_queued_statement( headers: HeaderMap, @@ -218,8 +223,7 @@ pub async fn get_trino_queued_statement( /// In case the nextUri is null, the query will be stopped and removed from trino-lb. #[instrument( name = "GET /v1/statement/executing/{queryId}/{slug}/{token}", - skip(state), - fields(headers = ?headers.sanitize()), + skip(state, headers, uri) )] pub async fn get_trino_executing_statement( headers: HeaderMap, @@ -235,7 +239,7 @@ pub async fn get_trino_executing_statement( handle_query_running_on_trino(&state, headers, query_id, uri.path()).await } -#[instrument(skip(state, queued_query))] +#[instrument(skip_all)] async fn queue_or_hand_over_query( state: &Arc, queued_query: QueuedQuery, @@ -250,6 +254,7 @@ async fn queue_or_hand_over_query( last_accessed, cluster_group, } = &queued_query; + let proxy_mode = &state.config.trino_lb.proxy_mode; let start_of_request = Instant::now(); @@ -264,6 +269,7 @@ async fn queue_or_hand_over_query( cluster = cluster.name, "Found cluster that has sufficient space" ); + let has_increased = state .persistence .inc_cluster_query_count(&cluster.name.to_string(), cluster.max_running_queries) @@ -272,6 +278,39 @@ async fn queue_or_hand_over_query( trino_cluster: &cluster.name, })?; + if !has_increased { + debug!( + cluster = cluster.name, + "The cluster had enough space when asked for the best cluster, but inc_cluster_query_count returned None, \ + probably because the cluster has reached its maximum query count in the meantime" + ); + } + + // let cont = match proxy_mode { + // // Only continue when the increment was successful + // TrinoLbProxyMode::ProxyAllCalls => { + // let has_increased = state + // .persistence + // .inc_cluster_query_count(&cluster.name.to_string(), cluster.max_running_queries) + // .await + // .context(DecClusterQueryCounterSnafu { + // trino_cluster: &cluster.name, + // })?; + + // if !has_increased { + // debug!( + // cluster = cluster.name, + // "The cluster had enough space when asked for the best cluster, but inc_cluster_query_count returned None, \ + // probably because the cluster has reached its maximum query count in the meantime" + // ); + // } + + // has_increased + // } + // // Always continue, we don't store any query in the persistence + // TrinoLbProxyMode::ProxyFirstCall => true, + // }; + if has_increased { let mut send_to_trino_response = state .cluster_group_manager @@ -296,27 +335,43 @@ async fn queue_or_hand_over_query( ); if trino_query_api_response.next_uri.is_some() { - let query = TrinoQuery::new_from( - cluster.name.clone(), - trino_query_api_response.id.clone(), - cluster.endpoint.clone(), - *creation_time, - SystemTime::now(), - ); - let query_id = query.id.clone(); - - state.persistence.store_query(query).await.context( - StoreQueryInPersistenceSnafu { - query_id: &query_id, - }, - )?; - - trino_query_api_response - .change_next_uri_to_trino_lb(&state.config.trino_lb.external_address) - .context(ModifyNextUriSnafu)?; + match state.config.trino_lb.proxy_mode { + TrinoLbProxyMode::ProxyAllCalls => { + // Only store the query and change the nextURI to trino-lb in case it should proxy all + // calls + let query = TrinoQuery::new_from( + cluster.name.clone(), + trino_query_api_response.id.clone(), + cluster.endpoint.clone(), + *creation_time, + SystemTime::now(), + ); + let query_id = query.id.clone(); + + state.persistence.store_query(query).await.context( + StoreQueryInPersistenceSnafu { + query_id: &query_id, + }, + )?; + + trino_query_api_response + .change_next_uri_to_trino_lb( + &state.config.trino_lb.external_address, + ) + .context(ModifyNextUriSnafu)?; + } + TrinoLbProxyMode::ProxyFirstCall => { + // In case http-server.process-forwarded is set to true, Trino might choose the original + // host, which is trino-lb. But maybe also not, not entirely sure. As we *don't* want to + // send future calls to trino-lb, we need to actively change the nextUri to the Trino + // cluster. Better safe than sorry! + trino_query_api_response + .change_next_uri_to_trino(&cluster.endpoint) + .context(ModifyNextUriSnafu)?; + } + } info!( - query_id, trino_cluster_name = cluster.name, "Successfully handed query over to Trino cluster" ); @@ -327,19 +382,28 @@ async fn queue_or_hand_over_query( "Trino got our query but send no nextUri. Maybe an Syntax error or something similar?" ); - // The queued query will be removed from the persistence below. - // As the query is probably finished, lets decrement the query counter again. - state - .persistence - .dec_cluster_query_count(&cluster.name) - .await - .context(DecClusterQueryCounterSnafu { - trino_cluster: &cluster.name, - })?; + if proxy_mode == &TrinoLbProxyMode::ProxyAllCalls { + // The queued query will be removed from the persistence below. + // As the query is probably finished, lets decrement the query counter again. + state + .persistence + .dec_cluster_query_count(&cluster.name) + .await + .context(DecClusterQueryCounterSnafu { + trino_cluster: &cluster.name, + })?; + } + // We don't increment the query counter in ProxyFirstCall mode, as we assume + // we get a query event that the query finished (even if it fails). It might + // be the case that this assumption is wrong, but in this case we are better + // safe than sorry and don't decrement the counter, we'd rather have one + // query to few on the cluster instead of too much. A periodic query counter + // sync is in place anyway. } } SendToTrinoResponse::Unauthorized { .. } => { - // As the query was not actually started decrement the query counter again. + // As the query was not actually started decrement the query counter again + // (in all proxy modes) state .persistence .dec_cluster_query_count(&cluster.name) @@ -364,12 +428,6 @@ async fn queue_or_hand_over_query( } return Ok(send_to_trino_response); - } else { - debug!( - cluster = cluster.name, - "The cluster had enough space when asked for the best cluster, but inc_cluster_query_count returned None, \ - probably because the cluster has reached its maximum query count in the meantime" - ); } } @@ -418,24 +476,23 @@ async fn queue_or_hand_over_query( }) } -#[instrument( - skip(state), - fields(headers = ?headers.sanitize()), -)] +#[instrument(skip(state, headers))] async fn handle_query_running_on_trino( state: &Arc, headers: HeaderMap, query_id: TrinoQueryId, requested_path: &str, ) -> Result<(HeaderMap, Json), Error> { - let query = - state - .persistence - .load_query(&query_id) - .await - .context(StoreQueryInPersistenceSnafu { - query_id: query_id.clone(), - })?; + let query = state + .persistence + .load_query(&query_id) + .await + .with_context(|_| LoadQueryFromPersistenceSnafu { + query_id: query_id.clone(), + })? + .with_context(|| QueryNotFoundSnafu { + query_id: query_id.clone(), + })?; let (mut trino_query_api_response, trino_headers) = state .cluster_group_manager @@ -452,16 +509,18 @@ async fn handle_query_running_on_trino( .context(AskTrinoForQueryStateSnafu)?; if trino_query_api_response.next_uri.is_some() { - // Change the nextUri to actually point to trino-lb instead of Trino. - trino_query_api_response - .change_next_uri_to_trino_lb(&state.config.trino_lb.external_address) - .context(ModifyNextUriSnafu)?; + // Only change the nextURI to trino-lb in case it should proxy all calls + if state.config.trino_lb.proxy_mode == TrinoLbProxyMode::ProxyAllCalls { + trino_query_api_response + .change_next_uri_to_trino_lb(&state.config.trino_lb.external_address) + .context(ModifyNextUriSnafu)?; + } } else { info!(%query_id, "Query completed (no next_uri send)"); tokio::try_join!( state.persistence.remove_query(&query_id).map_err(|err| { - Error::DeleteQueuedQueryFromPersistence { + Error::DeleteQueryFromPersistence { source: err, query_id: query_id.to_owned(), } @@ -538,7 +597,7 @@ pub async fn delete_trino_queued_statement( /// This function get's asked to cancel a query that is already sent to an Trino cluster and currently running. #[instrument( name = "DELETE /v1/statement/executing/{queryId}/{slug}/{token}", - skip(state), + skip(state, uri), fields(headers = ?headers.sanitize()), )] pub async fn delete_trino_executing_statement( @@ -558,10 +617,7 @@ pub async fn delete_trino_executing_statement( cancel_query_on_trino(headers, &state, query_id, uri.path()).await } -#[instrument( - skip(state), - fields(headers = ?headers.sanitize()), -)] +#[instrument(skip(state, headers))] async fn cancel_query_on_trino( headers: HeaderMap, state: &Arc, @@ -573,14 +629,14 @@ async fn cancel_query_on_trino( .http_counter .add(1, &[KeyValue::new("resource", "cancel_query_on_trino")]); - let query = - state - .persistence - .load_query(&query_id) - .await - .context(StoreQueryInPersistenceSnafu { - query_id: query_id.clone(), - })?; + let query = state + .persistence + .load_query(&query_id) + .await + .with_context(|_| StoreQueryInPersistenceSnafu { + query_id: query_id.clone(), + })? + .context(QueryNotFoundSnafu { query_id })?; state .cluster_group_manager diff --git a/trino-lb/src/http_server/v1/trino_event_listener.rs b/trino-lb/src/http_server/v1/trino_event_listener.rs new file mode 100644 index 0000000..15b515d --- /dev/null +++ b/trino-lb/src/http_server/v1/trino_event_listener.rs @@ -0,0 +1,121 @@ +//! Listens for Trino HTTP events, which are pushed to this endpoint +//! +//! See + +use std::sync::Arc; + +use axum::{ + Json, + extract::State, + response::{IntoResponse, Response}, +}; +use http::StatusCode; +use opentelemetry::KeyValue; +use snafu::{ResultExt, Snafu}; +use tracing::{instrument, warn}; +use trino_lb_core::{ + config::TrinoLbProxyMode, + trino_api::trino_events::{TrinoEvent, TrinoQueryState}, +}; +use trino_lb_persistence::Persistence; + +use crate::http_server::AppState; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display( + "Failed to decrement the query counter query trino cluster {trino_cluster:?}" + ))] + DecClusterQueryCounter { + source: trino_lb_persistence::Error, + trino_cluster: String, + }, +} + +impl IntoResponse for Error { + fn into_response(self) -> Response { + warn!(error = ?self, "Error while processing Trino HTTP event"); + + // We use a match here to let authors of new errors decide if Trino should retry the call + // (which it only does for a >= 500 HTTP return code according to + // https://trino.io/docs/current/admin/event-listeners-http.html#configuration-properties). + match self { + // Could be caused by temporary problems, so we let Trino retry + Error::DecClusterQueryCounter { .. } => { + (StatusCode::INTERNAL_SERVER_ERROR, format!("{self:?}")).into_response() + } + } + } +} + +#[instrument( + name = "POST /v1/trino-event-listener", + skip(state, trino_event), + fields( + query_id = trino_event.query_id(), + query_state = %trino_event.query_state(), + ) +)] +pub async fn post_trino_event_listener( + State(state): State>, + trino_event: Json, +) -> Result<(), Error> { + state + .metrics + .http_counter + .add(1, &[KeyValue::new("resource", "post_trino_event_listener")]); + + if state.config.trino_lb.proxy_mode != TrinoLbProxyMode::ProxyFirstCall { + warn!( + "Received a Trino event, but trino-lb is configured to proxy all calls. \ + Thus it is ignoring the received Trino HTTP events. \ + Either don't send HTTP events to trino-lb or configure it to only proxy the first call." + ); + return Ok(()); + } + + let query_id = trino_event.query_id(); + let trino_host = trino_event.server_address(); + let uri = trino_event.uri(); + + let Some(cluster) = state.cluster_name_for_host.get(trino_host) else { + warn!( + query_id, + %uri, "Got a Trino query event for a cluster that I don't know." + ); + return Ok(()); + }; + + match trino_event.query_state() { + TrinoQueryState::Queued => { + // We don't do anything, the query counter is incremented once trino-lb submits a query + // to an Trino cluster. + + // // As there query is already running we are acting after the fact. + // // So we can not reject the query, even if too many query are already running + // let max_allowed_queries = u64::MAX; + // state + // .persistence + // .inc_cluster_query_count(cluster, max_allowed_queries) + // .await + // .with_context(|_| IncClusterQueryCounterSnafu { + // trino_cluster: cluster, + // })?; + } + TrinoQueryState::Finished => { + state + .persistence + .dec_cluster_query_count(cluster) + .await + .with_context(|_| DecClusterQueryCounterSnafu { + trino_cluster: cluster, + })?; + } + TrinoQueryState::Executing => warn!( + "Got a Trino query event for an running query. As you should only send query started or \ + finished query events, only queued or finished should be received! Ignoring this event." + ), + } + + Ok(()) +} diff --git a/trino-lb/src/maintenance/query_count_fetcher.rs b/trino-lb/src/maintenance/query_count_fetcher.rs index 8c15072..b43e014 100644 --- a/trino-lb/src/maintenance/query_count_fetcher.rs +++ b/trino-lb/src/maintenance/query_count_fetcher.rs @@ -155,7 +155,7 @@ impl QueryCountFetcher { /// - In case we know the cluster is not reachable (e.g. the cluster is turned off or currently /// starting), store a query count of zero (0) to avoid dangling clusters with non-zero query /// counts. - #[instrument(skip(self, cluster), fields(cluster.name))] + #[instrument(skip(self, cluster, state), fields(cluster.name))] async fn process_cluster(&self, cluster: &TrinoClusterConfig, state: ClusterState) { match state { ClusterState::Ready | ClusterState::Unhealthy | ClusterState::Draining { .. } => { diff --git a/trino-lb/src/scaling/mod.rs b/trino-lb/src/scaling/mod.rs index c4e6475..f6ee44e 100644 --- a/trino-lb/src/scaling/mod.rs +++ b/trino-lb/src/scaling/mod.rs @@ -267,7 +267,7 @@ impl Scaler { while let Some(res) = join_set.join_next().await { res.context(JoinReconcileClusterGroupTaskSnafu)??; } - info!("All cluster groups reconciled"); + info!("All cluster groups successfully reconciled"); Ok(()) } @@ -436,7 +436,7 @@ impl Scaler { while let Some(res) = join_set.join_next().await { res.context(JoinApplyClusterTargetStateTaskSnafu)??; } - info!("Applied all target cluster states"); + debug!("Applied all target cluster states"); Ok(()) } diff --git a/trino-lb/src/trino_client/cluster_info.rs b/trino-lb/src/trino_client/cluster_info.rs index 6710465..81ded41 100644 --- a/trino-lb/src/trino_client/cluster_info.rs +++ b/trino-lb/src/trino_client/cluster_info.rs @@ -52,7 +52,7 @@ pub struct ClusterInfo { pub total_cpu_time_secs: u64, } -#[instrument(skip(credentials))] +#[instrument(skip(endpoint, ignore_certs, credentials))] pub async fn get_cluster_info( endpoint: &Url, ignore_certs: bool,