From 51fef18df2fc5ad27e30fc34d22b615d3cdbdb2c Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 8 Feb 2025 13:37:59 +0000 Subject: [PATCH 1/6] add env var for additional pull secrets --- apps/kubernetes-provider/src/index.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index a5a6cf55f8..2602e1f83d 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -38,6 +38,7 @@ const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_ const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi"; const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true"; +const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS; const logger = new SimpleLogger(`[${NODE_NAME}]`); logger.log(`running in ${RUNTIME_ENV} mode`); @@ -403,17 +404,22 @@ class KubernetesTaskOperations implements TaskOperations { } get #defaultPodSpec(): Omit { + const pullSecrets = ["registry-trigger", "registry-trigger-failover"]; + + if (ADDITIONAL_PULL_SECRETS) { + pullSecrets.push(...ADDITIONAL_PULL_SECRETS.split(",")); + } + + const imagePullSecrets = pullSecrets.map( + (name) => ({ name }) satisfies k8s.V1LocalObjectReference + ); + + console.log("imagePullSecrets", imagePullSecrets); + return { restartPolicy: "Never", automountServiceAccountToken: false, - imagePullSecrets: [ - { - name: "registry-trigger", - }, - { - name: "registry-trigger-failover", - }, - ], + imagePullSecrets, nodeSelector: { nodetype: "worker", }, From 86684c25270841884a97df47ceb2444eb2e7d191 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 8 Feb 2025 13:55:22 +0000 Subject: [PATCH 2/6] make static images configurable --- apps/kubernetes-provider/src/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index 2602e1f83d..a51b86e359 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -37,8 +37,11 @@ const UPTIME_MAX_PENDING_ERRORS = Number(process.env.UPTIME_MAX_PENDING_ERRORS | const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi"; const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi"; +// Image config const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true"; const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS; +const PAUSE_IMAGE = process.env.PAUSE_IMAGE || "registry.k8s.io/pause:3.9"; +const BUSYBOX_IMAGE = process.env.BUSYBOX_IMAGE || "registry.digitalocean.com/trigger/busybox"; const logger = new SimpleLogger(`[${NODE_NAME}]`); logger.log(`running in ${RUNTIME_ENV} mode`); @@ -237,7 +240,7 @@ class KubernetesTaskOperations implements TaskOperations { }, { name: "populate-taskinfo", - image: "registry.digitalocean.com/trigger/busybox", + image: BUSYBOX_IMAGE, imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-c"], args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"], @@ -373,7 +376,7 @@ class KubernetesTaskOperations implements TaskOperations { containers: [ { name: "pause", - image: "registry.k8s.io/pause:3.9", + image: PAUSE_IMAGE, resources: { limits: { cpu: "1m", From ec2e40d1d4a9a0bf3762a8bb1101f493a36ae98a Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:02:22 +0000 Subject: [PATCH 3/6] optional image prefixes --- apps/kubernetes-provider/src/index.ts | 38 ++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index a51b86e359..e8478e2ff7 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -17,6 +17,7 @@ import { import { PodCleaner } from "./podCleaner"; import { TaskMonitor } from "./taskMonitor"; import { UptimeHeartbeat } from "./uptimeHeartbeat"; +import { assertExhaustive } from "@trigger.dev/core"; const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; const NODE_NAME = process.env.NODE_NAME || "local"; @@ -42,6 +43,9 @@ const PRE_PULL_DISABLED = process.env.PRE_PULL_DISABLED === "true"; const ADDITIONAL_PULL_SECRETS = process.env.ADDITIONAL_PULL_SECRETS; const PAUSE_IMAGE = process.env.PAUSE_IMAGE || "registry.k8s.io/pause:3.9"; const BUSYBOX_IMAGE = process.env.BUSYBOX_IMAGE || "registry.digitalocean.com/trigger/busybox"; +const DEPLOYMENT_IMAGE_PREFIX = process.env.DEPLOYMENT_IMAGE_PREFIX; +const RESTORE_IMAGE_PREFIX = process.env.RESTORE_IMAGE_PREFIX; +const UTILITY_IMAGE_PREFIX = process.env.UTILITY_IMAGE_PREFIX; const logger = new SimpleLogger(`[${NODE_NAME}]`); logger.log(`running in ${RUNTIME_ENV} mode`); @@ -107,7 +111,7 @@ class KubernetesTaskOperations implements TaskOperations { containers: [ { name: this.#getIndexContainerName(opts.shortCode), - image: opts.imageRef, + image: getImageRef("deployment", opts.imageRef), ports: [ { containerPort: 8000, @@ -174,7 +178,7 @@ class KubernetesTaskOperations implements TaskOperations { containers: [ { name: containerName, - image: opts.image, + image: getImageRef("deployment", opts.image), ports: [ { containerPort: 8000, @@ -235,12 +239,12 @@ class KubernetesTaskOperations implements TaskOperations { initContainers: [ { name: "pull-base-image", - image: opts.imageRef, + image: getImageRef("deployment", opts.imageRef), command: ["sleep", "0"], }, { name: "populate-taskinfo", - image: BUSYBOX_IMAGE, + image: getImageRef("utility", BUSYBOX_IMAGE), imagePullPolicy: "IfNotPresent", command: ["/bin/sh", "-c"], args: ["printenv COORDINATOR_HOST | tee /etc/taskinfo/coordinator-host"], @@ -256,7 +260,7 @@ class KubernetesTaskOperations implements TaskOperations { containers: [ { name: this.#getRunContainerName(opts.runId), - image: opts.checkpointRef, + image: getImageRef("restore", opts.checkpointRef), ports: [ { containerPort: 8000, @@ -362,7 +366,7 @@ class KubernetesTaskOperations implements TaskOperations { initContainers: [ { name: "prepull", - image: opts.imageRef, + image: getImageRef("deployment", opts.imageRef), command: ["/usr/bin/true"], resources: { limits: { @@ -376,7 +380,7 @@ class KubernetesTaskOperations implements TaskOperations { containers: [ { name: "pause", - image: PAUSE_IMAGE, + image: getImageRef("utility", PAUSE_IMAGE), resources: { limits: { cpu: "1m", @@ -682,6 +686,26 @@ class KubernetesTaskOperations implements TaskOperations { } } +type ImageType = "deployment" | "restore" | "utility"; + +function getImagePrefix(type: ImageType) { + switch (type) { + case "deployment": + return DEPLOYMENT_IMAGE_PREFIX; + case "restore": + return RESTORE_IMAGE_PREFIX; + case "utility": + return UTILITY_IMAGE_PREFIX; + default: + assertExhaustive(type); + } +} + +function getImageRef(type: ImageType, ref: string) { + const prefix = getImagePrefix(type); + return prefix ? `${prefix}/${ref}` : ref; +} + const provider = new ProviderShell({ tasks: new KubernetesTaskOperations({ namespace: KUBERNETES_NAMESPACE, From 2f2e0e2896d1eba3f99608229200d4d1c780dd8d Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 10 Feb 2025 13:29:47 +0000 Subject: [PATCH 4/6] optional labels with sample rates --- apps/kubernetes-provider/src/index.ts | 5 + apps/kubernetes-provider/src/labelHelper.ts | 160 ++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 apps/kubernetes-provider/src/labelHelper.ts diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index e8478e2ff7..4ed007ca4c 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -18,6 +18,7 @@ import { PodCleaner } from "./podCleaner"; import { TaskMonitor } from "./taskMonitor"; import { UptimeHeartbeat } from "./uptimeHeartbeat"; import { assertExhaustive } from "@trigger.dev/core"; +import { CustomLabelHelper } from "./labelHelper"; const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local"; const NODE_NAME = process.env.NODE_NAME || "local"; @@ -73,6 +74,8 @@ class KubernetesTaskOperations implements TaskOperations { apps: k8s.AppsV1Api; }; + #labelHelper = new CustomLabelHelper(); + constructor(opts: { namespace?: string } = {}) { if (opts.namespace) { this.#namespace.metadata.name = opts.namespace; @@ -165,6 +168,7 @@ class KubernetesTaskOperations implements TaskOperations { name: containerName, namespace: this.#namespace.metadata.name, labels: { + ...this.#labelHelper.getAdditionalLabels("create"), ...this.#getSharedLabels(opts), app: "task-run", "app.kubernetes.io/part-of": "trigger-worker", @@ -226,6 +230,7 @@ class KubernetesTaskOperations implements TaskOperations { name: `${this.#getRunContainerName(opts.runId)}-${opts.checkpointId.slice(-8)}`, namespace: this.#namespace.metadata.name, labels: { + ...this.#labelHelper.getAdditionalLabels("restore"), ...this.#getSharedLabels(opts), app: "task-run", "app.kubernetes.io/part-of": "trigger-worker", diff --git a/apps/kubernetes-provider/src/labelHelper.ts b/apps/kubernetes-provider/src/labelHelper.ts new file mode 100644 index 0000000000..6ad6bf0b56 --- /dev/null +++ b/apps/kubernetes-provider/src/labelHelper.ts @@ -0,0 +1,160 @@ +import { assertExhaustive } from "@trigger.dev/core"; + +const CREATE_LABEL_ENV_VAR_PREFIX = "DEPLOYMENT_LABEL_"; +const RESTORE_LABEL_ENV_VAR_PREFIX = "RESTORE_LABEL_"; +const LABEL_SAMPLE_RATE_POSTFIX = "_SAMPLE_RATE"; + +type OperationType = "create" | "restore"; + +type CustomLabel = { + key: string; + value: string; + sampleRate: number; +}; + +export class CustomLabelHelper { + // Labels and sample rates are defined in environment variables so only need to be computed once + private createLabels?: CustomLabel[]; + private restoreLabels?: CustomLabel[]; + + private getLabelPrefix(type: OperationType) { + const prefix = type === "create" ? CREATE_LABEL_ENV_VAR_PREFIX : RESTORE_LABEL_ENV_VAR_PREFIX; + return prefix.toLowerCase(); + } + + private getLabelSampleRatePostfix() { + return LABEL_SAMPLE_RATE_POSTFIX.toLowerCase(); + } + + // Can only range from 0 to 1 + private fractionFromPercent(percent: number) { + return Math.min(1, Math.max(0, percent / 100)); + } + + private isLabelSampleRateEnvVar(key: string) { + return key.toLowerCase().endsWith(this.getLabelSampleRatePostfix()); + } + + private isLabelEnvVar(type: OperationType, key: string) { + const prefix = this.getLabelPrefix(type); + return key.toLowerCase().startsWith(prefix) && !this.isLabelSampleRateEnvVar(key); + } + + private getSampleRateEnvVarKey(type: OperationType, envKey: string) { + return `${envKey.toLowerCase()}${this.getLabelSampleRatePostfix()}`; + } + + private getLabelNameFromEnvVarKey(type: OperationType, key: string) { + return key + .slice(this.getLabelPrefix(type).length) + .toLowerCase() + .replace(/___/g, ".") + .replace(/__/g, "/") + .replace(/_/g, "-"); + } + + private getCaseInsensitiveEnvValue(key: string) { + for (const [envKey, value] of Object.entries(process.env)) { + if (envKey.toLowerCase() === key.toLowerCase()) { + return value; + } + } + } + + /** Returns the sample rate for a given label as fraction of 100 */ + private getSampleRateFromEnvVarKey(type: OperationType, envKey: string) { + // Apply default: always sample + const DEFAULT_SAMPLE_RATE_PERCENT = 100; + const defaultSampleRateFraction = this.fractionFromPercent(DEFAULT_SAMPLE_RATE_PERCENT); + + const value = this.getCaseInsensitiveEnvValue(this.getSampleRateEnvVarKey(type, envKey)); + + if (!value) { + return defaultSampleRateFraction; + } + + const sampleRatePercent = parseFloat(value || String(DEFAULT_SAMPLE_RATE_PERCENT)); + + if (isNaN(sampleRatePercent)) { + return defaultSampleRateFraction; + } + + const fractionalSampleRate = this.fractionFromPercent(sampleRatePercent); + + return fractionalSampleRate; + } + + private getCustomLabels(type: OperationType): CustomLabel[] { + switch (type) { + case "create": + if (this.createLabels) { + return this.createLabels; + } + break; + case "restore": + if (this.restoreLabels) { + return this.restoreLabels; + } + break; + default: + assertExhaustive(type); + } + + const customLabels: CustomLabel[] = []; + + for (const [envKey, value] of Object.entries(process.env)) { + const key = envKey.toLowerCase(); + + // Only process env vars that start with the expected prefix + if (!this.isLabelEnvVar(type, key)) { + continue; + } + + // Skip sample rates - deal with them separately + if (this.isLabelSampleRateEnvVar(key)) { + continue; + } + + const labelName = this.getLabelNameFromEnvVarKey(type, key); + const sampleRate = this.getSampleRateFromEnvVarKey(type, key); + + console.log({ key, value, sampleRate, labelName }); + + const label = { + key: labelName, + value: value || "", + sampleRate, + } satisfies CustomLabel; + + customLabels.push(label); + } + + console.log({ + type, + customLabels, + }); + + return customLabels; + } + + getAdditionalLabels(type: OperationType): Record { + const labels = this.getCustomLabels(type); + + const additionalLabels: Record = {}; + + for (const { key, value, sampleRate } of labels) { + // Always apply label if sample rate is 1 + if (sampleRate === 1) { + additionalLabels[key] = value; + continue; + } + + if (Math.random() <= sampleRate) { + additionalLabels[key] = value; + continue; + } + } + + return additionalLabels; + } +} From 1dfecbf64a5472dbd0a6044ce9a5c3591a82cb6b Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 10 Feb 2025 13:40:00 +0000 Subject: [PATCH 5/6] add missing core paths --- apps/kubernetes-provider/tsconfig.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/kubernetes-provider/tsconfig.json b/apps/kubernetes-provider/tsconfig.json index 661823ef74..057952409b 100644 --- a/apps/kubernetes-provider/tsconfig.json +++ b/apps/kubernetes-provider/tsconfig.json @@ -8,6 +8,8 @@ "strict": true, "skipLibCheck": true, "paths": { + "@trigger.dev/core": ["../../packages/core/src"], + "@trigger.dev/core/*": ["../../packages/core/src/*"], "@trigger.dev/core/v3": ["../../packages/core/src/v3"], "@trigger.dev/core/v3/*": ["../../packages/core/src/v3/*"] } From 96fbeeb3c96e3a8b916dadd4cf810ba35a17afc6 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 11 Feb 2025 11:53:46 +0000 Subject: [PATCH 6/6] remove excessive logs --- apps/kubernetes-provider/src/index.ts | 2 -- apps/kubernetes-provider/src/labelHelper.ts | 7 ------- 2 files changed, 9 deletions(-) diff --git a/apps/kubernetes-provider/src/index.ts b/apps/kubernetes-provider/src/index.ts index 4ed007ca4c..915b368d92 100644 --- a/apps/kubernetes-provider/src/index.ts +++ b/apps/kubernetes-provider/src/index.ts @@ -426,8 +426,6 @@ class KubernetesTaskOperations implements TaskOperations { (name) => ({ name }) satisfies k8s.V1LocalObjectReference ); - console.log("imagePullSecrets", imagePullSecrets); - return { restartPolicy: "Never", automountServiceAccountToken: false, diff --git a/apps/kubernetes-provider/src/labelHelper.ts b/apps/kubernetes-provider/src/labelHelper.ts index 6ad6bf0b56..98cd3d68be 100644 --- a/apps/kubernetes-provider/src/labelHelper.ts +++ b/apps/kubernetes-provider/src/labelHelper.ts @@ -118,8 +118,6 @@ export class CustomLabelHelper { const labelName = this.getLabelNameFromEnvVarKey(type, key); const sampleRate = this.getSampleRateFromEnvVarKey(type, key); - console.log({ key, value, sampleRate, labelName }); - const label = { key: labelName, value: value || "", @@ -129,11 +127,6 @@ export class CustomLabelHelper { customLabels.push(label); } - console.log({ - type, - customLabels, - }); - return customLabels; }