hotfix(067): Switch from @logtail/pino to Vector log collection — fixes heap growth crash

.github/workflows/porter-us-west.yml

@@ -0,0 +1,38 @@
+name: 🚀 Deploy cloud-prod to US West only
+
+on:
+  workflow_dispatch: # Manual trigger only — no auto-deploy
+  push:
+    branches:
+      - cloud/064-bstack-cli
+    paths:
+      - "cloud/**"
+      - ".github/workflows/porter-us-west.yml"
+jobs:
+  us-west:
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set Git SHA tag
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Setup Porter CLI
+        uses: porter-dev/setup-porter@v0.1.0
+
+      - name: Deploy to us-west
+        timeout-minutes: 30
+        run: exec porter apply -f ./cloud/porter.yaml
+        env:
+          PORTER_APP_NAME: cloud-prod
+          NODE_ENV: production
+          PORTER_CLUSTER: "4965"
+          PORTER_PROJECT: "15081"
+          PORTER_HOST: https://dashboard.porter.run
+          PORTER_PR_NUMBER: ${{ github.event.number }}
+          PORTER_REPO_NAME: ${{ github.event.repository.name }}
+          PORTER_TAG: ${{ steps.vars.outputs.sha_short }}
+          PORTER_TOKEN: ${{ secrets.PORTER_APP_15081_4965 }}
+          PORTER_DEPLOYMENT_TARGET_ID: 540690ee-b1d7-4a5e-80e9-683d11001c75

cloud/infra/betterstack-logs/values.yaml

@@ -0,0 +1,130 @@
+# BetterStack Logs Helm Chart — values.yaml
+#
+# Deploys Vector as a DaemonSet to collect container stdout/stderr logs
+# and ship them to BetterStack with Pino JSON flattened to top-level fields.
+#
+# This replaces the in-process @logtail/pino transport that causes
+# unbounded heap growth (see cloud/issues/067-heap-growth-investigation).
+#
+# The transforms below:
+#   1. Filter to only our cloud app containers (no kube-system noise)
+#   2. Parse the Pino JSON from the message field
+#   3. Flatten it to the top level so region=, feature=, service= work in Live Tail
+#   4. Add a log_source="vector" marker to distinguish from @logtail/pino during migration
+#
+# Usage per cluster:
+#   porter helm --cluster <CLUSTER_ID> -- install betterstack-logs \
+#     betterstack-logs/betterstack-logs \
+#     --namespace betterstack \
+#     --create-namespace \
+#     --values cloud/infra/betterstack-logs/values.yaml \
+#     --set "vector.customConfig.sinks.better_stack_http_sink.auth.token=$SOURCE_TOKEN" \
+#     --set "vector.customConfig.sinks.better_stack_http_metrics_sink.auth.token=$SOURCE_TOKEN" \
+#     --set "vector.customConfig.sinks.better_stack_http_sink.uri=https://$INGESTING_HOST/" \
+#     --set "vector.customConfig.sinks.better_stack_http_metrics_sink.uri=https://$INGESTING_HOST/metrics"
+#
+# Source tokens per region (stored in Doppler as BETTERSTACK_SOURCE_TOKEN):
+#   doppler secrets get BETTERSTACK_SOURCE_TOKEN --project mentraos-cloud --config <CONFIG> --plain
+#
+# The ingesting host for MentraCloud-Prod source:
+#   s2324289.eu-nbg-2.betterstackdata.com
+
+vector:
+  customConfig:
+    transforms:
+      # Step 1: Filter to only our cloud app containers.
+      # Drops all kube-system, cert-manager, ingress-nginx, porter-agent, etc.
+      cloud_only_filter:
+        type: "filter"
+        inputs: ["better_stack_kubernetes_parser"]
+        condition: >
+          contains(to_string!(.kubernetes.container_name), "cloud-prod-cloud") ||
+          contains(to_string!(.kubernetes.container_name), "cloud-staging-cloud") ||
+          contains(to_string!(.kubernetes.container_name), "cloud-debug-cloud") ||
+          contains(to_string!(.kubernetes.container_name), "cloud-dev-cloud")
+
+      # Step 2: Flatten the Pino JSON so all fields are top-level.
+      # Input from Vector: { kubernetes: {...}, message: { level: 30, region: "us-west", ... } }
+      # Output:            { level: 30, region: "us-west", ..., kubernetes_pod: "...", log_source: "vector" }
+      flatten_pino:
+        type: "remap"
+        inputs: ["cloud_only_filter"]
+        source: |
+          # Grab kubernetes context before we restructure
+          kube_pod = to_string(.kubernetes.pod_name) ?? "unknown"
+          kube_container = to_string(.kubernetes.container_name) ?? "unknown"
+
+          # The Pino JSON lives in .message
+          # It may already be parsed as an object, or it may be a raw JSON string
+          pino_data = .message
+
+          if is_string(pino_data) {
+            parsed, err = parse_json(string!(pino_data))
+            if err == null {
+              pino_data = parsed
+            }
+          }
+
+          # If we successfully got a Pino object, flatten it to the top level
+          # This makes region=, feature=, service=, heapUsedMB= all queryable in Live Tail
+          if is_object(pino_data) {
+            . = object!(pino_data)
+          }
+
+          # Normalize fields to match what @logtail/pino sends to BetterStack.
+          # Without this, existing queries and Live Tail display break.
+
+          # msg → message (Pino uses "msg", @logtail/pino renames to "message")
+          if exists(.msg) {
+            .message = del(.msg)
+          }
+
+          # time → dt (@logtail/pino sends "dt", Pino natively uses "time")
+          if exists(.time) {
+            .dt = del(.time)
+          }
+
+          # level: numeric → string (Pino outputs 10/20/30/40/50/60, BetterStack expects strings)
+          if is_integer(.level) {
+            numeric_level = to_int!(.level)
+            if numeric_level >= 60 {
+              .level = "fatal"
+            } else if numeric_level >= 50 {
+              .level = "error"
+            } else if numeric_level >= 40 {
+              .level = "warn"
+            } else if numeric_level >= 30 {
+              .level = "info"
+            } else if numeric_level >= 20 {
+              .level = "debug"
+            } else {
+              .level = "trace"
+            }
+          }
+
+          # Nest Vector metadata into _meta so it's out of the way of app fields.
+          # Everything from Pino stays flat at the root. The extra stuff Vector adds
+          # goes into _meta — there if you need it, not cluttering the log.
+          ._meta.kubernetes_pod = kube_pod
+          ._meta.kubernetes_container = kube_container
+          ._meta.log_source = "vector"
+
+    sinks:
+      # Override the default sink to use our flattened output instead of raw
+      better_stack_http_sink:
+        inputs: ["flatten_pino"]
+        # Override uri and auth.token via --set flags (do NOT commit tokens)
+        uri: "https://PLACEHOLDER_INGESTING_HOST/"
+        auth:
+          strategy: "bearer"
+          token: "PLACEHOLDER_SOURCE_TOKEN"
+      better_stack_http_metrics_sink:
+        uri: "https://PLACEHOLDER_INGESTING_HOST/metrics"
+        auth:
+          strategy: "bearer"
+          token: "PLACEHOLDER_SOURCE_TOKEN"
+
+# The existing better-stack-collector DaemonSet already installs a metrics
+# server on each cluster. Disable it here to avoid conflicts.
+metrics-server:
+  enabled: false