feat: bundle api for image size reduction (#937)

all-in-one image: 1.51 gb -> 1.21 gb
app image: 573 mb -> 271 mb

Ref HDX-1803

Author: knudtty

.changeset/gentle-cheetahs-carry.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+feat: bundle api via esbuild for smaller image distribution

docker/hyperdx/Dockerfile

@@ -13,8 +13,6 @@ ARG OTEL_COLLECTOR_VERSION=0.126.0
 ARG OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION=0.126.0
 
 # base #############################################################################################
-# == Clickhouse/Base Image ==
-FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS clickhouse_base
 # == Otel Collector Image ==
 FROM otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION} AS otel_collector_base
 FROM otel/opentelemetry-collector-opampsupervisor:${OTEL_COLLECTOR_OPAMPSUPERVISOR_VERSION} AS otel_collector_opampsupervisor_base
@@ -51,8 +49,6 @@ ENV NEXT_TELEMETRY_DISABLED 1
 ENV NEXT_OUTPUT_STANDALONE true
 ENV NEXT_PUBLIC_IS_LOCAL_MODE false
 RUN npx nx run-many --target=build --projects=@hyperdx/common-utils,@hyperdx/api,@hyperdx/app
-RUN rm -rf node_modules && yarn workspaces focus @hyperdx/api --production
-
 
 # prod ############################################################################################
 FROM node:${NODE_VERSION}-alpine AS prod
@@ -69,10 +65,7 @@ USER node
 
 # Set up API and App
 WORKDIR /app
-COPY --chown=node:node --from=builder /app/node_modules ./node_modules
-COPY --chown=node:node --from=builder /app/packages/api/build ./packages/api/build
-COPY --chown=node:node --from=builder /app/packages/common-utils/dist ./packages/common-utils/dist
-COPY --chown=node:node --from=node_base /app/packages/common-utils/node_modules ./packages/common-utils/node_modules
+COPY --chown=node:node --from=builder /app/packages/api/dist ./packages/api
 COPY --chown=node:node --from=builder /app/packages/app/.next/standalone ./packages/app
 COPY --chown=node:node --from=builder /app/packages/app/.next/static ./packages/app/packages/app/.next/static
 COPY --chown=node:node --from=builder /app/packages/app/public ./packages/app/packages/app/public
@@ -82,13 +75,12 @@ COPY --chown=node:node --from=hyperdx ./entry.prod.sh /etc/local/entry.sh
 ENTRYPOINT ["sh", "/etc/local/entry.sh"]
 
 # all-in-one base ############################################################################################
-FROM scratch AS all-in-one-base
+FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION}-alpine AS all-in-one-base 
 
 ARG CODE_VERSION
 
 ENV CODE_VERSION=$CODE_VERSION
-# Copy from clickhouse and otel collector bases
-COPY --from=clickhouse_base / /
+# Copy from otel collector bases
 COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol
 COPY --from=otel_collector_opampsupervisor_base --chmod=755 /usr/local/bin/opampsupervisor /usr/local/bin/opampsupervisor
 
@@ -98,6 +90,9 @@ COPY --from=node_base --link /usr/local/lib /usr/local/lib
 COPY --from=node_base /usr/lib /usr/lib
 COPY --from=node_base /usr/local/include /usr/local/include
 
+# Install libs used for the start script
+RUN npm install -g [email protected]
+
 # Set up Clickhouse
 COPY --from=clickhouse ./local/*.xml /etc/clickhouse-server
 COPY --from=hyperdx ./clickhouseConfig.xml /etc/clickhouse-server/config.xml

docker/hyperdx/entry.local.base.sh

@@ -61,9 +61,9 @@ opampsupervisor --config /etc/otel/supervisor.yaml > /var/log/otel-collector.log
 npx concurrently \
   "--kill-others" \
   "--names=API,APP,ALERT-TASK" \
-  "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r ./node_modules/@hyperdx/node-opentelemetry/build/src/tracing ./packages/api/build/index.js" \
+  "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r ./packages/api/tracing ./packages/api/index" \
   "cd ./packages/app/packages/app && HOSTNAME='0.0.0.0' HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} PORT=${HYPERDX_APP_PORT:-8080} node server.js" \
-  "node -r ./node_modules/@hyperdx/node-opentelemetry/build/src/tracing ./packages/api/build/tasks/index.js check-alerts" \
+  "node -r ./packages/api/tracing ./packages/api/tasks/index check-alerts"
   > /var/log/app.log 2>&1 &
 
 echo ""

docker/hyperdx/entry.prod.sh

@@ -15,6 +15,6 @@ echo ""
 npx concurrently \
   "--kill-others" \
   "--names=API,APP,ALERT-TASK" \
-  "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r ./node_modules/@hyperdx/node-opentelemetry/build/src/tracing ./packages/api/build/index.js" \
+  "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} node -r ./packages/api/tracing ./packages/api/index" \
   "cd ./packages/app/packages/app && HOSTNAME='0.0.0.0' HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} PORT=${HYPERDX_APP_PORT:-8080} node server.js" \
-  "node -r ./node_modules/@hyperdx/node-opentelemetry/build/src/tracing ./packages/api/build/tasks/index.js check-alerts"
+  "node -r ./packages/api/tracing ./packages/api/tasks/index check-alerts"

packages/api/package.json

@@ -75,6 +75,7 @@
     "@types/supertest": "^2.0.12",
     "@types/swagger-jsdoc": "^6",
     "@types/uuid": "^8.3.4",
+    "esbuild": "^0.25.5",
     "jest": "^28.1.3",
     "migrate-mongo": "^11.0.0",
     "nodemon": "^2.0.20",
@@ -89,10 +90,10 @@
     "typescript": "^4.9.5"
   },
   "scripts": {
-    "start": "node ./build/index.js",
+    "start": "node ./dist/index.js",
     "dev": "DOTENV_CONFIG_PATH=.env.development nodemon --signal SIGTERM -e ts,json --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r dotenv-expand/config -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/index.ts",
     "dev-task": "DOTENV_CONFIG_PATH=.env.development nodemon --signal SIGTERM -e ts,json --exec 'ts-node' --transpile-only -r tsconfig-paths/register -r dotenv-expand/config -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/tasks/index.ts",
-    "build": "rimraf ./build && tsc && tsc-alias && cp -r src/opamp/proto build/opamp/",
+    "build": "tsc && tsc-alias && esbuild ../../node_modules/@hyperdx/node-opentelemetry/build/src/tracing --bundle --minify --platform=node --outfile=dist/tracing.js && esbuild ./build/src/index.js ./build/src/tasks/index.js --bundle --minify --platform=node --target=node22 --outdir=dist --alias:@='./src' && mkdir -p dist/opamp && cp -r src/opamp/proto dist/opamp/proto && rimraf ./build",
     "lint": "eslint --quiet . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
     "ci:lint": "yarn lint && yarn tsc --noEmit",

packages/api/src/opamp/utils/protobuf.ts

@@ -3,10 +3,14 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as protobuf from 'protobufjs';
 
+import { IS_PROD } from '@/config';
 import logger from '@/utils/logger';
 
 // Define the root path of the proto file
-const PROTO_PATH = path.resolve(__dirname, '../proto/opamp.proto');
+const PROTO_PATH = path.resolve(
+  __dirname,
+  `${IS_PROD ? 'opamp' : '..'}/proto/opamp.proto`,
+);
 
 // Load the OpAMP proto definition
 let root: protobuf.Root;

packages/api/tsconfig.json

@@ -4,6 +4,7 @@
     "paths": {
       "@/*": ["./*"]
     },
+    "rootDir": ".",
     "allowSyntheticDefaultImports": true,
     "downlevelIteration": true,
     "esModuleInterop": true,