From 9dad0e2ae67f5b863563a87e66c4e965525315d0 Mon Sep 17 00:00:00 2001 From: Kamil Kisiela Date: Thu, 27 Feb 2025 17:20:34 +0100 Subject: [PATCH 001/123] wip --- docker/configs/otel-collector/Dockerfile | 31 + .../otel-collector/builder-config.yaml | 26 + docker/configs/otel-collector/config.yaml | 65 + docker/docker-compose.dev.yml | 17 + docker/docker.hcl | 30 + docker/otel-collector.dockerfile | 27 + packages/services/server/src/index.ts | 3 + .../services/server/src/otel-auth-endpoint.ts | 28 + packages/web/app/package.json | 4 + packages/web/app/src/components/ui/chart.tsx | 328 +++ .../web/app/src/components/ui/resizable.tsx | 39 + packages/web/app/src/components/ui/sheet.tsx | 8 +- .../web/app/src/components/ui/sidebar.tsx | 732 ++++++ packages/web/app/src/index.css | 30 + .../src/pages/target-insights-new-filter.tsx | 513 ++++ .../src/pages/target-insights-new-trace.tsx | 649 +++++ .../src/pages/target-insights-new-width.tsx | 58 + .../web/app/src/pages/target-insights-new.tsx | 2181 +++++++++++++++++ packages/web/app/src/router.tsx | 34 + packages/web/app/tailwind.config.ts | 10 + pnpm-lock.yaml | 237 +- tracing/notes.md | 14 + tracing/signoz.md | 39 + tracing/uptrace.md | 17 + 24 files changed, 5084 insertions(+), 36 deletions(-) create mode 100644 docker/configs/otel-collector/Dockerfile create mode 100644 docker/configs/otel-collector/builder-config.yaml create mode 100644 docker/configs/otel-collector/config.yaml create mode 100644 docker/otel-collector.dockerfile create mode 100644 packages/services/server/src/otel-auth-endpoint.ts create mode 100644 packages/web/app/src/components/ui/chart.tsx create mode 100644 packages/web/app/src/components/ui/resizable.tsx create mode 100644 packages/web/app/src/components/ui/sidebar.tsx create mode 100644 packages/web/app/src/pages/target-insights-new-filter.tsx create mode 100644 packages/web/app/src/pages/target-insights-new-trace.tsx create mode 100644 packages/web/app/src/pages/target-insights-new-width.tsx create mode 100644 packages/web/app/src/pages/target-insights-new.tsx create mode 100644 tracing/notes.md create mode 100644 tracing/signoz.md create mode 100644 tracing/uptrace.md diff --git a/docker/configs/otel-collector/Dockerfile b/docker/configs/otel-collector/Dockerfile new file mode 100644 index 0000000000..5df59e115e --- /dev/null +++ b/docker/configs/otel-collector/Dockerfile @@ -0,0 +1,31 @@ +FROM scratch AS config + +COPY builder-config.yaml . + +FROM golang:1.23.7-bookworm AS builder + +ARG OTEL_VERSION=0.122.0 + +WORKDIR /build + +RUN go install go.opentelemetry.io/collector/cmd/builder@v${OTEL_VERSION} + +# Copy the manifest file and other necessary files +COPY --from=config builder-config.yaml . + +# Build the custom collector +RUN CGO_ENABLED=0 builder --config=/build/builder-config.yaml + +# Stage 2: Final Image +FROM alpine:3.14 + +WORKDIR /app + +# Copy the generated collector binary from the builder stage +COPY --from=builder /build/otelcol-custom . + +# Expose necessary ports +EXPOSE 4317/tcp 4318/tcp 13133/tcp + +# Set the default command +CMD ["./otelcol-custom", "--config=/etc/otel-config.yaml"] diff --git a/docker/configs/otel-collector/builder-config.yaml b/docker/configs/otel-collector/builder-config.yaml new file mode 100644 index 0000000000..8242a77fee --- /dev/null +++ b/docker/configs/otel-collector/builder-config.yaml @@ -0,0 +1,26 @@ +dist: + version: 0.122.0 + name: otelcol-custom + description: Custom OTel Collector distribution + output_path: ./otelcol-custom + +receivers: + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.122.0 + +processors: + - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.122.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor + v0.122.0 + +exporters: + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.122.0 + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter v0.122.0 + +extensions: + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension + v0.122.0 + - gomod: + github.com/graphql-hive/opentelemetry-collector-contrib/extension/hiveauthextension cd0c57cf22 diff --git a/docker/configs/otel-collector/config.yaml b/docker/configs/otel-collector/config.yaml new file mode 100644 index 0000000000..3804d72036 --- /dev/null +++ b/docker/configs/otel-collector/config.yaml @@ -0,0 +1,65 @@ +extensions: + hiveauth: + endpoint: ${HIVE_OTEL_AUTH_ENDPOINT} +receivers: + otlp: + protocols: + grpc: + include_metadata: true + endpoint: '0.0.0.0:4317' + auth: + authenticator: hiveauth + http: + cors: + allowed_origins: ['*'] + allowed_headers: ['*'] + include_metadata: true + endpoint: '0.0.0.0:4318' + auth: + authenticator: hiveauth +processors: + batch: + timeout: 5s + send_batch_size: 100000 + attributes: + actions: + - key: hive.target_id + from_context: auth.targetId + action: insert +exporters: + # debug: + # verbosity: detailed + # sampling_initial: 5 + # sampling_thereafter: 200 + clickhouse: + endpoint: tcp://clickhouse:9000?dial_timeout=10s&compress=lz4 + database: default + async_insert: true + username: test + password: test + create_schema: true + ttl: 720h + compress: lz4 + logs_table_name: otel_logs + traces_table_name: otel_traces + metrics_table_name: otel_metrics + timeout: 5s + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s +service: + extensions: + - hiveauth + telemetry: + logs: + level: INFO + encoding: json + output_paths: ['stdout'] + error_output_paths: ['stderr'] + pipelines: + traces: + receivers: [otlp] + processors: [attributes, batch] + exporters: [clickhouse] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index a76deecc29..fbacfcc4ff 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -16,6 +16,7 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_DB: registry PGDATA: /var/lib/postgresql/data + HIVE_OTEL_AUTH_ENDPOINT: 'http://host.docker.internal:3001/otel-auth' volumes: - ./.hive-dev/postgresql/db:/var/lib/postgresql/data ports: @@ -176,5 +177,21 @@ services: networks: - 'stack' + otel-collector: + depends_on: + clickhouse: + condition: service_healthy + build: + context: ./configs/otel-collector + dockerfile: Dockerfile + volumes: + - ./configs/otel-collector/builder-config.yaml:/builder-config.yaml + - ./configs/otel-collector/config.yaml:/etc/otel-config.yaml + ports: + - '4317:4317' + - '4318:4318' + networks: + - 'stack' + networks: stack: {} diff --git a/docker/docker.hcl b/docker/docker.hcl index 0910aadbdc..4e9e421965 100644 --- a/docker/docker.hcl +++ b/docker/docker.hcl @@ -82,6 +82,13 @@ target "router-base" { } } +target "otel-collector-base" { + dockerfile = "${PWD}/docker/otel-collector.dockerfile" + args = { + RELEASE = "${RELEASE}" + } +} + target "cli-base" { dockerfile = "${PWD}/docker/cli.dockerfile" args = { @@ -370,6 +377,23 @@ target "apollo-router" { ] } +target "otel-collector" { + inherits = ["otel-collector-base", get_target()] + contexts = { + config = "${PWD}/docker/configs/otel-collector" + } + args = { + IMAGE_TITLE = "graphql-hive/otel-collector" + IMAGE_DESCRIPTION = "OTEL Collector for GraphQL Hive." + } + tags = [ + local_image_tag("otel-collector"), + stable_image_tag("otel-collector"), + image_tag("otel-collector", COMMIT_SHA), + image_tag("otel-collector", BRANCH_NAME) + ] +} + target "cli" { inherits = ["cli-base", get_target()] context = "${PWD}/packages/libraries/cli" @@ -426,6 +450,12 @@ group "apollo-router-hive-build" { ] } +group "otel-collector-hive-build" { + targets = [ + "otel-collector" + ] +} + group "cli" { targets = [ "cli" diff --git a/docker/otel-collector.dockerfile b/docker/otel-collector.dockerfile new file mode 100644 index 0000000000..6bbe6b48bd --- /dev/null +++ b/docker/otel-collector.dockerfile @@ -0,0 +1,27 @@ +FROM golang:1.23.7-bookworm AS builder + +ARG OTEL_VERSION=0.122.0 + +WORKDIR /build + +RUN go install go.opentelemetry.io/collector/cmd/builder@v${OTEL_VERSION} + +# Copy the manifest file and other necessary files +COPY --from=config builder-config.yaml . + +# Build the custom collector +RUN builder --config=/build/builder-config.yaml + +# Stage 2: Final Image +FROM alpine:3.14 + +WORKDIR /app + +# Copy the generated collector binary from the builder stage +COPY --from=builder /build/otelcol-custom . + +# Expose necessary ports +EXPOSE 4317/tcp 4318/tcp 13133/tcp + +# Set the default command +CMD ["./otelcol-custom", "--config=/etc/otel-config.yaml"] diff --git a/packages/services/server/src/index.ts b/packages/services/server/src/index.ts index 51716090ab..f9efcdca60 100644 --- a/packages/services/server/src/index.ts +++ b/packages/services/server/src/index.ts @@ -66,6 +66,7 @@ import { env } from './environment'; import { graphqlHandler } from './graphql-handler'; import { clickHouseElapsedDuration, clickHouseReadDuration } from './metrics'; import { createPublicGraphQLHandler } from './public-graphql-handler'; +import { createOtelAuthEndpoint } from './otel-auth-endpoint'; import { initSupertokens, oidcIdLookup } from './supertokens'; export async function main() { @@ -592,6 +593,8 @@ export async function main() { return; }); + createOtelAuthEndpoint(server); + if (env.cdn.providers.api !== null) { const s3 = { client: new AwsClient({ diff --git a/packages/services/server/src/otel-auth-endpoint.ts b/packages/services/server/src/otel-auth-endpoint.ts new file mode 100644 index 0000000000..ed477ade11 --- /dev/null +++ b/packages/services/server/src/otel-auth-endpoint.ts @@ -0,0 +1,28 @@ +import type { FastifyInstance } from 'fastify'; + +export function createOtelAuthEndpoint(server: FastifyInstance) { + server.get('/otel-auth', (req, res) => { + const authHeader = req.headers.authorization; + const targetRefHeader = req.headers['x-hive-target-ref']; + req.log.info('request! ' + authHeader); + + if (authHeader === 'Bearer 123') { + if (targetRefHeader !== 'target-1') { + // No access to target-1 + res.status(403).send({ + message: 'Forbidden access to the target', + }); + return; + } + + res.status(200).send({ + message: 'Authenticated', + targetId: targetRefHeader, + }); + } else { + res.status(401).send({ + message: 'Unauthorized', + }); + } + }); +} diff --git a/packages/web/app/package.json b/packages/web/app/package.json index 94985cb798..2152884d72 100644 --- a/packages/web/app/package.json +++ b/packages/web/app/package.json @@ -81,12 +81,14 @@ "@urql/exchange-auth": "2.2.0", "@urql/exchange-graphcache": "7.1.0", "@vitejs/plugin-react": "4.3.4", + "@xyflow/react": "12.4.4", "autoprefixer": "10.4.21", "class-variance-authority": "0.7.1", "clsx": "2.1.1", "cmdk": "0.2.1", "crypto-js": "^4.2.0", "date-fns": "4.1.0", + "date-fns-tz": "3.2.0", "dompurify": "3.2.6", "dotenv": "16.4.7", "echarts": "5.6.0", @@ -112,6 +114,7 @@ "react-highlight-words": "0.20.0", "react-hook-form": "7.54.2", "react-icons": "5.4.0", + "react-resizable-panels": "2.1.7", "react-select": "5.9.0", "react-string-replace": "1.1.1", "react-textarea-autosize": "8.5.9", @@ -119,6 +122,7 @@ "react-virtualized-auto-sizer": "1.0.25", "react-virtuoso": "4.12.3", "react-window": "1.8.11", + "recharts": "2.15.1", "regenerator-runtime": "0.14.1", "snarkdown": "2.0.0", "storybook": "8.4.7", diff --git a/packages/web/app/src/components/ui/chart.tsx b/packages/web/app/src/components/ui/chart.tsx new file mode 100644 index 0000000000..e11d102365 --- /dev/null +++ b/packages/web/app/src/components/ui/chart.tsx @@ -0,0 +1,328 @@ +'use client'; + +import * as React from 'react'; +import * as RechartsPrimitive from 'recharts'; +import { cn } from '@/lib/utils'; + +// Format: { THEME_NAME: CSS_SELECTOR } +const THEMES = { light: '', dark: '.dark' } as const; + +export type ChartConfig = { + [k in string]: { + label?: React.ReactNode; + icon?: React.ComponentType; + } & ( + | { color?: string; theme?: never } + | { color?: never; theme: Record } + ); +}; + +type ChartContextProps = { + config: ChartConfig; +}; + +const ChartContext = React.createContext(null); + +function useChart() { + const context = React.useContext(ChartContext); + + if (!context) { + throw new Error('useChart must be used within a '); + } + + return context; +} + +const ChartContainer = React.forwardRef< + HTMLDivElement, + React.ComponentProps<'div'> & { + config: ChartConfig; + children: React.ComponentProps['children']; + } +>(({ id, className, children, config, ...props }, ref) => { + const uniqueId = React.useId(); + const chartId = `chart-${id || uniqueId.replace(/:/g, '')}`; + + return ( + +
+ + {children} +
+
+ ); +}); +ChartContainer.displayName = 'Chart'; + +const ChartStyle = ({ id, config }: { id: string; config: ChartConfig }) => { + const colorConfig = Object.entries(config).filter(([, config]) => config.theme || config.color); + + if (!colorConfig.length) { + return null; + } + + return ( +