Merge pull request #13 from ifsp-projects/feat/add-filter

Feat: Implement open telemetry

Author: vitingr

.github/workflows/cd.yml

@@ -81,6 +81,16 @@ jobs:
           source: ".env"
           target: "/home/gcpuser/app"
 
+      - name: Copy OTel config to VM
+        uses: appleboy/scp-action@master
+        with:
+          host: ${{ env.VM_HOST }}
+          username: gcpuser
+          key: ${{ secrets.VM_SSH_KEY }}
+          source: "devops/otel/collector-config.prod.yaml"
+          target: "/opt/otel"
+          strip_components: 2
+
       - name: Deploy via SSH
         uses: appleboy/ssh-action@master
         env:
@@ -89,27 +99,43 @@ jobs:
           IMAGE_TAG: ${{ github.sha }}
           CONTAINER_NAME: api-ifsp
           REGION: ${{ env.REGION }}
+          GRAFANA_CLOUD_OTLP_ENDPOINT: ${{ secrets.GRAFANA_CLOUD_OTLP_ENDPOINT }}
+          GRAFANA_CLOUD_AUTH: ${{ secrets.GRAFANA_CLOUD_AUTH }}
         with:
           host: ${{ env.VM_HOST }}
           username: gcpuser
           key: ${{ secrets.VM_SSH_KEY }}
-          envs: REGISTRY_URL,IMAGE_REPOSITORY,IMAGE_TAG,CONTAINER_NAME,REGION
+          envs: REGISTRY_URL,IMAGE_REPOSITORY,IMAGE_TAG,CONTAINER_NAME,REGION,GRAFANA_CLOUD_OTLP_ENDPOINT,GRAFANA_CLOUD_AUTH
           script: |
             until sudo docker info >/dev/null 2>&1; do sleep 3; done
 
             TOKEN=$(curl -sf "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \
               -H "Metadata-Flavor: Google" | python3 -c "import sys,json; print(json.load(sys.stdin)['access_token'])")
-
             echo $TOKEN | sudo docker login -u oauth2accesstoken --password-stdin https://$REGION-docker.pkg.dev
 
-            sudo docker pull $REGISTRY_URL/$IMAGE_REPOSITORY:$IMAGE_TAG
+            sudo docker network create observability 2>/dev/null || true
 
-            sudo docker stop $CONTAINER_NAME 2>/dev/null || true
-            sudo docker rm $CONTAINER_NAME 2>/dev/null || true
+            sudo docker rm -f otel-collector 2>/dev/null || true
+            sudo docker run -d \
+              --name otel-collector \
+              --restart always \
+              --network observability \
+              --memory 128m \
+              --cpus 0.15 \
+              -p 127.0.0.1:4317:4317 \
+              -p 127.0.0.1:4318:4318 \
+              -v /opt/otel/collector-config.prod.yaml:/etc/otelcol-contrib/config.yaml:ro \ 
+              -e GRAFANA_CLOUD_OTLP_ENDPOINT="$GRAFANA_CLOUD_OTLP_ENDPOINT" \
+              -e GRAFANA_CLOUD_AUTH="$GRAFANA_CLOUD_AUTH" \
+              otel/opentelemetry-collector-contrib:0.96.0
 
+            sudo docker pull $REGISTRY_URL/$IMAGE_REPOSITORY:$IMAGE_TAG
+            sudo docker stop $CONTAINER_NAME 2>/dev/null || true
+            sudo docker rm   $CONTAINER_NAME 2>/dev/null || true
             sudo docker run -d \
               --name $CONTAINER_NAME \
               --restart unless-stopped \
+              --network observability \
               --env-file /home/gcpuser/app/.env \
               -p 80:8000 \
               $REGISTRY_URL/$IMAGE_REPOSITORY:$IMAGE_TAG

devops/otel/collector-config.dev.yaml

@@ -0,0 +1,36 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+  
+exporters:
+  debug:
+    verbosity: detailed
+  prometheus:
+    endpoint: 0.0.0.0:8889
+    send_timestamps: true
+
+processors:
+  batch:
+  attributes:
+    actions:
+      - action: insert
+        key: env
+        value: "${DEPLOY_ENV}"
+
+extensions:
+  zpages:
+    endpoint: 0.0.0.0:55679
+
+service:
+  extensions: [zpages]
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [batch, attributes]
+      exporters: [prometheus]
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug]

devops/otel/collector-config.prod.yaml

@@ -0,0 +1,53 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 100
+    spike_limit_mib: 20
+  batch:
+    timeout: 10s
+    send_batch_size: 1024
+  filter/drop_health:
+    traces:
+      span:
+        - 'attributes["http.target"] == "/health"'
+        - 'attributes["http.target"] == "/metrics"'
+
+exporters:
+  otlphttp/grafana:
+    endpoint: ${env:GRAFANA_CLOUD_OTLP_ENDPOINT}
+    headers:
+      Authorization: "Basic ${env:GRAFANA_CLOUD_AUTH}"
+    compression: gzip
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 120s
+    sending_queue:
+      enabled: true
+      queue_size: 1000
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+service:
+  extensions: [health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, filter/drop_health, batch]
+      exporters: [otlphttp/grafana]
+    metrics:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [otlphttp/grafana]
+      

package.json

@@ -19,6 +19,18 @@
     "@fastify/cookie": "^11.0.2",
     "@fastify/cors": "^11.1.0",
     "@fastify/jwt": "^10.0.0",
+    "@fastify/otel": "^0.17.1",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.71.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.213.0",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.213.0",
+    "@opentelemetry/instrumentation-fastify": "^0.57.0",
+    "@opentelemetry/instrumentation-http": "^0.213.0",
+    "@opentelemetry/otlp-exporter-base": "^0.213.0",
+    "@opentelemetry/resources": "^2.6.0",
+    "@opentelemetry/sdk-metrics": "^2.6.0",
+    "@opentelemetry/sdk-node": "^0.213.0",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
     "@prisma/adapter-pg": "7",
     "@prisma/client": "^7.4.2",
     "@prisma/client-runtime-utils": "^7.4.2",