Merge pull request #1 from filecoin-station/cache-response-inside-cloudflare-worker

Add cloudflare worker to collect API key usage inside Influx DB

Author: pyropy

.dev.vars.example

@@ -0,0 +1 @@
+INFLUX_TOKEN="example"

.env.example

@@ -0,0 +1,42 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: cp wrangler.toml.example wrangler.toml
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - run: npm ci
+      - run: npm test
+
+  deploy:
+    if: github.ref == 'refs/heads/main'
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Deploy
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          preCommands: |
+            cp wranger.toml.example wrangler.toml
+            envsubst < wrangler.toml.example > wrangler.toml
+          postCommands: |
+            rm -f wrangler.toml
+          environment: production
+          secrets: |
+            INFLUX_TOKEN
+        env:
+          INFLUX_METRIC_NAME: ${{ env.INFLUX_METRIC }}
+          INFLUX_URL: ${{ env.INFLUX_URL }}
+          INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
+          INFLUX_DATABASE: ${{ env.INFLUX_DATABASE }}

.github/workflows/ci.yml

@@ -2,14 +2,14 @@
 /dist
 **/*.rs.bk
 Cargo.lock
-bin/
 pkg/
 wasm-pack.log
 worker/
 node_modules/
 .cargo-ok
 .env
+.dev.vars
 .wrangler
 data/
 config/
-wrangler.toml
+wrangler.toml

.gitignore

@@ -1,34 +1,37 @@
 # Cloudflare metrics worker
 
-Send your page views from [Cloudflare worker](https://developers.cloudflare.com/workers/) to InfluxDB.
+[Couldflare worker](https://developers.cloudflare.com/workers/) used to proxy requests to your application and send metrics to InfluxDB.
 
-![Dashboard views](static/dashboard.png)
+## Development
 
-## Requirements
-
-1. Your site need to be setup behind Cloudflare CDN.
-2. You need to setup InfluxDB with external access (make sure you have set [authentication](https://docs.influxdata.com/influxdb/v1.7/administration/authentication_and_authorization/#set-up-authentication))
-   1. Make sure InfluxDB is hosted under [supported port](https://blog.cloudflare.com/cloudflare-now-supporting-more-ports/) for Workers. Best option is 80 or 443.
-
-## How to use
-
-1. Install wrangler package
+1. Install dependencies
 
 ```
-npm i @cloudflare/wrangler -g
+npm install
 ```
 
 2. Copy example files
 
 ```
-cp .env.example .env
+cp .dev.vars.example .dev.vars
 cp wrangler.toml.example wrangler.toml
 ```
 
-3. Deploy your worker to a site with wrangler
+3. Edit secrets inside `.dev.vars` and environment variables inside `wrangler.toml` files
+
+3. Run your worker
 
 ```
-wrangler publish --env production
+npm run dev
 ```
 
-4. (Optional) If you're using Grafana with InfluxDB, then you can import [example Dashboard](static/dashboard.json) from first screen.
+## Deployment to production
+
+In order to deploy your worker via Github Actions, you need to have a [Cloudflare API token](https://developers.cloudflare.com/fundamentals/api/get-started/create-token/) and running instance of InfluxDB.
+
+Add generated API token to Github secrets as `CLOUDFLARE_API_TOKEN` and authentication token under `INFLUX_TOKEN`.
+
+Other required environment variables include the following:
+- `INFLUX_URL` - InfluxDB URL
+- `INFLUX_DATABASE` - InfluxDB database (bucket) name
+- `INFLUX_METRIC_NAME` - InfluxDB metric name

README.md

@@ -0,0 +1,9 @@
+import { reportRequestMetrics as reportRequestMetricsToInflux } from '../lib/influx.js'
+
+export default {
+  async fetch(request, env, ctx, { reportRequestMetrics = reportRequestMetricsToInflux } = {}) {
+    const response = await fetch(request)
+    ctx.waitUntil(reportRequestMetrics(request, env))
+    return response
+  },
+}

bin/worker.js

@@ -0,0 +1,47 @@
+/* 
+* Reports request metrics to InfluxDB
+* @param {Request} request - incoming request
+* @param {object} env - environment variables
+*/
+export const reportRequestMetrics = async (request, env) => {
+  await writeMetrics(createMetricsFromRequest(request, env), env)
+}
+
+/*
+* Returns request metrics in InfluxDB line protocol format
+* https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/ 
+* @param {Request} request - incoming request
+* @param {object} env - environment variables
+*/
+export const createMetricsFromRequest = (request, env) => {
+  const url = new URL(request.url);
+  const timestamp = Date.now();
+  const apiKey = request.headers.get('api-key') ||
+    url.searchParams.get('api-key') || (
+      request.headers.get('Authorization')?.startsWith('Bearer ')
+        ? request.headers.get('Authorization').substring(7)
+        : 'unknown'
+    );
+
+  return `${env.INFLUX_METRIC_NAME} api_key="${apiKey}" ${timestamp}`
+}
+
+/*
+* Sends request metrics to InfluxDB
+* https://docs.influxdata.com/enterprise_influxdb/v1/guides/write_data/
+* @param {string} lineProtocolData - InfluxDB line protocol formatted data
+* @param {object} env - environment variables
+*/
+export const writeMetrics = async (lineProtocolData, env) => {
+  // Define API endpoint and headers
+  const url = `${env.INFLUX_URL}/api/v2/write?&bucket=${env.INFLUX_DATABASE}&precision=ms`;
+
+  return fetch(url, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Token ${env.INFLUX_TOKEN}`,
+      'Content-Type': 'application/octet-stream'
+    },
+    body: lineProtocolData,
+  })
+}

index.js

@@ -6,9 +6,11 @@
   "main": "index.js",
   "scripts": {
     "deploy": "wrangler deploy",
-    "dev": "wrangler dev",
+    "deploy:prod": "wrangler deploy --env production",
+    "dev": "wrangler dev --env dev",
     "start": "wrangler dev",
-    "test": "vitest"
+    "test": "vitest run",
+    "test:watch": "vitest"
   },
   "author": "vanadium23 <[email protected]>",
   "license": "MIT",