feat!: add cloudflare queue & convex http actions for workers (#11)

Author: chenxin-yan

.github/workflows/scraper.yaml

@@ -25,6 +25,12 @@ jobs:
       - name: 📦 Install dependencies
         run: bun install
 
+      - name: 📦 Install Doppler CLI
+        uses: dopplerhq/cli-action@v3
+
+      - name: 🔑 Sync secrets with Cloudflare Worker
+        run: doppler secrets --json | jq -c 'with_entries(.value = .value.computed)') | wrangler secret bulk
+
       - name: 🚀 Deploy to Cloudflare Workers
         run: bun run deploy
         env:

apps/scraper/drizzle-dev.config.ts

@@ -0,0 +1,14 @@
+import { config } from "dotenv";
+import { defineConfig } from "drizzle-kit";
+
+config({ path: "./.env" });
+
+export default defineConfig({
+  out: "./src/drizzle/migrations",
+  schema: "./src/drizzle/schema.ts",
+  dialect: "sqlite",
+  dbCredentials: {
+    // biome-ignore lint/style/noNonNullAssertion: env variables must exist
+    url: process.env.DEV_DATABASE_URL!,
+  },
+});

apps/scraper/drizzle-prod.config.ts

@@ -0,0 +1,21 @@
+/** biome-ignore-all lint/style/noNonNullAssertion: env variables must exist */
+import { config } from "dotenv";
+import { defineConfig } from "drizzle-kit";
+
+config({ path: "./.env" });
+
+const accountId = process.env.CLOUDFLARE_ACCOUNT_ID!;
+const databaseId = process.env.CLOUDFLARE_DATABASE_ID!;
+const token = process.env.CLOUDFLARE_D1_TOKEN!;
+
+export default defineConfig({
+  out: "./src/drizzle/migrations",
+  schema: "./src/drizzle/schema.ts",
+  dialect: "sqlite",
+  driver: "d1-http",
+  dbCredentials: {
+    accountId,
+    databaseId,
+    token,
+  },
+});

apps/scraper/package.json

@@ -7,13 +7,22 @@
     "deploy": "wrangler deploy --minify",
     "check": "biome check",
     "check:types": "tsc --noEmit",
+    "db:studio:local": "drizzle-kit studio --config=drizzle-dev.config.ts",
+    "db:studio:remote": "drizzle-kit studio --config=drizzle-prod.config.ts",
+    "db:push:local": "drizzle-kit push --config=drizzle-dev.config.ts",
+    "db:push:remote": "drizzle-kit push --config=drizzle-prod.config.ts",
     "cf-typegen": "wrangler types --env-interface CloudflareBindings"
   },
   "dependencies": {
-    "hono": "^4.9.9"
+    "dotenv": "^17.2.3",
+    "drizzle-orm": "^0.44.5",
+    "hono": "^4.9.9",
+    "zod": "^4.1.11",
+    "@dev-team-fall-25/server": "workspace:*"
   },
   "devDependencies": {
     "@biomejs/biome": "2.2.4",
+    "drizzle-kit": "^0.31.5",
     "wrangler": "^4.40.3"
   }
 }

apps/scraper/src/drizzle/index.ts

@@ -0,0 +1,7 @@
+import { drizzle } from "drizzle-orm/d1";
+
+const createDB = async (env: CloudflareBindings) => {
+  return drizzle(env.DB);
+};
+
+export default createDB;

apps/scraper/src/drizzle/schema.ts

@@ -0,0 +1,29 @@
+import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+
+export const jobs = sqliteTable("jobs", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  url: text("url").notNull(),
+  status: text("status", {
+    enum: ["pending", "processing", "completed", "failed"],
+  }).notNull(),
+  jobType: text("job_type", {
+    enum: ["discovery", "program", "course"],
+  }).notNull(),
+  createdAt: integer("created_at", { mode: "timestamp" })
+    .notNull()
+    .$defaultFn(() => new Date()),
+  startedAt: integer("started_at", { mode: "timestamp" }),
+  completedAt: integer("completed_at", { mode: "timestamp" }),
+});
+
+export const errorLogs = sqliteTable("error_logs", {
+  id: integer("id").primaryKey({ autoIncrement: true }),
+  jobId: integer("job_id").references(() => jobs.id),
+  errorType: text("error_type", {
+    enum: ["network", "parsing", "validation", "timeout"],
+  }).notNull(),
+  errorMessage: text("error_message").notNull(),
+  stackTrace: text("stack_trace"),
+  retryCount: integer("retry_count").notNull(),
+  timestamp: integer("timestamp", { mode: "timestamp" }).notNull(),
+});

apps/scraper/src/index.ts

@@ -1,9 +1,31 @@
+/** biome-ignore-all lint/correctness/noUnusedFunctionParameters: disable for now as they haven't been implemented yet */
 import { Hono } from "hono";
 
-const app = new Hono();
+const app = new Hono<{ Bindings: CloudflareBindings }>();
 
-app.get("/", (c) => {
-  return c.text("Hello Hono!");
+app.get("/", async (c) => {
+  // const db = await createDB(c.env);
+  // TODO: use hono to render a dashboard to monitor the scraping status
 });
 
-export default app;
+export default {
+  fetch: app.fetch,
+
+  async scheduled(event: ScheduledEvent, env: CloudflareBindings) {
+    // const db = await createDB(env);
+    // const api = new ConvexApi({
+    //   baseUrl: env.CONVEX_SITE_URL,
+    //   apiKey: env.CONVEX_API_KEY,
+    // });
+    // TODO: set up jobs for scraping a list of urls need to be scraped and add them to queue as "discovery"
+  },
+
+  async queue(batch: MessageBatch<Error>, env: CloudflareBindings) {
+    // const db = await createDB(env);
+    // const api = new ConvexApi({
+    //   baseUrl: env.CONVEX_SITE_URL,
+    //   apiKey: env.CONVEX_API_KEY,
+    // });
+    // TODO: set up jobs for scrping given url and save structured data to convex database
+  },
+};

apps/scraper/src/lib/convex.ts

@@ -0,0 +1,99 @@
+import {
+  ZCreatePrerequisite,
+  ZCreateRequirement,
+  ZDeleteCourse,
+  ZDeletePrerequisites,
+  ZDeleteProgram,
+  ZDeleteRequirements,
+  ZUpsertCourse,
+  ZUpsertProgram,
+} from "@dev-team-fall-25/server/convex/http";
+import type * as z from "zod/mini";
+
+type ConvexApiConfig = {
+  baseUrl: string;
+  apiKey: string;
+};
+
+export class ConvexApi {
+  private config: ConvexApiConfig;
+
+  constructor(config: ConvexApiConfig) {
+    this.config = config;
+  }
+
+  private async request<T extends z.ZodMiniType>(
+    path: string,
+    schema: T,
+    data: z.infer<T>,
+  ): Promise<{ success: boolean; id?: string }> {
+    const validated = schema.parse(data);
+
+    const response = await fetch(`${this.config.baseUrl}/${path}`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "X-API-KEY": this.config.apiKey,
+      },
+      body: JSON.stringify(validated),
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${await response.text()}`);
+    }
+
+    return response.json();
+  }
+
+  async upsertCourse(data: z.infer<typeof ZUpsertCourse>) {
+    const result = await this.request(
+      "/api/courses/upsert",
+      ZUpsertCourse,
+      data,
+    );
+    return result.id;
+  }
+
+  async deleteCourse(data: z.infer<typeof ZDeleteCourse>) {
+    await this.request("/api/courses/delete", ZDeleteCourse, data);
+  }
+
+  async upsertProgram(data: z.infer<typeof ZUpsertProgram>) {
+    const result = await this.request(
+      "/api/programs/upsert",
+      ZUpsertProgram,
+      data,
+    );
+    return result.id;
+  }
+
+  async deleteProgram(data: z.infer<typeof ZDeleteProgram>) {
+    await this.request("/api/programs/delete", ZDeleteProgram, data);
+  }
+
+  async createRequirement(data: z.infer<typeof ZCreateRequirement>) {
+    const result = await this.request(
+      "/api/requirements/create",
+      ZCreateRequirement,
+      data,
+    );
+    return result.id;
+  }
+
+  async deleteRequirements(data: z.infer<typeof ZDeleteRequirements>) {
+    await this.request("/api/requirements/delete", ZDeleteRequirements, data);
+  }
+
+  async createPrerequisite(data: z.infer<typeof ZCreatePrerequisite>) {
+    const result = await this.request(
+      "/api/prerequisites/create",
+      ZCreatePrerequisite,
+      data,
+    );
+    return result.id;
+  }
+
+  async deletePrerequisites(data: z.infer<typeof ZDeletePrerequisites>) {
+    await this.request("/api/prerequisites/delete", ZDeletePrerequisites, data);
+  }
+}

apps/scraper/tsconfig.json

@@ -8,5 +8,6 @@
     "lib": ["ESNext"],
     "jsx": "react-jsx",
     "jsxImportSource": "hono/jsx"
-  }
+  },
+  "include": ["src/**/*", "worker-configuration.d.ts"]
 }

apps/scraper/worker-configuration.d.ts

@@ -1,11 +1,16 @@
 /* eslint-disable */
-// Generated by Wrangler by running `wrangler types --env-interface CloudflareBindings` (hash: 502e2034a04451831617b45e933366c8)
-// Runtime types generated with workerd@1.20250924.0 2025-09-26 
+// Generated by Wrangler by running `wrangler types --env-interface CloudflareBindings` (hash: e48c63bb4c89a59e02be6825aa1e7c60)
+// Runtime types generated with workerd@1.20250927.0 2025-09-26 
 declare namespace Cloudflare {
 	interface GlobalProps {
 		mainModule: typeof import("./src/index");
 	}
 	interface Env {
+		SCRAPING_BASE_URL: "https://bulletins.nyu.edu/";
+		CONVEX_SITE_URL: string;
+		CONVEX_API_KEY: string;
+		DB: D1Database;
+		SCRAPING_QUEUE: Queue;
 	}
 }
 interface CloudflareBindings extends Cloudflare.Env {}
@@ -7463,6 +7468,19 @@ interface MediaError extends Error {
     readonly message: string;
     readonly stack?: string;
 }
+declare module 'cloudflare:node' {
+    interface NodeStyleServer {
+        listen(...args: unknown[]): this;
+        address(): {
+            port?: number | null | undefined;
+        };
+    }
+    export function httpServerHandler(port: number): ExportedHandler;
+    export function httpServerHandler(options: {
+        port: number;
+    }): ExportedHandler;
+    export function httpServerHandler(server: NodeStyleServer): ExportedHandler;
+}
 type Params<P extends string = any> = Record<P, string | string[]>;
 type EventContext<Env, P extends string, Data> = {
     request: Request<unknown, IncomingRequestCfProperties<unknown>>;
@@ -7720,19 +7738,6 @@ declare namespace Cloudflare {
          & (K extends GlobalProp<"durableNamespaces", never> ? MainModule[K] extends new (...args: any[]) => infer DoInstance ? DoInstance extends Rpc.DurableObjectBranded ? DurableObjectNamespace<DoInstance> : DurableObjectNamespace<undefined> : DurableObjectNamespace<undefined> : {});
     };
 }
-declare module 'cloudflare:node' {
-    export interface DefaultHandler {
-        fetch?(request: Request): Response | Promise<Response>;
-        tail?(events: TraceItem[]): void | Promise<void>;
-        trace?(traces: TraceItem[]): void | Promise<void>;
-        scheduled?(controller: ScheduledController): void | Promise<void>;
-        queue?(batch: MessageBatch<unknown>): void | Promise<void>;
-        test?(controller: TestController): void | Promise<void>;
-    }
-    export function httpServerHandler(options: {
-        port: number;
-    }, handlers?: Omit<DefaultHandler, 'fetch'>): DefaultHandler;
-}
 declare namespace CloudflareWorkersModule {
     export type RpcStub<T extends Rpc.Stubable> = Rpc.Stub<T>;
     export const RpcStub: {