feat: export full database weekly

Co-authored-by: computers <computers@nickthesick.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>

Author: nperez0111

.env.template

@@ -8,6 +8,11 @@ LOG_LEVEL="info"                         # The log level. Options: 'trace', 'deb
 OPEN_OBSERVE_USER="user@bookhive.buzz"   # The email of the user that will be used to observe the open books
 OPEN_OBSERVE_PASSWORD="password"         # The password of the user that will be used to observe the open books
 OPEN_OBSERVE_URL="http://localhost:5080" # The password of the user that will be used to observe the open books
+# Export (optional)
+# If set, enables GET /admin/export (Authorization: Bearer <secret>)
+EXPORT_SHARED_SECRET=""
+# Optional directory for temporary export files (defaults to dirname(DB_PATH))
+DB_EXPORT_DIR=""
 # Secrets
 # Must set this in production. May be generated with `openssl rand -base64 33`
 # COOKIE_SECRET=""

.github/workflows/database-export.yml

@@ -0,0 +1,43 @@
+name: Publish database export artifact
+
+on:
+  schedule:
+    # Weekly (Sunday 02:15 UTC)
+    - cron: "15 2 * * 0"
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+  actions: write
+
+concurrency:
+  group: database-export
+  cancel-in-progress: false
+
+jobs:
+  export:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set export filename date
+        run: echo "EXPORT_DATE=$(date -u +%Y-%m-%d)" >> "$GITHUB_ENV"
+
+      - name: Download export from BookHive instance
+        env:
+          EXPORT_URL: ${{ secrets.BOOKHIVE_EXPORT_URL }}
+          EXPORT_SECRET: ${{ secrets.BOOKHIVE_EXPORT_SHARED_SECRET }}
+        run: |
+          test -n "$EXPORT_URL"
+          test -n "$EXPORT_SECRET"
+          curl -fL --retry 3 --retry-delay 5 \
+            -H "Authorization: Bearer $EXPORT_SECRET" \
+            "$EXPORT_URL" \
+            -o "bookhive-export.tgz"
+          ls -lh "bookhive-export.tgz"
+
+      - name: Upload export as GitHub Actions artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bookhive-export-${{ env.EXPORT_DATE }}
+          path: bookhive-export.tgz
+          retention-days: 30
+

README.md

@@ -79,3 +79,16 @@ pnpm test:ui
 - **Backend**: [Hono](https://hono.dev) with AT Proto for OAuth
 - **Frontend**: Mostly static HTML, with some Hono JSX for dynamic content (Fast as possible)
 - **Database**: SQLite, with Kyesly as the ORM
+
+## 🗄️ Weekly database export (GitHub Actions artifact)
+
+This repo includes a workflow that can fetch a **sanitized SQLite export** from your running BookHive instance and upload it as a GitHub Actions artifact (weekly cron + manual trigger).
+
+- **Server endpoint**: `GET /admin/export`
+  - Requires `EXPORT_SHARED_SECRET` to be set
+  - Request header: `Authorization: Bearer <EXPORT_SHARED_SECRET>`
+  - Returns a `.tgz` containing `db.sqlite`, `kv.sqlite` (with auth tables excluded), and `manifest.json`
+- **Workflow**: `.github/workflows/database-export.yml`
+  - Configure GitHub repo secrets:
+    - `BOOKHIVE_EXPORT_URL` (e.g. `https://bookhive.example.com/admin/export`)
+    - `BOOKHIVE_EXPORT_SHARED_SECRET`

eslint.config.js

@@ -0,0 +1,28 @@
+import js from "@eslint/js";
+import tseslintPlugin from "@typescript-eslint/eslint-plugin";
+import tsParser from "@typescript-eslint/parser";
+
+export default [
+  {
+    ignores: ["dist/**", "src/bsky/lexicon/**", ".eslintrc.cjs"],
+  },
+  js.configs.recommended,
+  {
+    files: ["**/*.ts", "**/*.tsx"],
+    languageOptions: {
+      parser: tsParser,
+      parserOptions: {
+        ecmaVersion: "latest",
+        sourceType: "module",
+        ecmaFeatures: { jsx: true },
+      },
+    },
+    plugins: {
+      "@typescript-eslint": tseslintPlugin,
+    },
+    rules: {
+      ...tseslintPlugin.configs.recommended.rules,
+    },
+  },
+];
+

src/env.ts

@@ -15,6 +15,16 @@ export const env = cleanEnv(process.env, {
     devDefault: ":memory:",
     desc: "Path to the KV SQLite database",
   }),
+  EXPORT_SHARED_SECRET: str({
+    default: "",
+    desc:
+      "Shared secret for triggering DB exports via /admin/export (Bearer token). Leave empty to disable.",
+  }),
+  DB_EXPORT_DIR: str({
+    default: "",
+    desc:
+      "Directory to write temporary export artifacts. Defaults to the directory containing DB_PATH.",
+  }),
   LOG_LEVEL: str({ default: "info", desc: "Log level for the app" }),
   COOKIE_SECRET: str({ devDefault: "00000000000000000000000000000000" }),
   OPEN_OBSERVE_URL: str({ devDefault: "" }),

src/index.ts

@@ -41,6 +41,12 @@ import { createRouter, searchBooks } from "./routes.tsx";
 import sqliteKv from "./sqlite-kv.ts";
 import type { HiveId } from "./types.ts";
 import { createBatchTransform } from "./utils/batchTransform.ts";
+import {
+  cleanupExportPaths,
+  createExportReadStream,
+  createSanitizedExportArchive,
+  isAuthorizedExportRequest,
+} from "./utils/dbExport.ts";
 import {
   getGoodreadsCsvParser,
   getStorygraphCsvParser,
@@ -55,6 +61,8 @@ import {
 
 import { lazy } from "./utils/lazy.ts";
 import { readThroughCache } from "./utils/readThroughCache.ts";
+import fs from "node:fs";
+import path from "node:path";
 
 // Application state passed to the router and elsewhere
 export type AppContext = {
@@ -255,6 +263,133 @@ export class Server {
     app.use("*", registerMetrics);
     app.get("/metrics", printMetrics);
 
+    // Download a sanitized SQLite export bundle (db + kv without auth tables)
+    app.get("/admin/export", async (c) => {
+      const ctx = c.get("ctx");
+      const clientIp =
+        c.req.header("x-forwarded-for")?.split(",")[0].trim() ||
+        c.req.header("x-real-ip") ||
+        "unknown";
+
+      try {
+        // Hide endpoint if not configured
+        if (!env.EXPORT_SHARED_SECRET) {
+          ctx.logger.warn(
+            { ip: clientIp, reason: "endpoint_not_configured" },
+            "export endpoint access attempt - endpoint disabled",
+          );
+          return c.json({ message: "Not Found" }, 404);
+        }
+
+        // Check authorization
+        const authorization = c.req.header("authorization");
+        if (
+          !isAuthorizedExportRequest({
+            authorizationHeader: authorization,
+            sharedSecret: env.EXPORT_SHARED_SECRET,
+          })
+        ) {
+          ctx.logger.warn(
+            { ip: clientIp, reason: "invalid_authorization" },
+            "export endpoint unauthorized access attempt",
+          );
+          return c.json({ message: "Not Found" }, 404);
+        }
+
+        // Validate database path
+        if (!env.DB_PATH || env.DB_PATH === ":memory:") {
+          ctx.logger.error(
+            { ip: clientIp, dbPath: env.DB_PATH },
+            "export endpoint called but DB_PATH is not a file path",
+          );
+          return c.json(
+            { message: "DB exports require DB_PATH to be a file path" },
+            400,
+          );
+        }
+
+        const exportDir =
+          env.DB_EXPORT_DIR?.trim() ||
+          path.join(path.dirname(env.DB_PATH), "exports");
+
+        ctx.logger.info(
+          { ip: clientIp, exportDir },
+          "starting database export",
+        );
+
+        const startTime = Date.now();
+        let result;
+
+        try {
+          await fs.promises.mkdir(exportDir, { recursive: true });
+
+          const includeKv =
+            Boolean(env.KV_DB_PATH) &&
+            env.KV_DB_PATH !== ":memory:" &&
+            fs.existsSync(env.KV_DB_PATH);
+
+          result = await createSanitizedExportArchive({
+            dbPath: env.DB_PATH,
+            kvPath: includeKv ? env.KV_DB_PATH : undefined,
+            exportDir,
+            includeKv,
+          });
+        } catch (err) {
+          const duration = Date.now() - startTime;
+          ctx.logger.error(
+            {
+              ip: clientIp,
+              duration,
+              error: err instanceof Error ? err.message : String(err),
+              stack: err instanceof Error ? err.stack : undefined,
+            },
+            "database export failed",
+          );
+          return c.json({ message: "Failed to create export archive" }, 500);
+        }
+
+        const duration = Date.now() - startTime;
+        const stream = createExportReadStream(result.archivePath, {
+          onClose: () => {
+            ctx.logger.info(
+              { ip: clientIp, filename: result.filename, duration },
+              "database export completed successfully",
+            );
+            cleanupExportPaths({
+              archivePath: result.archivePath,
+              tmpDir: result.tmpDir,
+            });
+          },
+          onError: (err) => {
+            ctx.logger.error(
+              { ip: clientIp, filename: result.filename, error: err.message },
+              "error streaming export file",
+            );
+            cleanupExportPaths({
+              archivePath: result.archivePath,
+              tmpDir: result.tmpDir,
+            });
+          },
+        });
+
+        return c.body(stream, 200, {
+          "Content-Type": "application/gzip",
+          "Content-Encoding": "gzip",
+          "Content-Disposition": `attachment; filename="${result.filename}"`,
+          "Cache-Control": "no-store",
+        });
+      } catch (err) {
+        ctx.logger.error(
+          {
+            ip: clientIp,
+            error: err instanceof Error ? err.message : String(err),
+          },
+          "unexpected error in export endpoint",
+        );
+        return c.json({ message: "Internal server error" }, 500);
+      }
+    });
+
     // This is to import a Goodreads CSV export
     // It is here because we don't want it behind the etag middleware
     app.post(