fix: harden GitHub backups

Author: steipete

.gitignore

@@ -16,6 +16,7 @@ count.txt
 .wrangler
 .output
 .vinxi
+*.bun-build
 todos.json
 .cta.json
 .vscode

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Added
 - API: v1 public REST endpoints with rate limits, raw file fetch, and OpenAPI spec.
 - Docs: `docs/api.md` and `DEPRECATIONS.md` for the v1 cutover plan.
+- Registry: GitHub App backs up published skills to `clawdbot/skills` (thanks @thewilloftheshadow, #5).
 
 ### Changed
 - CLI: publish now uses single multipart `POST /api/v1/skills`.

convex/_generated/api.d.ts

@@ -10,6 +10,7 @@
 
 import type * as auth from "../auth.js";
 import type * as comments from "../comments.js";
+import type * as crons from "../crons.js";
 import type * as downloads from "../downloads.js";
 import type * as githubBackups from "../githubBackups.js";
 import type * as githubBackupsNode from "../githubBackupsNode.js";
@@ -20,6 +21,7 @@ import type * as lib_access from "../lib/access.js";
 import type * as lib_apiTokenAuth from "../lib/apiTokenAuth.js";
 import type * as lib_changelog from "../lib/changelog.js";
 import type * as lib_embeddings from "../lib/embeddings.js";
+import type * as lib_githubBackup from "../lib/githubBackup.js";
 import type * as lib_skillBackfill from "../lib/skillBackfill.js";
 import type * as lib_skillPublish from "../lib/skillPublish.js";
 import type * as lib_skills from "../lib/skills.js";
@@ -45,6 +47,7 @@ import type {
 declare const fullApi: ApiFromModules<{
   auth: typeof auth;
   comments: typeof comments;
+  crons: typeof crons;
   downloads: typeof downloads;
   githubBackups: typeof githubBackups;
   githubBackupsNode: typeof githubBackupsNode;
@@ -55,6 +58,7 @@ declare const fullApi: ApiFromModules<{
   "lib/apiTokenAuth": typeof lib_apiTokenAuth;
   "lib/changelog": typeof lib_changelog;
   "lib/embeddings": typeof lib_embeddings;
+  "lib/githubBackup": typeof lib_githubBackup;
   "lib/skillBackfill": typeof lib_skillBackfill;
   "lib/skillPublish": typeof lib_skillPublish;
   "lib/skills": typeof lib_skills;

convex/crons.ts

@@ -7,7 +7,7 @@ crons.interval(
   'github-backup-sync',
   { minutes: 30 },
   internal.githubBackupsNode.syncGitHubBackupsInternal,
-  {},
+  { batchSize: 50, maxBatches: 5 },
 )
 
 export default crons

convex/githubBackups.ts

@@ -1,11 +1,12 @@
 import { v } from 'convex/values'
 import { internal } from './_generated/api'
 import type { Doc, Id } from './_generated/dataModel'
-import { action, internalQuery } from './_generated/server'
+import { action, internalMutation, internalQuery } from './_generated/server'
 import { assertRole, requireUserFromAction } from './lib/access'
 
-const MAX_BATCH_SIZE = 500
-const DEFAULT_BATCH_SIZE = MAX_BATCH_SIZE
+const DEFAULT_BATCH_SIZE = 50
+const MAX_BATCH_SIZE = 200
+const SYNC_STATE_KEY = 'default'
 
 type BackupPageItem =
   | {
@@ -29,6 +30,23 @@ type BackupPageResult = {
   isDone: boolean
 }
 
+type BackupSyncState = {
+  cursor: string | null
+}
+
+export type SyncGitHubBackupsResult = {
+  stats: {
+    skillsScanned: number
+    skillsSkipped: number
+    skillsBackedUp: number
+    skillsMissingVersion: number
+    skillsMissingOwner: number
+    errors: number
+  }
+  cursor: string | null
+  isDone: boolean
+}
+
 export const getGitHubBackupPageInternal = internalQuery({
   args: {
     cursor: v.optional(v.string()),
@@ -72,7 +90,7 @@ export const getGitHubBackupPageInternal = internalQuery({
         slug: skill.slug,
         displayName: skill.displayName,
         version: version.version,
-        ownerHandle: owner.handle ?? owner.name ?? owner.email ?? owner._id,
+        ownerHandle: owner.handle ?? owner._id,
         files: version.files,
         publishedAt: version.createdAt,
       })
@@ -82,20 +100,68 @@ export const getGitHubBackupPageInternal = internalQuery({
   },
 })
 
-export const syncGitHubBackups = action({
+export const getGitHubBackupSyncStateInternal = internalQuery({
+  args: {},
+  handler: async (ctx): Promise<BackupSyncState> => {
+    const state = await ctx.db
+      .query('githubBackupSyncState')
+      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
+      .unique()
+    return { cursor: state?.cursor ?? null }
+  },
+})
+
+export const setGitHubBackupSyncStateInternal = internalMutation({
+  args: {
+    cursor: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const now = Date.now()
+    const state = await ctx.db
+      .query('githubBackupSyncState')
+      .withIndex('by_key', (q) => q.eq('key', SYNC_STATE_KEY))
+      .unique()
+
+    if (!state) {
+      await ctx.db.insert('githubBackupSyncState', {
+        key: SYNC_STATE_KEY,
+        cursor: args.cursor,
+        updatedAt: now,
+      })
+      return { ok: true as const }
+    }
+
+    await ctx.db.patch(state._id, {
+      cursor: args.cursor,
+      updatedAt: now,
+    })
+
+    return { ok: true as const }
+  },
+})
+
+export const syncGitHubBackups: ReturnType<typeof action> = action({
   args: {
     dryRun: v.optional(v.boolean()),
     batchSize: v.optional(v.number()),
     maxBatches: v.optional(v.number()),
+    resetCursor: v.optional(v.boolean()),
   },
-  handler: async (ctx, args) => {
+  handler: async (ctx, args): Promise<SyncGitHubBackupsResult> => {
     const { user } = await requireUserFromAction(ctx)
     assertRole(user, ['admin'])
+
+    if (args.resetCursor && !args.dryRun) {
+      await ctx.runMutation(internal.githubBackups.setGitHubBackupSyncStateInternal, {
+        cursor: undefined,
+      })
+    }
+
     return ctx.runAction(internal.githubBackupsNode.syncGitHubBackupsInternal, {
       dryRun: args.dryRun,
       batchSize: args.batchSize,
       maxBatches: args.maxBatches,
-    })
+    }) as Promise<SyncGitHubBackupsResult>
   },
 })