feat(docs): reindex docs on change (#863)

* Add github action to index docs

* Add new line

* Update .github/workflows/docs-embeddings.yml

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Updates

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

Author: Sg312

.github/workflows/docs-embeddings.yml

@@ -0,0 +1,38 @@
+name: Process Docs Embeddings
+
+on:
+  push:
+    branches: [main, staging]
+    paths:
+      - 'apps/docs/**'
+  workflow_dispatch: # Allow manual triggering
+
+jobs:
+  process-docs-embeddings:
+    name: Process Documentation Embeddings
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: latest
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Process docs embeddings
+        working-directory: ./apps/sim
+        env:
+          DATABASE_URL: ${{ github.ref == 'refs/heads/main' && secrets.DATABASE_URL || secrets.STAGING_DATABASE_URL }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: bun run scripts/process-docs-embeddings.ts --clear 

apps/sim/scripts/process-docs-embeddings.ts

@@ -53,18 +53,6 @@ async function processDocsEmbeddings(options: ProcessingOptions = {}) {
       clearExisting: config.clearExisting,
     })
 
-    // Clear existing embeddings if requested
-    if (config.clearExisting) {
-      logger.info('🗑️ Clearing existing docs embeddings...')
-      try {
-        const deleteResult = await db.delete(docsEmbeddings)
-        logger.info(`✅ Successfully deleted existing embeddings`)
-      } catch (error) {
-        logger.error('❌ Failed to delete existing embeddings:', error)
-        throw new Error('Failed to clear existing embeddings')
-      }
-    }
-
     // Initialize the docs chunker
     const chunker = new DocsChunker({
       chunkSize: config.chunkSize,
@@ -73,7 +61,7 @@ async function processDocsEmbeddings(options: ProcessingOptions = {}) {
       baseUrl: config.baseUrl,
     })
 
-    // Process all .mdx files
+    // Process all .mdx files first (compute embeddings before clearing)
     logger.info(`📚 Processing docs from: ${config.docsPath}`)
     const chunks = await chunker.chunkAllDocs(config.docsPath)
 
@@ -84,6 +72,18 @@ async function processDocsEmbeddings(options: ProcessingOptions = {}) {
 
     logger.info(`📊 Generated ${chunks.length} chunks with embeddings`)
 
+    // Clear existing embeddings if requested (after computing new ones to minimize downtime)
+    if (config.clearExisting) {
+      logger.info('🗑️ Clearing existing docs embeddings...')
+      try {
+        const deleteResult = await db.delete(docsEmbeddings)
+        logger.info(`✅ Successfully deleted existing embeddings`)
+      } catch (error) {
+        logger.error('❌ Failed to delete existing embeddings:', error)
+        throw new Error('Failed to clear existing embeddings')
+      }
+    }
+
     // Save chunks to database in batches for better performance
     const batchSize = 10
     logger.info(`💾 Saving chunks to database (batch size: ${batchSize})...`)