feat(content api): add management api references to semantic search (supabase#36289)

* docs: add cursor rule for embedding generation process

Add documentation for cursor IDE about how docs embeddings are generated,
including the workflow for creating and uploading semantic search content.

* feat: improve API reference metadata upload with descriptive content

- Add preembeddings script to run codegen before embedding generation
- Enhance OpenApiReferenceSource to generate more descriptive content including
  parameters, responses, path information, and better structured documentation

* feat: add Management API references to searchDocs GraphQL query

- Add ManagementApiReference GraphQL type and model for API endpoint search results
- Integrate Management API references into global search results
- Update test snapshots and add comprehensive test coverage for Management API search

* style: format

Author: charislam

.cursor/rules/docs-embeddings-generation.md

@@ -0,0 +1,59 @@
+# Documentation Embeddings Generation System
+
+## Overview
+
+The documentation embeddings generation system processes various documentation sources and uploads their metadata to a database for semantic search functionality. The system is located in `apps/docs/scripts/search/` and works by:
+
+1. **Discovering content sources** from multiple types of documentation
+2. **Processing content** into structured sections with checksums
+3. **Generating embeddings** using OpenAI's text-embedding-ada-002 model
+4. **Storing in database** with vector embeddings for semantic search
+
+## Architecture
+
+### Main Entry Point
+- `generate-embeddings.ts` - Main script that orchestrates the entire process
+- Supports `--refresh` flag to force regeneration of all content
+
+### Content Sources (`sources/` directory)
+
+#### Base Classes
+- `BaseLoader` - Abstract class for loading content from different sources
+- `BaseSource` - Abstract class for processing and formatting content
+
+#### Source Types
+1. **Markdown Sources** (`markdown.ts`)
+   - Processes `.mdx` files from guides and documentation
+   - Extracts frontmatter metadata and content sections
+
+2. **Reference Documentation** (`reference-doc.ts`)
+   - **OpenAPI References** - Management API documentation from OpenAPI specs
+   - **Client Library References** - JavaScript, Dart, Python, C#, Swift, Kotlin SDKs
+   - **CLI References** - Command-line interface documentation
+   - Processes YAML/JSON specs and matches with common sections
+
+3. **GitHub Discussions** (`github-discussion.ts`)
+   - Fetches troubleshooting discussions from GitHub using GraphQL API
+   - Uses GitHub App authentication for access
+
+4. **Partner Integrations** (`partner-integrations.ts`)
+   - Fetches approved partner integration documentation from Supabase database
+   - Technology integrations only (excludes agencies)
+
+### Processing Flow
+
+1. **Content Discovery**: Each source loader discovers and loads content files/data
+2. **Content Processing**: Each source processes content into:
+   - Checksum for change detection
+   - Metadata (title, subtitle, etc.)
+   - Sections with headings and content
+3. **Change Detection**: Compares checksums against existing database records
+4. **Embedding Generation**: Uses OpenAI to generate embeddings for new/changed content
+5. **Database Storage**: Stores in `page` and `page_section` tables with embeddings
+6. **Cleanup**: Removes outdated pages using version tracking
+
+### Database Schema
+
+- **`page`** table: Stores page metadata, content, checksum, version
+- **`page_section`** table: Stores individual sections with embeddings, token counts
+

apps/docs/app/api/graphql/__snapshots__/route.test.ts.snap

@@ -84,6 +84,20 @@ type CLICommandReference implements SearchResult {
   content: String
 }
 
+"""
+A reference document containing a description of a Supabase Management API endpoint
+"""
+type ManagementApiReference implements SearchResult {
+  """The title of the document"""
+  title: String
+
+  """The URL of the document"""
+  href: String
+
+  """The content of the reference document, as text"""
+  content: String
+}
+
 """
 A reference document containing a description of a function from a Supabase client library
 """

apps/docs/app/api/graphql/tests/searchDocs.smoke.test.ts

@@ -204,4 +204,40 @@ describe('prod smoke test: graphql: searchDocs', () => {
     expect(guideNode).toHaveProperty('href')
     expect(guideNode).toHaveProperty('content')
   })
+
+  it('searchDocs query includes Management API references', async () => {
+    const query = `
+        query SearchDocsQuery($query: String!) {
+          searchDocs(query: $query) {
+            nodes {
+              ...on ManagementApiReference {
+                title
+                href
+                content
+              }
+            }
+          }
+        }
+      `
+    const result = await fetch(GRAPHQL_URL, {
+      method: 'POST',
+      body: JSON.stringify({ query, variables: { query: 'create SSO provider' } }),
+    })
+
+    expect(result.status).toBe(200)
+    const { data, errors } = await result.json()
+    expect(errors).toBeUndefined()
+
+    const {
+      searchDocs: { nodes },
+    } = data
+    expect(Array.isArray(nodes)).toBe(true)
+    expect(nodes.length).toBeGreaterThan(0)
+
+    const managementApiNode = nodes.find((node: any) => !!node.title)
+    expect(managementApiNode).toBeDefined()
+    expect(managementApiNode).toHaveProperty('title')
+    expect(managementApiNode).toHaveProperty('href')
+    expect(managementApiNode).toHaveProperty('content')
+  })
 })

apps/docs/app/api/graphql/tests/searchDocs.test.ts

@@ -33,6 +33,16 @@ const rpcSpy = vi.fn().mockImplementation((funcName, params) => {
         content: params?.include_full_content ? 'Another content' : null,
         subsections: [{ title: 'Getting Started', content: 'Getting Started content' }],
       },
+      {
+        type: 'reference',
+        page_title: 'Create a SSO provider',
+        href: 'https://supabase.com/docs/reference/api/v1-create-a-sso-provider',
+        content: params?.include_full_content ? 'Creates a new SSO provider for a project' : null,
+        metadata: {
+          title: 'Create a SSO provider',
+          subtitle: 'Management API Reference: Create a SSO provider',
+        },
+      },
     ]
     return Promise.resolve({ data: mockResults.slice(0, limit), error: null })
   }
@@ -190,4 +200,40 @@ describe('/api/graphql searchDocs', () => {
     expect(json.errors).toBeDefined()
     expect(json.errors[0].message).toContain('required')
   })
+
+  it('should return Management API references with proper fields', async () => {
+    const searchQuery = `
+      query {
+        searchDocs(query: "SSO provider", limit: 3) {
+          nodes {
+            ... on ManagementApiReference {
+              title
+              href
+              content
+            }
+          }
+        }
+      }
+    `
+    const request = new Request('http://localhost/api/graphql', {
+      method: 'POST',
+      body: JSON.stringify({ query: searchQuery }),
+    })
+
+    const response = await POST(request)
+    const json = await response.json()
+
+    expect(json.errors).toBeUndefined()
+    expect(json.data).toBeDefined()
+    expect(json.data.searchDocs).toBeDefined()
+    expect(json.data.searchDocs.nodes).toBeInstanceOf(Array)
+    expect(json.data.searchDocs.nodes).toHaveLength(3)
+
+    const managementApiNode = json.data.searchDocs.nodes[2]
+    expect(managementApiNode).toMatchObject({
+      title: 'Create a SSO provider',
+      href: 'https://supabase.com/docs/reference/api/v1-create-a-sso-provider',
+      content: 'Creates a new SSO provider for a project',
+    })
+  })
 })

apps/docs/lib/supabase.ts

@@ -18,7 +18,12 @@ type Database = {
             DatabaseGenerated['public']['Functions']['search_content']['Returns'][number],
             'subsections' | 'metadata'
           > & {
-            metadata: { language?: string; methodName?: string; platform?: string }
+            metadata: {
+              subtitle?: string
+              language?: string
+              methodName?: string
+              platform?: string
+            }
             subsections: Array<{ title?: string; href?: string; content?: string }>
           }
         >

apps/docs/package.json

@@ -25,6 +25,7 @@
     "postbuild": "pnpm run build:sitemap && pnpm run build:llms && ./../../scripts/upload-static-assets.sh",
     "prebuild": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples",
     "predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples",
+    "preembeddings": "pnpm run codegen:references",
     "preinstall": "npx only-allow pnpm",
     "presync": "pnpm run codegen:graphql",
     "pretest": "pnpm run codegen:examples",

apps/docs/resources/globalSearch/globalSearchModel.ts

@@ -8,6 +8,7 @@ import {
   DB_METADATA_TAG_PLATFORM_CLI,
   ReferenceCLICommandModel,
 } from '../reference/referenceCLIModel'
+import { ReferenceManagementApiModel } from '../reference/referenceManagementApiModel'
 import { ReferenceSDKFunctionModel, SDKLanguageValues } from '../reference/referenceSDKModel'
 import { TroubleshootingModel } from '../troubleshooting/troubleshootingModel'
 import { SearchResultInterface } from './globalSearchInterface'
@@ -74,6 +75,13 @@ function createModelFromMatch({
           content,
           subsections,
         })
+        // TODO [Charis 2025-06-09] replace with less hacky check
+      } else if (metadata.subtitle?.startsWith('Management API Reference')) {
+        return new ReferenceManagementApiModel({
+          title: page_title,
+          href,
+          content,
+        })
       } else {
         return null
       }

apps/docs/resources/reference/referenceManagementApiModel.ts

@@ -0,0 +1,13 @@
+import { type SearchResultInterface } from '../globalSearch/globalSearchInterface'
+
+export class ReferenceManagementApiModel implements SearchResultInterface {
+  public title?: string
+  public href?: string
+  public content?: string
+
+  constructor({ title, href, content }: { title?: string; href?: string; content?: string }) {
+    this.title = title
+    this.href = href
+    this.content = content
+  }
+}

apps/docs/resources/reference/referenceManagementApiSchema.ts

@@ -0,0 +1,25 @@
+import { GraphQLObjectType, GraphQLString } from 'graphql'
+import { GraphQLInterfaceTypeSearchResult } from '../globalSearch/globalSearchSchema'
+import { ReferenceManagementApiModel } from './referenceManagementApiModel'
+
+export const GraphQLObjectTypeReferenceManagementApi = new GraphQLObjectType({
+  name: 'ManagementApiReference',
+  interfaces: [GraphQLInterfaceTypeSearchResult],
+  isTypeOf: (value: unknown) => value instanceof ReferenceManagementApiModel,
+  description:
+    'A reference document containing a description of a Supabase Management API endpoint',
+  fields: {
+    title: {
+      type: GraphQLString,
+      description: 'The title of the document',
+    },
+    href: {
+      type: GraphQLString,
+      description: 'The URL of the document',
+    },
+    content: {
+      type: GraphQLString,
+      description: 'The content of the reference document, as text',
+    },
+  },
+})

apps/docs/resources/rootSchema.ts

@@ -10,6 +10,7 @@ import { errorRoot, errorsRoot } from './error/errorResolver'
 import { searchRoot } from './globalSearch/globalSearchResolver'
 import { GraphQLObjectTypeGuide } from './guide/guideSchema'
 import { GraphQLObjectTypeReferenceCLICommand } from './reference/referenceCLISchema'
+import { GraphQLObjectTypeReferenceManagementApi } from './reference/referenceManagementApiSchema'
 import { GraphQLObjectTypeReferenceSDKFunction } from './reference/referenceSDKSchema'
 import { GraphQLObjectTypeTroubleshooting } from './troubleshooting/troubleshootingSchema'
 
@@ -43,6 +44,7 @@ export const rootGraphQLSchema = new GraphQLSchema({
   types: [
     GraphQLObjectTypeGuide,
     GraphQLObjectTypeReferenceCLICommand,
+    GraphQLObjectTypeReferenceManagementApi,
     GraphQLObjectTypeReferenceSDKFunction,
     GraphQLObjectTypeTroubleshooting,
   ],