Expand Ask Fern documentation

fern/apis/fai/definition/commons.yml

@@ -7,3 +7,10 @@ errors:
   InternalError:
     status-code: 401
     type: string
+
+types:
+  Pagination:
+    properties:
+      total: integer
+      page: integer
+      limit: integer

fern/apis/fai/definition/document.yml

@@ -40,7 +40,59 @@ service:
             authed:
               type: optional<boolean>
               docs: Whether the document is authed
-      response: CreateDocumentResponse
+      response: DocumentIdResponse
+      errors:
+        - commons.BadRequestError
+        - commons.InternalError
+
+    updateDocument:
+      docs: Update a document for a given domain
+      path: /{domain}/{document_id}
+      method: PATCH
+      audiences:
+        - customers
+      path-parameters:
+        domain: string
+        document_id: string
+      request:
+        name: UpdateDocumentRequest
+        body:
+          properties:
+            document:
+              type: optional<string>
+              docs: The content of the document that will be returned in the tool response.
+            chunk:
+              type: optional<string>
+              docs: The chunk of the document that will be indexed as a vector.
+            title:
+              type: optional<string>
+              docs: The title of the document
+            url:
+              type: optional<string>
+              docs: The url of the document
+            version:
+              type: optional<string>
+              docs: The version of the document
+            keywords:
+              type: optional<list<string>>
+              docs: The keywords of the document
+            authed:
+              type: optional<boolean>
+              docs: Whether the document is authed
+      response: Document
+      errors:
+        - commons.BadRequestError
+        - commons.InternalError
+
+    deleteDocumentById:
+      docs: Delete a document for a given domain
+      path: /{domain}/{document_id}
+      method: DELETE
+      audiences:
+        - customers
+      path-parameters:
+        domain: string
+        document_id: string
       errors:
         - commons.BadRequestError
         - commons.InternalError
@@ -60,7 +112,7 @@ service:
         - commons.InternalError
 
 types:
-  CreateDocumentResponse:
+  DocumentIdResponse:
     properties:
       document_id: string
 

fern/apis/fai/definition/guidance.yml

@@ -25,6 +25,7 @@ service:
             document:
               type: string
               docs: The content of the guidance document that will be returned in the tool response
+      response: GuidanceIdResponse
       errors:
         - commons.BadRequestError
         - commons.InternalError
@@ -101,6 +102,10 @@ service:
         - commons.InternalError
 
 types:
+  GuidanceIdResponse:
+    properties:
+      guidance_id: string
+
   Guidance:
     properties:
       guidance_id: string
@@ -113,10 +118,4 @@ types:
   GuidanceList:
     properties:
       guidances: list<Guidance>
-      pagination: Pagination
-
-  Pagination:
-    properties:
-      total: integer
-      page: integer
-      limit: integer
+      pagination: commons.Pagination

fern/products/ask-fern/ask-fern.yml

@@ -13,8 +13,14 @@ navigation:
     contents:
       - page: Citations
         path: ./pages/features/citations.mdx
-      - page: Custom FAQs
-        path: ./pages/features/custom-faqs.mdx
+      - page: Custom Guidance
+        path: ./pages/features/guidance.mdx
+      - page: Custom Documents
+        path: ./pages/features/documents.mdx
+      - page: Insights
+        path: ./pages/features/insights.mdx
+      - page: Evaluation
+        path: ./pages/features/evals.mdx
   - api: API reference
     api-name: fai
     icon: fa-regular fa-pro

fern/products/ask-fern/pages/features/documents.mdx

@@ -0,0 +1,10 @@
+---
+title: Documents
+subtitle: Add arbitrary documents to Ask Fern. 
+---
+
+You can add arbitrary documents to Ask Fern. This is useful for adding additional content that may improve Ask Fern's accuracy 
+(such as help desk tickets, internal company FAQs, etc.). 
+
+Fern offers an internal CMS (content management system) via the `documents` API that allows you to add, update,
+and delete documents as needed. 

fern/products/ask-fern/pages/features/evals.mdx

@@ -0,0 +1,15 @@
+---
+title: Evals
+description: Benchmark Ask Fern's accuracy against your documentation.
+---
+
+<Markdown src="/snippets/wip-callout.mdx" />
+
+Fern evaluates how accurately Ask Fern answers questions on your documentation site compared to competitor AI search solutions. 
+
+Fern first generates evaluation questions from your documentation files, then runs those questions through both Ask Fern and competitor systems. 
+Finally, Fern evaluates each answer against the original documentation for accuracy. You receive detailed performance metrics and comparative analysis.
+
+## Availability
+
+Evals are currently provided as custom analysis delivered directly by the Fern team.

fern/products/ask-fern/pages/features/guidance.mdx

@@ -0,0 +1,11 @@
+---
+title: Custom Guidance
+subtitle: Fine-tune your AI search responses. 
+---
+
+You can add custom guidance (FAQs) to "override" Ask Fern's responses to specific user queries. 
+This is useful for content that you may not wish to display publically, such as billing information, 
+legal terms, and other sensitive content.
+
+Fern offers an internal CMS (content management system) via the `guidance` API that allows you to add,
+update, and delete FAQs as needed. 

fern/products/ask-fern/pages/features/insights.mdx

@@ -0,0 +1,12 @@
+---
+title: Insights
+description: Fern identifies documentation gaps and improvement opportunities from your user queries.
+---
+
+<Markdown src="/snippets/wip-callout.mdx" />
+
+Fern analyzes Ask Fern query patterns to surface actionable recommendations for improving your documentation. After collecting sufficient query data over a month, Fern processes this through specialized models that analyze query-response pairs to identify documentation gaps, extract common themes, and generate specific suggestions for enhancing your docs.
+
+## Availability
+
+Insights are currently provided as a monthly report delivered directly by the Fern team.

fern/products/ask-fern/pages/getting-started/what-is-ask-fern.mdx

@@ -1,19 +1,21 @@
 ---
-title: AI Search Overview
+title: What is Ask Fern?
 subtitle: Let your customers find answers in your documentation instantly. 
 ---
 
+<Note>Ask Fern is available on the [Pro plan](https://buildwithfern.com/pricing#Docs) of Fern Docs. Billing is by usage.</Note>
+
 ## Overview
 
 <Frame caption="AI Search is accessed via the search bar, alongside keyword search.">
 <img src="../assets/search-modal.png" alt="Accessing AI Search" />
 </Frame>
 
-Fern AI Search indexes your documentation and provides an interface for your users to ask questions and get answers. We've found that it helps our customers:
+Ask Fern is Fern's AI Search feature that indexes your documentation and provides an interface for your users to ask questions and get answers. It helps our customers
 
-- **Decrease the time to find needed information** - Help users quickly locate crucial documentation without navigating through a maze of tabs and endpoints.
-- **Integrate your product faster** - Accelerate implementation with ready-to-use code samples that demonstrate practical applications.
-- **Surface where your docs have gaps** - Identify documentation weaknesses through user feedback and search patterns.
+- **Decrease the time to find needed information** – Help users quickly locate crucial documentation without navigating through a maze of tabs and endpoints.
+- **Integrate your product faster** – Accelerate implementation with ready-to-use code samples that demonstrate practical applications.
+- **Surface where your docs have gaps** – Identify documentation weaknesses through user feedback and search patterns.
 
 <Frame caption="Users can ask questions and get support in an intuitive chat interface.">
 <img src="../assets/ask-ai-modal.png" alt="Preview of AI Search" />
@@ -27,7 +29,7 @@ Fern AI Search indexes your documentation and provides an interface for your use
     icon="regular book-open"
     href="/learn/ai-search/custom-prompting"
   >
-    Tailor AI Search results to your users' needs. 
+    Tailor Ask Fern results to your users' needs. 
   </Card>
 
   <Card 
@@ -43,19 +45,52 @@ Fern AI Search indexes your documentation and provides an interface for your use
     icon="regular plug"
     href="/learn/docs/building-and-customizing-your-docs/search" 
     >
-    Offer flexibility to have users 'Ask AI' or search your docs directly with Algolia. 
+    Offer flexibility to have users "Ask AI" or search your docs directly with Algolia. 
   </Card>
 
   <Card 
     title="All-in-one platform" 
     icon="regular fa-layer-group"
     href="/learn/docs/getting-started/overview"
   >
-    Create seamless UX by offering a 'one-stop-shop' for all docs questions. 
+    Create seamless UX by offering a one-stop-shop for all docs questions. 
   </Card>
 
 </CardGroup>
 
-## Pricing
+##  How it works
+
+Ask Fern is a **Retrieval Augmented Generation (RAG)** system built on top of your documentation site that transforms your documentation into an intelligent, searchable knowledge base. The main parts of the Ask Fern system are:
+
+* **Content indexing** – Fern automatically processes your documentation pages,
+  breaking them into semantic chunks and converting each chunk into a vector
+  using sentence embedding models. They're stored in a database that serves as
+  Ask Fern's search index. 
+* **Query processing** – When users ask questions, Ask Fern vectorizes their
+  query and searches the database to find the most relevant documentation
+  chunks.
+* **Response generation** – Ask Fern uses the retrieved chunks as context to
+  generate accurate answers with [citations](/ask-fern/features/citations) for the user. If the initial context isn't sufficient, it performs an additional keyword search. 
+
+### Life of a query
+
+Each Ask Fern user query follows these steps:
 
-Ask Fern is available on the [Pro plan](https://buildwithfern.com/pricing#Docs) of Fern Docs. Billing is by usage.
+```mermaid
+sequenceDiagram
+  autonumber
+    participant U as User
+    participant C as /chat Endpoint
+    participant V as Documentation Database
+    participant A as Ask Fern
+
+    U->>C: Submit question via Ask Fern searchbox
+    C->>C: Convert query to vector
+    C->>V: Search for relevant chunks
+    V->>C: Return matching documents
+    C->>A: Send query + context
+    A->>V: Perform additional keyword search if needed
+    V->>A: Return additional chunks
+    A->>A: Generate response
+    A->>U: Return answer with citations
+```