feat(fai): add chunk_count to content hash table (#5819)

Author: sahil485

fern/apis/fai/openapi.json

@@ -4701,6 +4701,10 @@
       },
       "ContentHashEntry": {
         "properties": {
+          "domain": {
+            "type": "string",
+            "title": "Domain"
+          },
           "parent_id": {
             "type": "string",
             "title": "Parent Id"
@@ -4709,6 +4713,10 @@
             "type": "string",
             "title": "Content Hash"
           },
+          "chunk_count": {
+            "type": "integer",
+            "title": "Chunk Count"
+          },
           "indexed_at": {
             "anyOf": [
               {
@@ -4723,8 +4731,10 @@
         },
         "type": "object",
         "required": [
+          "domain",
           "parent_id",
-          "content_hash"
+          "content_hash",
+          "chunk_count"
         ],
         "title": "ContentHashEntry",
         "description": "A single content hash entry."
@@ -4738,12 +4748,17 @@
           "content_hash": {
             "type": "string",
             "title": "Content Hash"
+          },
+          "chunk_count": {
+            "type": "integer",
+            "title": "Chunk Count"
           }
         },
         "type": "object",
         "required": [
           "parent_id",
-          "content_hash"
+          "content_hash",
+          "chunk_count"
         ],
         "title": "ContentHashUpsertEntry",
         "description": "Entry for upserting a content hash (without timestamps)."

packages/fai-sdk/src/api/resources/contentHash/client/Client.ts

@@ -140,7 +140,8 @@ export class ContentHash {
      *     await client.contentHash.batchUpsertContentHashes("domain", {
      *         entries: [{
      *                 parent_id: "parent_id",
-     *                 content_hash: "content_hash"
+     *                 content_hash: "content_hash",
+     *                 chunk_count: 1
      *             }]
      *     })
      */

packages/fai-sdk/src/api/resources/contentHash/client/requests/BatchUpsertContentHashesRequest.ts

@@ -9,7 +9,8 @@ import * as FernAI from "../../../../index.js";
  *     {
  *         entries: [{
  *                 parent_id: "parent_id",
- *                 content_hash: "content_hash"
+ *                 content_hash: "content_hash",
+ *                 chunk_count: 1
  *             }]
  *     }
  */

packages/fai-sdk/src/api/types/ContentHashEntry.ts

@@ -6,7 +6,9 @@
  * A single content hash entry.
  */
 export interface ContentHashEntry {
+    domain: string;
     parent_id: string;
     content_hash: string;
+    chunk_count: number;
     indexed_at?: string;
 }

packages/fai-sdk/src/api/types/ContentHashUpsertEntry.ts

@@ -8,4 +8,5 @@
 export interface ContentHashUpsertEntry {
     parent_id: string;
     content_hash: string;
+    chunk_count: number;
 }

servers/fai/alembic/versions/bef764295d1b_add_chunk_count_to_content_hashes.py

@@ -0,0 +1,28 @@
+"""add_chunk_count_to_content_hashes
+
+Revision ID: bef764295d1b
+Revises: create_content_hashes_table
+Create Date: 2025-12-10 16:52:03.487012
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'bef764295d1b'
+down_revision: Union[str, Sequence[str], None] = 'create_content_hashes_table'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    op.add_column('content_hashes', sa.Column('chunk_count', sa.Integer(), nullable=False, server_default='0', comment='Number of chunks created during indexing'))
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    op.drop_column('content_hashes', 'chunk_count')

servers/fai/src/fai/models/api/content_hash_api.py

@@ -6,8 +6,10 @@
 class ContentHashEntry(BaseModel):
     """A single content hash entry."""
 
+    domain: str
     parent_id: str
     content_hash: str
+    chunk_count: int
     indexed_at: str | None = None
 
 
@@ -16,6 +18,7 @@ class ContentHashUpsertEntry(BaseModel):
 
     parent_id: str
     content_hash: str
+    chunk_count: int
 
 
 class BatchGetContentHashesRequest(BaseModel):

servers/fai/src/fai/models/db/content_hash_db.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import Column, DateTime, Index, String
+from sqlalchemy import Column, DateTime, Index, Integer, String
 
 from fai.models.base import Base
 
@@ -17,6 +17,7 @@ class ContentHashDb(Base):
     parent_id = Column(String, primary_key=True, nullable=False, comment="FDR PageId or EndpointId")
 
     content_hash = Column(String, nullable=False, comment="SHA-256 hash of page markdown or endpoint document content")
+    chunk_count = Column(Integer, nullable=False, comment="Number of chunks created during indexing")
 
     # Metadata
     indexed_at = Column(DateTime, nullable=False, default=datetime.utcnow)

servers/fai/src/fai/routes/content_hash.py

@@ -57,8 +57,10 @@ async def batch_get_content_hashes(
 
         entries = [
             ContentHashEntry(
+                domain=h.domain,
                 parent_id=h.parent_id,
                 content_hash=h.content_hash,
+                chunk_count=h.chunk_count,
                 indexed_at=h.indexed_at.isoformat(),
             )
             for h in hashes
@@ -99,9 +101,15 @@ async def batch_upsert_content_hashes(
 
             if existing:
                 existing.content_hash = entry.content_hash
+                existing.chunk_count = entry.chunk_count
                 existing.updated_at = datetime.utcnow()
             else:
-                new_hash = ContentHashDb(domain=domain, parent_id=entry.parent_id, content_hash=entry.content_hash)
+                new_hash = ContentHashDb(
+                    domain=domain,
+                    parent_id=entry.parent_id,
+                    content_hash=entry.content_hash,
+                    chunk_count=entry.chunk_count,
+                )
                 db.add(new_hash)
 
             upserted_count += 1

servers/fai/tests/routes/test_content_hash.py

@@ -29,9 +29,9 @@ async def test_get_all_hashes_for_domain(self, test_client: TestClient, test_ses
         domain = "test-domain"
 
         # Insert test data
-        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1")
-        hash2 = ContentHashDb(domain=domain, parent_id="page-2", content_hash="hash2")
-        hash3 = ContentHashDb(domain="other-domain", parent_id="page-3", content_hash="hash3")
+        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1", chunk_count=5)
+        hash2 = ContentHashDb(domain=domain, parent_id="page-2", content_hash="hash2", chunk_count=10)
+        hash3 = ContentHashDb(domain="other-domain", parent_id="page-3", content_hash="hash3", chunk_count=15)
         test_session.add_all([hash1, hash2, hash3])
         await test_session.commit()
 
@@ -52,9 +52,9 @@ async def test_get_specific_hashes(self, test_client: TestClient, test_session:
         domain = "test-domain"
 
         # Insert test data
-        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1")
-        hash2 = ContentHashDb(domain=domain, parent_id="page-2", content_hash="hash2")
-        hash3 = ContentHashDb(domain=domain, parent_id="page-3", content_hash="hash3")
+        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1", chunk_count=5)
+        hash2 = ContentHashDb(domain=domain, parent_id="page-2", content_hash="hash2", chunk_count=10)
+        hash3 = ContentHashDb(domain=domain, parent_id="page-3", content_hash="hash3", chunk_count=15)
         test_session.add_all([hash1, hash2, hash3])
         await test_session.commit()
 
@@ -94,8 +94,8 @@ async def test_upsert_new_hashes(self, test_client: TestClient, test_session: As
             f"/content-hash/{domain}/batch-upsert",
             json={
                 "entries": [
-                    {"parent_id": "page-1", "content_hash": "hash1"},
-                    {"parent_id": "page-2", "content_hash": "hash2"},
+                    {"parent_id": "page-1", "content_hash": "hash1", "chunk_count": 5},
+                    {"parent_id": "page-2", "content_hash": "hash2", "chunk_count": 10},
                 ]
             },
         )
@@ -119,14 +119,14 @@ async def test_upsert_update_existing_hashes(self, test_client: TestClient, test
         domain = "test-domain"
 
         # Insert initial hash
-        initial = ContentHashDb(domain=domain, parent_id="page-1", content_hash="old-hash")
+        initial = ContentHashDb(domain=domain, parent_id="page-1", content_hash="old-hash", chunk_count=5)
         test_session.add(initial)
         await test_session.commit()
 
         # Update with new hash
         response = test_client.post(
             f"/content-hash/{domain}/batch-upsert",
-            json={"entries": [{"parent_id": "page-1", "content_hash": "new-hash"}]},
+            json={"entries": [{"parent_id": "page-1", "content_hash": "new-hash", "chunk_count": 10}]},
         )
 
         assert response.status_code == 200
@@ -146,7 +146,7 @@ async def test_upsert_mixed(self, test_client: TestClient, test_session: AsyncSe
         domain = "test-domain"
 
         # Insert existing hash
-        existing = ContentHashDb(domain=domain, parent_id="page-1", content_hash="old-hash")
+        existing = ContentHashDb(domain=domain, parent_id="page-1", content_hash="old-hash", chunk_count=5)
         test_session.add(existing)
         await test_session.commit()
 
@@ -155,8 +155,8 @@ async def test_upsert_mixed(self, test_client: TestClient, test_session: AsyncSe
             f"/content-hash/{domain}/batch-upsert",
             json={
                 "entries": [
-                    {"parent_id": "page-1", "content_hash": "updated-hash"},
-                    {"parent_id": "page-2", "content_hash": "new-hash"},
+                    {"parent_id": "page-1", "content_hash": "updated-hash", "chunk_count": 10},
+                    {"parent_id": "page-2", "content_hash": "new-hash", "chunk_count": 15},
                 ]
             },
         )
@@ -184,9 +184,9 @@ async def test_delete_hashes(self, test_client: TestClient, test_session: AsyncS
         domain = "test-domain"
 
         # Insert test hashes
-        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1")
-        hash2 = ContentHashDb(domain=domain, parent_id="page-2", content_hash="hash2")
-        hash3 = ContentHashDb(domain=domain, parent_id="page-3", content_hash="hash3")
+        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1", chunk_count=5)
+        hash2 = ContentHashDb(domain=domain, parent_id="page-2", content_hash="hash2", chunk_count=10)
+        hash3 = ContentHashDb(domain=domain, parent_id="page-3", content_hash="hash3", chunk_count=15)
         test_session.add_all([hash1, hash2, hash3])
         await test_session.commit()
 
@@ -215,7 +215,7 @@ async def test_delete_empty_list(self, test_client: TestClient, test_session: As
         domain = "test-domain"
 
         # Insert test hash
-        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1")
+        hash1 = ContentHashDb(domain=domain, parent_id="page-1", content_hash="hash1", chunk_count=5)
         test_session.add(hash1)
         await test_session.commit()