improvement(kb): improve chunkers, respect user-specified chunk configurations, added tests (#2539)

* improvement(kb): improve chunkers, respect user-specified chunk configurations, added tests

* ack PR commnets

* updated docs

* cleanup

Author: waleedlatif1

apps/docs/content/docs/en/knowledgebase/index.mdx

@@ -34,9 +34,15 @@ Once your documents are processed, you can view and edit the individual chunks.
 <Image src="/static/knowledgebase/knowledgebase.png" alt="Document chunks view showing processed content" width={800} height={500} />
 
 ### Chunk Configuration
-- **Default chunk size**: 1,024 characters
-- **Configurable range**: 100-4,000 characters per chunk
-- **Smart overlap**: 200 characters by default for context preservation
+
+When creating a knowledge base, you can configure how documents are split into chunks:
+
+| Setting | Unit | Default | Range | Description |
+|---------|------|---------|-------|-------------|
+| **Max Chunk Size** | tokens | 1,024 | 100-4,000 | Maximum size of each chunk (1 token ≈ 4 characters) |
+| **Min Chunk Size** | characters | 1 | 1-2,000 | Minimum chunk size to avoid tiny fragments |
+| **Overlap** | characters | 200 | 0-500 | Context overlap between consecutive chunks |
+
 - **Hierarchical splitting**: Respects document structure (sections, paragraphs, sentences)
 
 ### Editing Capabilities

apps/sim/app/api/knowledge/[id]/documents/route.ts

@@ -34,13 +34,24 @@ const CreateDocumentSchema = z.object({
   documentTagsData: z.string().optional(),
 })
 
+/**
+ * Schema for bulk document creation with processing options
+ *
+ * Processing options units:
+ * - chunkSize: tokens (1 token ≈ 4 characters)
+ * - minCharactersPerChunk: characters
+ * - chunkOverlap: characters
+ */
 const BulkCreateDocumentsSchema = z.object({
   documents: z.array(CreateDocumentSchema),
   processingOptions: z.object({
+    /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
     chunkSize: z.number().min(100).max(4000),
+    /** Minimum chunk size in characters */
     minCharactersPerChunk: z.number().min(1).max(2000),
     recipe: z.string(),
     lang: z.string(),
+    /** Overlap between chunks in characters */
     chunkOverlap: z.number().min(0).max(500),
   }),
   bulk: z.literal(true),

apps/sim/app/api/knowledge/[id]/route.ts

@@ -12,6 +12,14 @@ import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/a
 
 const logger = createLogger('KnowledgeBaseByIdAPI')
 
+/**
+ * Schema for updating a knowledge base
+ *
+ * Chunking config units:
+ * - maxSize: tokens (1 token ≈ 4 characters)
+ * - minSize: characters
+ * - overlap: tokens (1 token ≈ 4 characters)
+ */
 const UpdateKnowledgeBaseSchema = z.object({
   name: z.string().min(1, 'Name is required').optional(),
   description: z.string().optional(),
@@ -20,10 +28,23 @@ const UpdateKnowledgeBaseSchema = z.object({
   workspaceId: z.string().nullable().optional(),
   chunkingConfig: z
     .object({
-      maxSize: z.number(),
-      minSize: z.number(),
-      overlap: z.number(),
+      /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
+      maxSize: z.number().min(100).max(4000),
+      /** Minimum chunk size in characters */
+      minSize: z.number().min(1).max(2000),
+      /** Overlap between chunks in characters */
+      overlap: z.number().min(0).max(500),
     })
+    .refine(
+      (data) => {
+        // Convert maxSize from tokens to characters for comparison (1 token ≈ 4 chars)
+        const maxSizeInChars = data.maxSize * 4
+        return data.minSize < maxSizeInChars
+      },
+      {
+        message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
+      }
+    )
     .optional(),
 })
 

apps/sim/app/api/knowledge/route.test.ts

@@ -139,8 +139,8 @@ describe('Knowledge Base API Route', () => {
       const invalidData = {
         name: 'Test KB',
         chunkingConfig: {
-          maxSize: 100,
-          minSize: 200, // Invalid: minSize > maxSize
+          maxSize: 100, // 100 tokens = 400 characters
+          minSize: 500, // Invalid: minSize (500 chars) > maxSize (400 chars)
           overlap: 50,
         },
       }
@@ -168,7 +168,7 @@ describe('Knowledge Base API Route', () => {
       expect(data.data.embeddingDimension).toBe(1536)
       expect(data.data.chunkingConfig).toEqual({
         maxSize: 1024,
-        minSize: 1,
+        minSize: 100,
         overlap: 200,
       })
     })

apps/sim/app/api/knowledge/route.ts

@@ -7,6 +7,14 @@ import { createLogger } from '@/lib/logs/console/logger'
 
 const logger = createLogger('KnowledgeBaseAPI')
 
+/**
+ * Schema for creating a knowledge base
+ *
+ * Chunking config units:
+ * - maxSize: tokens (1 token ≈ 4 characters)
+ * - minSize: characters
+ * - overlap: tokens (1 token ≈ 4 characters)
+ */
 const CreateKnowledgeBaseSchema = z.object({
   name: z.string().min(1, 'Name is required'),
   description: z.string().optional(),
@@ -15,18 +23,28 @@ const CreateKnowledgeBaseSchema = z.object({
   embeddingDimension: z.literal(1536).default(1536),
   chunkingConfig: z
     .object({
+      /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
       maxSize: z.number().min(100).max(4000).default(1024),
-      minSize: z.number().min(1).max(2000).default(1),
+      /** Minimum chunk size in characters */
+      minSize: z.number().min(1).max(2000).default(100),
+      /** Overlap between chunks in tokens (1 token ≈ 4 characters) */
       overlap: z.number().min(0).max(500).default(200),
     })
     .default({
       maxSize: 1024,
-      minSize: 1,
+      minSize: 100,
       overlap: 200,
     })
-    .refine((data) => data.minSize < data.maxSize, {
-      message: 'Min chunk size must be less than max chunk size',
-    }),
+    .refine(
+      (data) => {
+        // Convert maxSize from tokens to characters for comparison (1 token ≈ 4 chars)
+        const maxSizeInChars = data.maxSize * 4
+        return data.minSize < maxSizeInChars
+      },
+      {
+        message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
+      }
+    ),
 })
 
 export async function GET(req: NextRequest) {

apps/sim/app/api/knowledge/utils.test.ts

@@ -183,7 +183,12 @@ describe('Knowledge Utils', () => {
 
   describe('processDocumentAsync', () => {
     it.concurrent('should insert embeddings before updating document counters', async () => {
-      kbRows.push({ id: 'kb1', userId: 'user1', workspaceId: null })
+      kbRows.push({
+        id: 'kb1',
+        userId: 'user1',
+        workspaceId: null,
+        chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 200 },
+      })
       docRows.push({ id: 'doc1', knowledgeBaseId: 'kb1' })
 
       await processDocumentAsync(

apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx

@@ -44,23 +44,33 @@ const FormSchema = z
       .max(100, 'Name must be less than 100 characters')
       .refine((value) => value.trim().length > 0, 'Name cannot be empty'),
     description: z.string().max(500, 'Description must be less than 500 characters').optional(),
+    /** Minimum chunk size in characters */
     minChunkSize: z
       .number()
-      .min(1, 'Min chunk size must be at least 1')
-      .max(2000, 'Min chunk size must be less than 2000'),
+      .min(1, 'Min chunk size must be at least 1 character')
+      .max(2000, 'Min chunk size must be less than 2000 characters'),
+    /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
     maxChunkSize: z
       .number()
-      .min(100, 'Max chunk size must be at least 100')
-      .max(4000, 'Max chunk size must be less than 4000'),
+      .min(100, 'Max chunk size must be at least 100 tokens')
+      .max(4000, 'Max chunk size must be less than 4000 tokens'),
+    /** Overlap between chunks in tokens */
     overlapSize: z
       .number()
-      .min(0, 'Overlap size must be non-negative')
-      .max(500, 'Overlap size must be less than 500'),
-  })
-  .refine((data) => data.minChunkSize < data.maxChunkSize, {
-    message: 'Min chunk size must be less than max chunk size',
-    path: ['minChunkSize'],
+      .min(0, 'Overlap must be non-negative')
+      .max(500, 'Overlap must be less than 500 tokens'),
   })
+  .refine(
+    (data) => {
+      // Convert maxChunkSize from tokens to characters for comparison (1 token ≈ 4 chars)
+      const maxChunkSizeInChars = data.maxChunkSize * 4
+      return data.minChunkSize < maxChunkSizeInChars
+    },
+    {
+      message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
+      path: ['minChunkSize'],
+    }
+  )
 
 type FormValues = z.infer<typeof FormSchema>
 
@@ -123,7 +133,7 @@ export function CreateBaseModal({
     defaultValues: {
       name: '',
       description: '',
-      minChunkSize: 1,
+      minChunkSize: 100,
       maxChunkSize: 1024,
       overlapSize: 200,
     },
@@ -143,7 +153,7 @@ export function CreateBaseModal({
       reset({
         name: '',
         description: '',
-        minChunkSize: 1,
+        minChunkSize: 100,
         maxChunkSize: 1024,
         overlapSize: 200,
       })
@@ -381,10 +391,10 @@ export function CreateBaseModal({
                 <div className='space-y-[12px] rounded-[6px] bg-[var(--surface-6)] px-[12px] py-[14px]'>
                   <div className='grid grid-cols-2 gap-[12px]'>
                     <div className='flex flex-col gap-[8px]'>
-                      <Label htmlFor='minChunkSize'>Min Chunk Size</Label>
+                      <Label htmlFor='minChunkSize'>Min Chunk Size (characters)</Label>
                       <Input
                         id='minChunkSize'
-                        placeholder='1'
+                        placeholder='100'
                         {...register('minChunkSize', { valueAsNumber: true })}
                         className={cn(errors.minChunkSize && 'border-[var(--text-error)]')}
                         autoComplete='off'
@@ -394,7 +404,7 @@ export function CreateBaseModal({
                     </div>
 
                     <div className='flex flex-col gap-[8px]'>
-                      <Label htmlFor='maxChunkSize'>Max Chunk Size</Label>
+                      <Label htmlFor='maxChunkSize'>Max Chunk Size (tokens)</Label>
                       <Input
                         id='maxChunkSize'
                         placeholder='1024'
@@ -408,7 +418,7 @@ export function CreateBaseModal({
                   </div>
 
                   <div className='flex flex-col gap-[8px]'>
-                    <Label htmlFor='overlapSize'>Overlap Size</Label>
+                    <Label htmlFor='overlapSize'>Overlap (tokens)</Label>
                     <Input
                       id='overlapSize'
                       placeholder='200'
@@ -419,6 +429,9 @@ export function CreateBaseModal({
                       name='overlap-size'
                     />
                   </div>
+                  <p className='text-[11px] text-[var(--text-muted)]'>
+                    1 token ≈ 4 characters. Max chunk size and overlap are in tokens.
+                  </p>
                 </div>
 
                 <div className='flex flex-col gap-[8px]'>

apps/sim/lib/chunkers/docs-chunker.ts

@@ -32,8 +32,8 @@ export class DocsChunker {
     // Use the existing TextChunker for chunking logic
     this.textChunker = new TextChunker({
       chunkSize: options.chunkSize ?? 300, // Max 300 tokens per chunk
-      minChunkSize: options.minChunkSize ?? 1,
-      overlap: options.overlap ?? 50,
+      minCharactersPerChunk: options.minCharactersPerChunk ?? 1,
+      chunkOverlap: options.chunkOverlap ?? 50,
     })
     // Use localhost docs in development, production docs otherwise
     this.baseUrl = options.baseUrl ?? 'https://docs.sim.ai'

apps/sim/lib/chunkers/json-yaml-chunker.ts

@@ -21,19 +21,20 @@ function getTokenCount(text: string): number {
  * Reduced limits to ensure we stay well under OpenAI's 8,191 token limit per embedding request
  */
 const JSON_YAML_CHUNKING_CONFIG = {
-  TARGET_CHUNK_SIZE: 1000, // Target tokens per chunk
-  MIN_CHUNK_SIZE: 100, // Minimum tokens per chunk
+  TARGET_CHUNK_SIZE: 1024, // Target tokens per chunk
+  MIN_CHARACTERS_PER_CHUNK: 100, // Minimum characters per chunk to filter tiny fragments
   MAX_CHUNK_SIZE: 1500, // Maximum tokens per chunk
   MAX_DEPTH_FOR_SPLITTING: 5, // Maximum depth to traverse for splitting
 }
 
 export class JsonYamlChunker {
-  private chunkSize: number
-  private minChunkSize: number
+  private chunkSize: number // in tokens
+  private minCharactersPerChunk: number // in characters
 
   constructor(options: ChunkerOptions = {}) {
-    this.chunkSize = options.chunkSize || JSON_YAML_CHUNKING_CONFIG.TARGET_CHUNK_SIZE
-    this.minChunkSize = options.minChunkSize || JSON_YAML_CHUNKING_CONFIG.MIN_CHUNK_SIZE
+    this.chunkSize = options.chunkSize ?? JSON_YAML_CHUNKING_CONFIG.TARGET_CHUNK_SIZE
+    this.minCharactersPerChunk =
+      options.minCharactersPerChunk ?? JSON_YAML_CHUNKING_CONFIG.MIN_CHARACTERS_PER_CHUNK
   }
 
   /**
@@ -99,7 +100,8 @@ export class JsonYamlChunker {
     const content = JSON.stringify(data, null, 2)
     const tokenCount = getTokenCount(content)
 
-    if (tokenCount >= this.minChunkSize) {
+    // Filter tiny fragments using character count
+    if (content.length >= this.minCharactersPerChunk) {
       chunks.push({
         text: content,
         tokenCount,
@@ -318,7 +320,8 @@ export class JsonYamlChunker {
       }
     }
 
-    if (currentChunk && currentTokens >= this.minChunkSize) {
+    // Filter tiny fragments using character count
+    if (currentChunk && currentChunk.length >= this.minCharactersPerChunk) {
       chunks.push({
         text: currentChunk,
         tokenCount: currentTokens,