fix(min-chunk): remove minsize for chunk (#911)

* fix(min-chunk): remove minsize for chunk

* fix tests

Author: icecrasher321

apps/sim/app/api/knowledge/[id]/documents/route.test.ts

@@ -376,7 +376,7 @@ describe('Knowledge Base Documents API Route', () => {
         ],
         processingOptions: {
           chunkSize: 50, // Invalid: too small
-          minCharactersPerChunk: 10, // Invalid: too small
+          minCharactersPerChunk: 0, // Invalid: too small
           recipe: 'default',
           lang: 'en',
           chunkOverlap: 1000, // Invalid: too large

apps/sim/app/api/knowledge/[id]/documents/route.ts

@@ -295,7 +295,7 @@ const BulkCreateDocumentsSchema = z.object({
   documents: z.array(CreateDocumentSchema),
   processingOptions: z.object({
     chunkSize: z.number().min(100).max(4000),
-    minCharactersPerChunk: z.number().min(50).max(2000),
+    minCharactersPerChunk: z.number().min(1).max(2000),
     recipe: z.string(),
     lang: z.string(),
     chunkOverlap: z.number().min(0).max(500),

apps/sim/app/api/knowledge/route.test.ts

@@ -168,7 +168,7 @@ describe('Knowledge Base API Route', () => {
       expect(data.data.embeddingDimension).toBe(1536)
       expect(data.data.chunkingConfig).toEqual({
         maxSize: 1024,
-        minSize: 100,
+        minSize: 1,
         overlap: 200,
       })
     })

apps/sim/app/api/knowledge/route.ts

@@ -18,12 +18,12 @@ const CreateKnowledgeBaseSchema = z.object({
   chunkingConfig: z
     .object({
       maxSize: z.number().min(100).max(4000).default(1024),
-      minSize: z.number().min(50).max(2000).default(100),
+      minSize: z.number().min(1).max(2000).default(1),
       overlap: z.number().min(0).max(500).default(200),
     })
     .default({
       maxSize: 1024,
-      minSize: 100,
+      minSize: 1,
       overlap: 200,
     })
     .refine((data) => data.minSize < data.maxSize, {

apps/sim/app/api/knowledge/utils.ts

@@ -531,7 +531,8 @@ export async function processDocumentAsync(
           docData.filename,
           docData.mimeType,
           processingOptions.chunkSize || 1000,
-          processingOptions.chunkOverlap || 200
+          processingOptions.chunkOverlap || 200,
+          processingOptions.minCharactersPerChunk || 1
         )
 
         const now = new Date()

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/upload-modal/upload-modal.tsx

@@ -142,7 +142,7 @@ export function UploadModal({
     try {
       await uploadFiles(files, knowledgeBaseId, {
         chunkSize: chunkingConfig?.maxSize || 1024,
-        minCharactersPerChunk: chunkingConfig?.minSize || 100,
+        minCharactersPerChunk: chunkingConfig?.minSize || 1,
         chunkOverlap: chunkingConfig?.overlap || 200,
         recipe: 'default',
       })

apps/sim/app/workspace/[workspaceId]/knowledge/components/create-modal/create-modal.tsx

@@ -50,7 +50,7 @@ const FormSchema = z
     description: z.string().max(500, 'Description must be less than 500 characters').optional(),
     minChunkSize: z
       .number()
-      .min(50, 'Min chunk size must be at least 50')
+      .min(1, 'Min chunk size must be at least 1')
       .max(2000, 'Min chunk size must be less than 2000'),
     maxChunkSize: z
       .number()
@@ -115,7 +115,7 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
     defaultValues: {
       name: '',
       description: '',
-      minChunkSize: 100,
+      minChunkSize: 1,
       maxChunkSize: 1024,
       overlapSize: 200,
     },
@@ -299,7 +299,7 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
       reset({
         name: '',
         description: '',
-        minChunkSize: 100,
+        minChunkSize: 1,
         maxChunkSize: 1024,
         overlapSize: 200,
       })
@@ -423,7 +423,7 @@ export function CreateModal({ open, onOpenChange, onKnowledgeBaseCreated }: Crea
                         <Input
                           id='minChunkSize'
                           type='number'
-                          placeholder='100'
+                          placeholder='1'
                           {...register('minChunkSize', { valueAsNumber: true })}
                           className={errors.minChunkSize ? 'border-red-500' : ''}
                           autoComplete='off'

apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts

@@ -278,7 +278,7 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
         })),
         processingOptions: {
           chunkSize: processingOptions.chunkSize || 1024,
-          minCharactersPerChunk: processingOptions.minCharactersPerChunk || 100,
+          minCharactersPerChunk: processingOptions.minCharactersPerChunk || 1,
           chunkOverlap: processingOptions.chunkOverlap || 200,
           recipe: processingOptions.recipe || 'default',
           lang: 'en',

apps/sim/lib/documents/chunker.ts

@@ -57,7 +57,7 @@ export class TextChunker {
 
   constructor(options: ChunkerOptions = {}) {
     this.chunkSize = options.chunkSize ?? 512
-    this.minChunkSize = options.minChunkSize ?? 50
+    this.minChunkSize = options.minChunkSize ?? 1
     this.overlap = options.overlap ?? 0
   }
 

apps/sim/lib/documents/docs-chunker.ts

@@ -25,7 +25,7 @@ export class DocsChunker {
     // Use the existing TextChunker for chunking logic
     this.textChunker = new TextChunker({
       chunkSize: options.chunkSize ?? 300, // Max 300 tokens per chunk
-      minChunkSize: options.minChunkSize ?? 100,
+      minChunkSize: options.minChunkSize ?? 1,
       overlap: options.overlap ?? 50,
     })
     // Use localhost docs in development, production docs otherwise