feat(knowledge): add token, sentence, recursive, and regex chunkers

Author: waleedlatif1

apps/sim/app/api/knowledge/route.ts

@@ -37,6 +37,22 @@ const CreateKnowledgeBaseSchema = z.object({
       minSize: z.number().min(1).max(2000).default(100),
       /** Overlap between chunks in tokens (1 token ≈ 4 characters) */
       overlap: z.number().min(0).max(500).default(200),
+      /** Chunking strategy */
+      strategy: z
+        .enum(['auto', 'text', 'regex', 'recursive', 'sentence', 'token'])
+        .default('auto')
+        .optional(),
+      /** Strategy-specific options */
+      strategyOptions: z
+        .object({
+          /** Regex pattern for 'regex' strategy */
+          pattern: z.string().optional(),
+          /** Custom separator hierarchy for 'recursive' strategy */
+          separators: z.array(z.string()).optional(),
+          /** Pre-built separator recipe for 'recursive' strategy */
+          recipe: z.enum(['plain', 'markdown', 'code']).optional(),
+        })
+        .optional(),
     })
     .default({
       maxSize: 1024,
@@ -45,13 +61,23 @@ const CreateKnowledgeBaseSchema = z.object({
     })
     .refine(
       (data) => {
-        // Convert maxSize from tokens to characters for comparison (1 token ≈ 4 chars)
         const maxSizeInChars = data.maxSize * 4
         return data.minSize < maxSizeInChars
       },
       {
         message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
       }
+    )
+    .refine(
+      (data) => {
+        if (data.strategy === 'regex' && !data.strategyOptions?.pattern) {
+          return false
+        }
+        return true
+      },
+      {
+        message: 'Regex pattern is required when using the regex chunking strategy',
+      }
     ),
 })
 

apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx

@@ -18,6 +18,14 @@ import {
   ModalHeader,
   Textarea,
 } from '@/components/emcn'
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from '@/components/ui/select'
+import type { StrategyOptions } from '@/lib/chunkers/types'
 import { cn } from '@/lib/core/utils/cn'
 import { formatFileSize, validateKnowledgeBaseFile } from '@/lib/uploads/utils/file-utils'
 import { ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation'
@@ -35,6 +43,15 @@ interface CreateBaseModalProps {
   onOpenChange: (open: boolean) => void
 }
 
+const STRATEGY_OPTIONS = [
+  { value: 'auto', label: 'Auto (detect from content)' },
+  { value: 'text', label: 'Text (hierarchical splitting)' },
+  { value: 'recursive', label: 'Recursive (configurable separators)' },
+  { value: 'sentence', label: 'Sentence' },
+  { value: 'token', label: 'Token (fixed-size)' },
+  { value: 'regex', label: 'Regex (custom pattern)' },
+] as const
+
 const FormSchema = z
   .object({
     name: z
@@ -58,10 +75,17 @@ const FormSchema = z
       .number()
       .min(0, 'Overlap must be non-negative')
       .max(500, 'Overlap must be less than 500 tokens'),
+    /** Chunking strategy */
+    strategy: z
+      .enum(['auto', 'text', 'regex', 'recursive', 'sentence', 'token'])
+      .default('auto'),
+    /** Regex pattern (required when strategy is 'regex') */
+    regexPattern: z.string().optional(),
+    /** Custom separators for recursive strategy (comma-separated) */
+    customSeparators: z.string().optional(),
   })
   .refine(
     (data) => {
-      // Convert maxChunkSize from tokens to characters for comparison (1 token ≈ 4 chars)
       const maxChunkSizeInChars = data.maxChunkSize * 4
       return data.minChunkSize < maxChunkSizeInChars
     },
@@ -70,6 +94,18 @@ const FormSchema = z
       path: ['minChunkSize'],
     }
   )
+  .refine(
+    (data) => {
+      if (data.strategy === 'regex' && !data.regexPattern?.trim()) {
+        return false
+      }
+      return true
+    },
+    {
+      message: 'Regex pattern is required when using the regex strategy',
+      path: ['regexPattern'],
+    }
+  )
 
 type FormValues = z.infer<typeof FormSchema>
 
@@ -124,6 +160,7 @@ export const CreateBaseModal = memo(function CreateBaseModal({
     handleSubmit,
     reset,
     watch,
+    setValue,
     formState: { errors },
   } = useForm<FormValues>({
     resolver: zodResolver(FormSchema),
@@ -133,11 +170,15 @@ export const CreateBaseModal = memo(function CreateBaseModal({
       minChunkSize: 100,
       maxChunkSize: 1024,
       overlapSize: 200,
+      strategy: 'auto',
+      regexPattern: '',
+      customSeparators: '',
     },
     mode: 'onSubmit',
   })
 
   const nameValue = watch('name')
+  const strategyValue = watch('strategy')
 
   useEffect(() => {
     if (open) {
@@ -153,6 +194,9 @@ export const CreateBaseModal = memo(function CreateBaseModal({
         minChunkSize: 100,
         maxChunkSize: 1024,
         overlapSize: 200,
+        strategy: 'auto',
+        regexPattern: '',
+        customSeparators: '',
       })
     }
   }, [open, reset])
@@ -255,6 +299,17 @@ export const CreateBaseModal = memo(function CreateBaseModal({
     setSubmitStatus(null)
 
     try {
+      const strategyOptions: StrategyOptions | undefined =
+        data.strategy === 'regex' && data.regexPattern
+          ? { pattern: data.regexPattern }
+          : data.strategy === 'recursive' && data.customSeparators?.trim()
+            ? {
+                separators: data.customSeparators
+                  .split(',')
+                  .map((s) => s.trim().replace(/\\n/g, '\n').replace(/\\t/g, '\t')),
+              }
+            : undefined
+
       const newKnowledgeBase = await createKnowledgeBaseMutation.mutateAsync({
         name: data.name,
         description: data.description || undefined,
@@ -263,6 +318,8 @@ export const CreateBaseModal = memo(function CreateBaseModal({
           maxSize: data.maxChunkSize,
           minSize: data.minChunkSize,
           overlap: data.overlapSize,
+          ...(data.strategy !== 'auto' && { strategy: data.strategy }),
+          ...(strategyOptions && { strategyOptions }),
         },
       })
 
@@ -403,6 +460,69 @@ export const CreateBaseModal = memo(function CreateBaseModal({
                   </p>
                 </div>
 
+                <div className='flex flex-col gap-2'>
+                  <Label htmlFor='strategy'>Chunking Strategy</Label>
+                  <Select
+                    value={strategyValue}
+                    onValueChange={(value) =>
+                      setValue('strategy', value as FormValues['strategy'])
+                    }
+                  >
+                    <SelectTrigger id='strategy'>
+                      <SelectValue placeholder='Auto (detect from content)' />
+                    </SelectTrigger>
+                    <SelectContent>
+                      {STRATEGY_OPTIONS.map((option) => (
+                        <SelectItem key={option.value} value={option.value}>
+                          {option.label}
+                        </SelectItem>
+                      ))}
+                    </SelectContent>
+                  </Select>
+                  <p className='text-[var(--text-muted)] text-xs'>
+                    Auto detects the best strategy based on file content type.
+                  </p>
+                </div>
+
+                {strategyValue === 'regex' && (
+                  <div className='flex flex-col gap-2'>
+                    <Label htmlFor='regexPattern'>Regex Pattern</Label>
+                    <Input
+                      id='regexPattern'
+                      placeholder='e.g. \\n\\n or (?<=\\})\\s*(?=\\{)'
+                      {...register('regexPattern')}
+                      className={cn(errors.regexPattern && 'border-[var(--text-error)]')}
+                      autoComplete='off'
+                      data-form-type='other'
+                    />
+                    {errors.regexPattern && (
+                      <p className='text-[var(--text-error)] text-xs'>
+                        {errors.regexPattern.message}
+                      </p>
+                    )}
+                    <p className='text-[var(--text-muted)] text-xs'>
+                      Text will be split at each match of this regex pattern.
+                    </p>
+                  </div>
+                )}
+
+                {strategyValue === 'recursive' && (
+                  <div className='flex flex-col gap-2'>
+                    <Label htmlFor='customSeparators'>Custom Separators (optional)</Label>
+                    <Input
+                      id='customSeparators'
+                      placeholder='e.g. \n\n, \n, . ,  '
+                      {...register('customSeparators')}
+                      autoComplete='off'
+                      data-form-type='other'
+                    />
+                    <p className='text-[var(--text-muted)] text-xs'>
+                      Comma-separated list of delimiters in priority order. Leave empty for default
+                      separators.
+                    </p>
+                  </div>
+                )}
+
                 <div className='flex flex-col gap-2'>
                   <Label>Upload Documents</Label>
                   <Button

apps/sim/hooks/queries/kb/knowledge.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@sim/logger'
 import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
 import { toast } from '@/components/emcn'
+import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
 import type {
   ChunkData,
   ChunksPagination,
@@ -707,6 +708,8 @@ export interface CreateKnowledgeBaseParams {
     maxSize: number
     minSize: number
     overlap: number
+    strategy?: ChunkingStrategy
+    strategyOptions?: StrategyOptions
   }
 }
 

apps/sim/lib/chunkers/index.ts

@@ -1,5 +1,9 @@
 export { DocsChunker } from './docs-chunker'
 export { JsonYamlChunker } from './json-yaml-chunker'
+export { RecursiveChunker } from './recursive-chunker'
+export { RegexChunker } from './regex-chunker'
+export { SentenceChunker } from './sentence-chunker'
 export { StructuredDataChunker } from './structured-data-chunker'
 export { TextChunker } from './text-chunker'
+export { TokenChunker } from './token-chunker'
 export * from './types'