We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 5e8b051 commit a53f760Copy full SHA for a53f760
apps/sim/lib/chunkers/structured-data-chunker.ts
@@ -1,5 +1,6 @@
1
import { createLogger } from '@sim/logger'
2
import type { Chunk, StructuredDataOptions } from '@/lib/chunkers/types'
3
+
4
/** Structured data is denser in tokens (~3 chars/token vs ~4 for prose) */
5
function estimateStructuredTokens(text: string): number {
6
if (!text?.trim()) return 0
0 commit comments