diff --git a/.github/workflows/orama_sync.yml b/.github/workflows/orama_sync.yml index 30d83ee15e..f8715f8533 100644 --- a/.github/workflows/orama_sync.yml +++ b/.github/workflows/orama_sync.yml @@ -32,4 +32,5 @@ jobs: run: pnpm sync:orama env: ORAMA_PRIVATE_API_KEY: ${{ secrets.ORAMA_PRIVATE_API_KEY }} - ORAMA_PRIVATE_INDEX_ID: ${{ secrets.ORAMA_PRIVATE_INDEX_ID }} + ORAMA_DATASOURCE_ID: ${{ secrets.ORAMA_DATASOURCE_ID }} + ORAMA_PROJECT_ID: ${{ secrets.ORAMA_PROJECT_ID }} diff --git a/env.d.ts b/env.d.ts index df07a81008..eaa53030d6 100644 --- a/env.d.ts +++ b/env.d.ts @@ -10,8 +10,10 @@ interface ImportMeta { declare namespace NodeJS { interface ProcessEnv { + readonly ORAMA_PROJECT_ID: string; + readonly ORAMA_DATASOURCE_ID: string; + readonly ORAMA_PUBLIC_API_KEY: string; readonly ORAMA_PRIVATE_API_KEY: string; - readonly ORAMA_PRIVATE_INDEX_ID: string; } } diff --git a/package.json b/package.json index 57c04a8ed2..9613cd305c 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "dependencies": { "@kobalte/core": "^0.13.11", "@kobalte/solidbase": "^0.2.20", - "@oramacloud/client": "^2.1.4", + "@orama/core": "^1.2.14", "@solid-primitives/event-listener": "^2.4.3", "@solid-primitives/marker": "^0.2.2", "@solid-primitives/media": "^2.3.3", @@ -62,4 +62,4 @@ "node": ">=24", "pnpm": ">=10" } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 67e7721f5a..2398200099 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,9 +14,9 @@ importers: '@kobalte/solidbase': specifier: ^0.2.20 version: 0.2.20(@solidjs/start@1.2.0(solid-js@1.9.10)(vinxi@0.5.7(@types/node@24.10.1)(db0@0.3.2)(ioredis@5.6.1)(jiti@1.21.7)(terser@5.42.0)(yaml@2.8.1))(vite@6.3.5(@types/node@24.10.1)(jiti@1.21.7)(terser@5.42.0)(yaml@2.8.1)))(@vue/compiler-sfc@3.5.16)(solid-js@1.9.10)(vinxi@0.5.7(@types/node@24.10.1)(db0@0.3.2)(ioredis@5.6.1)(jiti@1.21.7)(terser@5.42.0)(yaml@2.8.1))(vite@6.3.5(@types/node@24.10.1)(jiti@1.21.7)(terser@5.42.0)(yaml@2.8.1)) - '@oramacloud/client': - specifier: ^2.1.4 - version: 2.1.4 + '@orama/core': + specifier: ^1.2.14 + version: 1.2.14 '@solid-primitives/event-listener': specifier: ^2.4.3 version: 2.4.3(solid-js@1.9.10) @@ -799,18 +799,17 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@orama/core@1.2.14': + resolution: {integrity: sha512-+D6PXdztYM9j2Z115p7wEwTIVG/yN03jsMpra12NZ2d0GNQFSKLd5P8C+5N+9hKyjx9J47GVUwr9guxnb800yg==} + '@orama/crawly@0.0.6': resolution: {integrity: sha512-8u0pv9IvKrYaO9gbOe7hcZ757Kd6y6qqyN5uPXXPJmAk0nXc2p172YsZEp8NzanZgvcH6G6t1XsG74t7Mptchw==} '@orama/cuid2@2.2.3': resolution: {integrity: sha512-Lcak3chblMejdlSHgYU2lS2cdOhDpU6vkfIJH4m+YKvqQyLqs1bB8+w6NT1MG5bO12NUK2GFc34Mn2xshMIQ1g==} - '@orama/orama@3.1.7': - resolution: {integrity: sha512-6yB0117ZjsgNevZw3LP+bkrZa9mU/POPVaXgzMPOBbBc35w2P3R+1vMMhEfC06kYCpd5bf0jodBaTkYQW5TVeQ==} - engines: {node: '>= 20.0.0'} - - '@oramacloud/client@2.1.4': - resolution: {integrity: sha512-uNPFs4wq/iOPbggCwTkVNbIr64Vfd7ZS/h+cricXVnzXWocjDTfJ3wLL4lr0qiSu41g8z+eCAGBqJ30RO2O4AA==} + '@orama/oramacore-events-parser@0.0.5': + resolution: {integrity: sha512-yAuSwog+HQBAXgZ60TNKEwu04y81/09mpbYBCmz1RCxnr4ObNY2JnPZI7HmALbjAhLJ8t5p+wc2JHRK93ubO4w==} '@parcel/watcher-android-arm64@2.4.1': resolution: {integrity: sha512-LOi/WTbbh3aTn2RYddrO8pnapixAziFl6SMxHM69r3tvdSm94JtCenaKgk1GRg5FJ5wpMCpHeW+7yqPlvZv7kg==} @@ -1720,10 +1719,6 @@ packages: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} - ansi-regex@6.2.0: - resolution: {integrity: sha512-TKY5pyBkHyADOPYlRT9Lx6F544mPl0vS5Ew7BJ45hA08Q+t3GjbueLliBWN3sMICk6+y7HdyxSzC4bWS8baBdg==} - engines: {node: '>=12'} - ansi-regex@6.2.2: resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} engines: {node: '>=12'} @@ -4465,10 +4460,6 @@ packages: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} - strip-ansi@7.1.0: - resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} - engines: {node: '>=12'} - strip-ansi@7.1.2: resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} engines: {node: '>=12'} @@ -5136,6 +5127,14 @@ packages: resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} engines: {node: '>= 14'} + zod-to-json-schema@3.24.5: + resolution: {integrity: sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==} + peerDependencies: + zod: ^3.24.1 + + zod@3.24.3: + resolution: {integrity: sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==} + zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} @@ -6052,6 +6051,13 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@orama/core@1.2.14': + dependencies: + '@orama/cuid2': 2.2.3 + '@orama/oramacore-events-parser': 0.0.5 + zod: 3.24.3 + zod-to-json-schema: 3.24.5(zod@3.24.3) + '@orama/crawly@0.0.6': dependencies: cheerio: 1.0.0-rc.12 @@ -6061,13 +6067,7 @@ snapshots: dependencies: '@noble/hashes': 1.8.0 - '@orama/orama@3.1.7': {} - - '@oramacloud/client@2.1.4': - dependencies: - '@orama/cuid2': 2.2.3 - '@orama/orama': 3.1.7 - lodash: 4.17.21 + '@orama/oramacore-events-parser@0.0.5': {} '@parcel/watcher-android-arm64@2.4.1': optional: true @@ -7115,8 +7115,6 @@ snapshots: ansi-regex@5.0.1: {} - ansi-regex@6.2.0: {} - ansi-regex@6.2.2: {} ansi-styles@3.2.1: @@ -10397,7 +10395,7 @@ snapshots: dependencies: emoji-regex: 10.4.0 get-east-asian-width: 1.3.0 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 string_decoder@1.1.1: dependencies: @@ -10416,10 +10414,6 @@ snapshots: dependencies: ansi-regex: 5.0.1 - strip-ansi@7.1.0: - dependencies: - ansi-regex: 6.2.0 - strip-ansi@7.1.2: dependencies: ansi-regex: 6.2.2 @@ -11127,7 +11121,7 @@ snapshots: dependencies: ansi-styles: 6.2.3 string-width: 7.2.0 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 wrappy@1.0.2: {} @@ -11197,6 +11191,12 @@ snapshots: compress-commons: 6.0.2 readable-stream: 4.7.0 + zod-to-json-schema@3.24.5(zod@3.24.3): + dependencies: + zod: 3.24.3 + + zod@3.24.3: {} + zod@3.25.76: {} zod@4.1.12: {} diff --git a/public/llms.txt b/public/llms.txt index 59c510b3f7..f0cd0ce392 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -128,7 +128,8 @@ - [Routing](https://docs.solidjs.com/solid-start/building-your-application/routing) - [API routes](https://docs.solidjs.com/solid-start/building-your-application/api-routes) - [CSS and styling](https://docs.solidjs.com/solid-start/building-your-application/css-and-styling) -- [Data loading](https://docs.solidjs.com/solid-start/building-your-application/data-loading) +- [Data fetching](https://docs.solidjs.com/solid-start/building-your-application/data-fetching) +- [Data mutation](https://docs.solidjs.com/solid-start/building-your-application/data-mutation) - [Head and metadata](https://docs.solidjs.com/solid-start/building-your-application/head-and-metadata) - [Route Pre-rendering](https://docs.solidjs.com/solid-start/building-your-application/route-prerendering) - [Static assets](https://docs.solidjs.com/solid-start/building-your-application/static-assets) @@ -176,6 +177,11 @@ - [Actions](https://docs.solidjs.com/solid-router/concepts/actions) - [Single page applications](https://docs.solidjs.com/solid-router/rendering-modes/spa) - [Server side rendering](https://docs.solidjs.com/solid-router/rendering-modes/ssr) +- [Queries](https://docs.solidjs.com/solid-router/data-fetching/queries) +- [Streaming](https://docs.solidjs.com/solid-router/data-fetching/streaming) +- [Revalidation](https://docs.solidjs.com/solid-router/data-fetching/revalidation) +- [Preload data](https://docs.solidjs.com/solid-router/data-fetching/how-to/preload-data) +- [Handle pending and error states](https://docs.solidjs.com/solid-router/data-fetching/how-to/handle-error-and-loading-states) - [Lazy loading](https://docs.solidjs.com/solid-router/advanced-concepts/lazy-loading) - [Migration from v0.9.x](https://docs.solidjs.com/solid-router/guides/migration) - [A](https://docs.solidjs.com/solid-router/reference/components/a) @@ -193,7 +199,7 @@ - [useAction](https://docs.solidjs.com/solid-router/reference/data-apis/use-action) - [useSubmission](https://docs.solidjs.com/solid-router/reference/data-apis/use-submission) - [useSubmissions](https://docs.solidjs.com/solid-router/reference/data-apis/use-submissions) -- [Preload](https://docs.solidjs.com/solid-router/reference/preload-functions/preload) +- [preload](https://docs.solidjs.com/solid-router/reference/preload-functions/preload) - [useBeforeLeave](https://docs.solidjs.com/solid-router/reference/primitives/use-before-leave) - [useCurrentMatches](https://docs.solidjs.com/solid-router/reference/primitives/use-current-matches) - [useIsRouting](https://docs.solidjs.com/solid-router/reference/primitives/use-is-routing) diff --git a/scripts/sync-orama.mjs b/scripts/sync-orama.mjs index 8d72936d4f..749d26032f 100644 --- a/scripts/sync-orama.mjs +++ b/scripts/sync-orama.mjs @@ -1,10 +1,12 @@ import { readFileSync } from "node:fs"; import { globSync } from "glob"; +import { OramaCloud } from "@orama/core"; import { generalPurposeCrawler } from "@orama/crawly"; import "dotenv/config"; const ORAMA_PRIVATE_API_KEY = process.env.ORAMA_PRIVATE_API_KEY; -const ORAMA_PRIVATE_INDEX_ID = process.env.ORAMA_PRIVATE_INDEX_ID; +const ORAMA_DATASOURCE_ID = process.env.ORAMA_DATASOURCE_ID; +const ORAMA_PROJECT_ID = process.env.ORAMA_PROJECT_ID; const baseURL = new URL("../dist", import.meta.url).pathname; const HTMLFiles = globSync("**/*.html", { cwd: baseURL }); @@ -18,71 +20,41 @@ const pagesToIndex = HTMLFiles.flatMap((file) => { const productionDocsURL = `https://docs.solidjs.com/${path}`; - return { - ...generalPurposeCrawler(productionDocsURL, pageContent, { - parseCodeBlocks: false, - })[0], - contentWithCode: generalPurposeCrawler(productionDocsURL, pageContent)?.[0] - ?.content, - }; -}); + const content = generalPurposeCrawler(productionDocsURL, pageContent, { parseCodeBlocks: false })[0]; + const contentWithCode = generalPurposeCrawler(productionDocsURL, pageContent, { parseCodeBlocks: true })[0]; -async function emptyIndex() { - await fetch( - `https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/snapshot`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - authorization: `Bearer ${ORAMA_PRIVATE_API_KEY}`, - }, - body: JSON.stringify([]), - } - ); -} + const fullContent = { + title: content.title, + path: content.path, + content: content.content, + contentWithCode: contentWithCode.content, + } -async function upsertFreshData() { - const batches = []; - const batchesSize = 25; + if (content?.category) { + fullContent.category = `enum('${content.category}')` + } - for (let i = 0; i < pagesToIndex.length; i += batchesSize) { - const batch = pagesToIndex.slice(i, i + batchesSize); - batches.push(batch); + if (content?.section) { + fullContent.section = `enum('${content.section}')` } - for (let i = 0; i < batches.length; i++) { - const batch = batches[i]; + return fullContent +}); - await fetch( - `https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/notify`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - authorization: `Bearer ${ORAMA_PRIVATE_API_KEY}`, - }, - body: JSON.stringify({ - upsert: batch, - }), - } - ); - } -} +const orama = new OramaCloud({ + apiKey: ORAMA_PRIVATE_API_KEY, + projectId: ORAMA_PROJECT_ID +}) -async function deployIndex() { - await fetch( - `https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/deploy`, - { - method: "POST", - headers: { - authorization: `Bearer ${ORAMA_PRIVATE_API_KEY}`, - }, - } - ); +const index = orama.index.set(ORAMA_DATASOURCE_ID) + +console.log(`[Orama] - Indexing ${pagesToIndex.length} documents to Orama...`) - console.log("Index deployed"); -} +const tempIndexId = `tempIndex-${Date.now()}` +await index.createTemporaryIndex(tempIndexId) +const tempIdx = orama.index.set(tempIndexId) +await tempIdx.insertDocuments(pagesToIndex) +await index.swapTemporaryIndex(ORAMA_DATASOURCE_ID, tempIndexId) +await orama.index.delete(tempIndexId) -await emptyIndex(); -await upsertFreshData(); -await deployIndex(); +console.log(`[Orama] - Indexed ${pagesToIndex.length} documents to Orama.`) diff --git a/src/ui/search.tsx b/src/ui/search.tsx index 33a3da7f29..4f29fc203c 100644 --- a/src/ui/search.tsx +++ b/src/ui/search.tsx @@ -1,4 +1,4 @@ -import { OramaClient } from "@oramacloud/client"; +import { OramaCloud, type SearchResult } from "@orama/core"; import { createEffect, createSignal, @@ -16,23 +16,17 @@ import { createEventListener } from "@solid-primitives/event-listener"; import { isAppleDevice } from "@solid-primitives/platform"; function getOramaClient({ - endpoint, - api_key, -}: Record<"endpoint" | "api_key", string | null>) { - return endpoint && api_key - ? new OramaClient({ - endpoint, - api_key, + projectId, + apiKey, +}: Record<"projectId" | "apiKey", string | null>) { + return projectId && apiKey + ? new OramaCloud({ + projectId, + apiKey, }) : null; } -type OramaResult = { - hits: { - document: OramaDocument; - }[]; -}; - type OramaDocument = { content: string; path: string; @@ -41,8 +35,8 @@ type OramaDocument = { }; const client = getOramaClient({ - endpoint: import.meta.env.VITE_ORAMA_ENDPOINT ?? null, - api_key: import.meta.env.VITE_ORAMA_API_KEY ?? null, + projectId: import.meta.env.ORAMA_PROJECT_ID ?? null, + apiKey: import.meta.env.ORAMA_PUBLIC_API_KEY ?? null, }); export function Search() { @@ -58,28 +52,27 @@ export function Search() { async () => { const _searchTerm = searchTerm(); if (!_searchTerm) return {}; - const result: OramaResult | null = await client.search({ + const result = (await client.search({ term: _searchTerm, mode: "fulltext", - }); + datasources: [], + })) as SearchResult; if (!result) return {}; const seen: Record = {}; - result.hits = result.hits.filter( - hit => { - hit.document.path = hit.document.path.replace("/index#", "#"); - if(!seen[hit.document.path]) { - seen[hit.document.path] = true; - return hit; - } + result.hits = result.hits.filter((hit) => { + hit.document.path = hit.document.path.replace("/index#", "#"); + if (!seen[hit.document.path]) { + seen[hit.document.path] = true; + return hit; } - ); + }); const groupedHits = result.hits.reduce( (groupedHits, hit) => { const section = hit.document.section.replace( /(^|-)([a-z])/g, - (_, sep, letter) => sep + letter.toUpperCase() + (_, sep, letter) => sep + letter.toUpperCase(), ); if (!groupedHits[section]) { groupedHits[section] = []; @@ -87,13 +80,13 @@ export function Search() { groupedHits[section].push(hit); return groupedHits; }, - {} as Record + {} as Record["hits"]>, ); setActive(0); setResultRefs([]); return groupedHits; }, - { initialValue: {} } + { initialValue: {} }, ); const resultArray = () => Object.values(result()).flatMap((hits) => hits); @@ -196,7 +189,7 @@ export function Search() { class="w-full rounded border border-blue-100 bg-white px-9 py-2 ring-2 ring-blue-400 focus:outline-none focus-visible:border focus-visible:border-blue-400 focus-visible:ring-2 dark:bg-slate-800" onInput={(e) => startTransition(() => - setSearchTerm((e.target as HTMLInputElement).value) + setSearchTerm((e.target as HTMLInputElement).value), ) } onFocus={() => setActive(0)} @@ -290,7 +283,7 @@ export function Search() { {highlightContent( trimContent(hit.document.content), - regex() + regex(), )} @@ -835,8 +828,7 @@ function KeyboardShortcut(props: { key: string; class?: string }) { return (