refactor: migrate to new orama infra (#1371)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

Author: micheleriva

.github/workflows/orama_sync.yml

@@ -32,4 +32,5 @@ jobs:
         run: pnpm sync:orama
         env:
           ORAMA_PRIVATE_API_KEY: ${{ secrets.ORAMA_PRIVATE_API_KEY }}
-          ORAMA_PRIVATE_INDEX_ID: ${{ secrets.ORAMA_PRIVATE_INDEX_ID }}
+          ORAMA_DATASOURCE_ID: ${{ secrets.ORAMA_DATASOURCE_ID }}
+          ORAMA_PROJECT_ID: ${{ secrets.ORAMA_PROJECT_ID }}

env.d.ts

@@ -10,8 +10,10 @@ interface ImportMeta {
 
 declare namespace NodeJS {
 	interface ProcessEnv {
+		readonly ORAMA_PROJECT_ID: string;
+		readonly ORAMA_DATASOURCE_ID: string;
+		readonly ORAMA_PUBLIC_API_KEY: string;
 		readonly ORAMA_PRIVATE_API_KEY: string;
-		readonly ORAMA_PRIVATE_INDEX_ID: string;
 	}
 }
 

package.json

@@ -19,7 +19,7 @@
 	"dependencies": {
 		"@kobalte/core": "^0.13.11",
 		"@kobalte/solidbase": "^0.2.20",
-		"@oramacloud/client": "^2.1.4",
+		"@orama/core": "^1.2.14",
 		"@solid-primitives/event-listener": "^2.4.3",
 		"@solid-primitives/marker": "^0.2.2",
 		"@solid-primitives/media": "^2.3.3",
@@ -62,4 +62,4 @@
 		"node": ">=24",
 		"pnpm": ">=10"
 	}
-}
+}

pnpm-lock.yaml

@@ -128,7 +128,8 @@
 - [Routing](https://docs.solidjs.com/solid-start/building-your-application/routing)
 - [API routes](https://docs.solidjs.com/solid-start/building-your-application/api-routes)
 - [CSS and styling](https://docs.solidjs.com/solid-start/building-your-application/css-and-styling)
-- [Data loading](https://docs.solidjs.com/solid-start/building-your-application/data-loading)
+- [Data fetching](https://docs.solidjs.com/solid-start/building-your-application/data-fetching)
+- [Data mutation](https://docs.solidjs.com/solid-start/building-your-application/data-mutation)
 - [Head and metadata](https://docs.solidjs.com/solid-start/building-your-application/head-and-metadata)
 - [Route Pre-rendering](https://docs.solidjs.com/solid-start/building-your-application/route-prerendering)
 - [Static assets](https://docs.solidjs.com/solid-start/building-your-application/static-assets)
@@ -176,6 +177,11 @@
 - [Actions](https://docs.solidjs.com/solid-router/concepts/actions)
 - [Single page applications](https://docs.solidjs.com/solid-router/rendering-modes/spa)
 - [Server side rendering](https://docs.solidjs.com/solid-router/rendering-modes/ssr)
+- [Queries](https://docs.solidjs.com/solid-router/data-fetching/queries)
+- [Streaming](https://docs.solidjs.com/solid-router/data-fetching/streaming)
+- [Revalidation](https://docs.solidjs.com/solid-router/data-fetching/revalidation)
+- [Preload data](https://docs.solidjs.com/solid-router/data-fetching/how-to/preload-data)
+- [Handle pending and error states](https://docs.solidjs.com/solid-router/data-fetching/how-to/handle-error-and-loading-states)
 - [Lazy loading](https://docs.solidjs.com/solid-router/advanced-concepts/lazy-loading)
 - [Migration from v0.9.x](https://docs.solidjs.com/solid-router/guides/migration)
 - [A](https://docs.solidjs.com/solid-router/reference/components/a)
@@ -193,7 +199,7 @@
 - [useAction](https://docs.solidjs.com/solid-router/reference/data-apis/use-action)
 - [useSubmission](https://docs.solidjs.com/solid-router/reference/data-apis/use-submission)
 - [useSubmissions](https://docs.solidjs.com/solid-router/reference/data-apis/use-submissions)
-- [Preload](https://docs.solidjs.com/solid-router/reference/preload-functions/preload)
+- [preload](https://docs.solidjs.com/solid-router/reference/preload-functions/preload)
 - [useBeforeLeave](https://docs.solidjs.com/solid-router/reference/primitives/use-before-leave)
 - [useCurrentMatches](https://docs.solidjs.com/solid-router/reference/primitives/use-current-matches)
 - [useIsRouting](https://docs.solidjs.com/solid-router/reference/primitives/use-is-routing)

public/llms.txt

@@ -1,10 +1,12 @@
 import { readFileSync } from "node:fs";
 import { globSync } from "glob";
+import { OramaCloud } from "@orama/core";
 import { generalPurposeCrawler } from "@orama/crawly";
 import "dotenv/config";
 
 const ORAMA_PRIVATE_API_KEY = process.env.ORAMA_PRIVATE_API_KEY;
-const ORAMA_PRIVATE_INDEX_ID = process.env.ORAMA_PRIVATE_INDEX_ID;
+const ORAMA_DATASOURCE_ID = process.env.ORAMA_DATASOURCE_ID;
+const ORAMA_PROJECT_ID = process.env.ORAMA_PROJECT_ID;
 
 const baseURL = new URL("../dist", import.meta.url).pathname;
 const HTMLFiles = globSync("**/*.html", { cwd: baseURL });
@@ -18,71 +20,41 @@ const pagesToIndex = HTMLFiles.flatMap((file) => {
 
 	const productionDocsURL = `https://docs.solidjs.com/${path}`;
 
-	return {
-		...generalPurposeCrawler(productionDocsURL, pageContent, {
-			parseCodeBlocks: false,
-		})[0],
-		contentWithCode: generalPurposeCrawler(productionDocsURL, pageContent)?.[0]
-			?.content,
-	};
-});
+	const content = generalPurposeCrawler(productionDocsURL, pageContent, { parseCodeBlocks: false })[0];
+	const contentWithCode = generalPurposeCrawler(productionDocsURL, pageContent, { parseCodeBlocks: true })[0];
 
-async function emptyIndex() {
-	await fetch(
-		`https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/snapshot`,
-		{
-			method: "POST",
-			headers: {
-				"Content-Type": "application/json",
-				authorization: `Bearer ${ORAMA_PRIVATE_API_KEY}`,
-			},
-			body: JSON.stringify([]),
-		}
-	);
-}
+	const fullContent = {
+		title: content.title,
+		path: content.path,
+		content: content.content,
+		contentWithCode: contentWithCode.content,
+	}
 
-async function upsertFreshData() {
-	const batches = [];
-	const batchesSize = 25;
+	if (content?.category) {
+		fullContent.category = `enum('${content.category}')`
+	}
 
-	for (let i = 0; i < pagesToIndex.length; i += batchesSize) {
-		const batch = pagesToIndex.slice(i, i + batchesSize);
-		batches.push(batch);
+	if (content?.section) {
+		fullContent.section = `enum('${content.section}')`
 	}
 
-	for (let i = 0; i < batches.length; i++) {
-		const batch = batches[i];
+	return fullContent
+});
 
-		await fetch(
-			`https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/notify`,
-			{
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					authorization: `Bearer ${ORAMA_PRIVATE_API_KEY}`,
-				},
-				body: JSON.stringify({
-					upsert: batch,
-				}),
-			}
-		);
-	}
-}
+const orama = new OramaCloud({
+	apiKey: ORAMA_PRIVATE_API_KEY,
+	projectId: ORAMA_PROJECT_ID
+})
 
-async function deployIndex() {
-	await fetch(
-		`https://api.oramasearch.com/api/v1/webhooks/${ORAMA_PRIVATE_INDEX_ID}/deploy`,
-		{
-			method: "POST",
-			headers: {
-				authorization: `Bearer ${ORAMA_PRIVATE_API_KEY}`,
-			},
-		}
-	);
+const index = orama.index.set(ORAMA_DATASOURCE_ID)
+
+console.log(`[Orama] - Indexing ${pagesToIndex.length} documents to Orama...`)
 
-	console.log("Index deployed");
-}
+const tempIndexId = `tempIndex-${Date.now()}`
+await index.createTemporaryIndex(tempIndexId)
+const tempIdx = orama.index.set(tempIndexId)
+await tempIdx.insertDocuments(pagesToIndex)
+await index.swapTemporaryIndex(ORAMA_DATASOURCE_ID, tempIndexId)
+await orama.index.delete(tempIndexId)
 
-await emptyIndex();
-await upsertFreshData();
-await deployIndex();
+console.log(`[Orama] - Indexed ${pagesToIndex.length} documents to Orama.`)