📦 NEW: memory refresh

Author: MaedahBatool

baseai/memory/docs/index.ts

@@ -1,14 +1,14 @@
 import { MemoryI } from '@baseai/core';
 
 const memoryDocs = (): MemoryI => ({
-	name: 'docs',
-	description: 'Docs folder of sourcegraph docs repository as an auto-synced memory',
+	name: 'memory-sg-docs-live',
+	description: 'An AI memory storing all Sourcegraph docs.',
 	git: {
 		enabled: true,
 		include: ['**/*.mdx'],
 		gitignore: true,
-		deployedAt: '5f3fec8530280d01a783aadcdeb0ccc3f9cd8b70',
-		embeddedAt: ''
+		embeddedAt: '',
+		deployedAt: '',
 	},
 	documents: {
 		meta: doc => {
@@ -22,3 +22,6 @@ const memoryDocs = (): MemoryI => ({
 });
 
 export default memoryDocs;
+
+// Old
+// deployedAt: '5f3fec8530280d01a783aadcdeb0ccc3f9cd8b70',

package.json

@@ -42,7 +42,7 @@
     "github-slugger": "^2.0.0",
     "js-yaml": "^4.1.0",
     "kbar": "^0.1.0-beta.44",
-    "langbase": "^1.1.26",
+    "langbase": "^1.1.46",
     "lucide-react": "^0.372.0",
     "mdx": "^0.3.1",
     "next": "^14.2.3",
@@ -68,6 +68,7 @@
   },
   "devDependencies": {
     "baseai": "^0.9.40",
+    "dotenv": "^16.4.7",
     "eslint": "8.45.0",
     "eslint-config-next": "13.4.16",
     "glob": "^11.0.1",

pnpm-lock.yaml

@@ -0,0 +1,18 @@
+import 'dotenv/config';
+import { Langbase } from 'langbase';
+
+const langbase = new Langbase({
+    apiKey: process.env.LANGBASE_API_KEY!,
+});
+
+async function main() {
+    const memory = await langbase.memories.create({
+        name: 'memory-sg-docs-live',
+        description: 'An AI memory storing all Sourcegraph docs.',
+        chunk_size: 10000,
+    });
+
+    console.log('Memory created:', memory);
+}
+
+main();

scripts/langbase-create-memory.ts

@@ -0,0 +1,83 @@
+const path = require('path');
+const fs = require('fs');
+const {sync: glob} = require('glob');
+
+// Configuration - Update this base URL as needed
+const baseUrl = 'https://sourcegraph.com/';
+
+// Find all .mdx files, ignoring node_modules
+const mdxFiles = glob('**/*.mdx', {ignore: ['node_modules/**']});
+
+// Array to store word counts
+const wordCounts = [];
+let totalWords = 0;
+
+// Process each file
+mdxFiles.forEach(filePath => {
+	const fileName = path.basename(filePath);
+	const content = fs.readFileSync(filePath, 'utf8');
+
+	// Count words (split by whitespace)
+	const words = content
+		.trim()
+		.split(/\s+/)
+		.filter(word => word.length > 0);
+	const wordCount = words.length;
+
+	// Add to total and array
+	totalWords += wordCount;
+	wordCounts.push({
+		fileName,
+		filePath,
+		wordCount
+	});
+
+	// Log word count for each file
+	console.log(`${filePath}: ${wordCount} words`);
+});
+
+// Calculate average
+const averageWordCount = totalWords / wordCounts.length;
+
+// Calculate median (need to sort the array first)
+const sortedCounts = [...wordCounts].sort((a, b) => a.wordCount - b.wordCount);
+const middleIndex = Math.floor(sortedCounts.length / 2);
+const medianWordCount =
+	sortedCounts.length % 2 === 0
+		? (sortedCounts[middleIndex - 1].wordCount +
+				sortedCounts[middleIndex].wordCount) /
+			2
+		: sortedCounts[middleIndex].wordCount;
+
+// Log summary statistics
+console.log('\n--- Summary Statistics ---');
+console.log(`Total files: ${wordCounts.length}`);
+console.log(`Total words: ${totalWords}`);
+console.log(`Average words per file: ${averageWordCount.toFixed(2)}`);
+console.log(`Median words per file: ${medianWordCount}`);
+
+// Find shortest and longest files
+const shortestFile = sortedCounts[0];
+const longestFile = sortedCounts[sortedCounts.length - 1];
+console.log(
+	`\nShortest file: ${shortestFile.filePath} (${shortestFile.wordCount} words)`
+);
+console.log(
+	`Longest file: ${longestFile.filePath} (${longestFile.wordCount} words)`
+);
+
+// Optional: Write summary to a file
+const summary = {
+	totalFiles: wordCounts.length,
+	totalWords,
+	averageWordCount,
+	medianWordCount,
+	fileCounts: wordCounts
+};
+
+// fs.writeFileSync(
+// 	'word-count-summary.json',
+// 	JSON.stringify(summary, null, 2),
+// 	'utf8'
+// );
+console.log('\nSummary written to word-count-summary.json');

scripts/word-count.js

@@ -9,11 +9,41 @@ const langbase = new Langbase({
 
 export async function POST(req: NextRequest) {
 	const options = await req.json();
+	console.log("🚀 ~ options:", options)
 
-	const { stream, threadId } = await langbase.pipe.run({
-		...options,
-		name: 'ask-sourcegraph-docs'
+	const { completion } = await langbase.pipes.run({
+		name: 'query-rewrite',
+		messages: [
+			{
+				role: 'user',
+				content: options.messages[options.messages.length - 1].content,
+			},
+		],
+		stream: false
 	});
+	console.log("🚀 ~ completion:", completion)
+
+
+	// Parse the completion to get the rewritten query
+	const rewrittenQuery = JSON.parse(completion).rewrittenQuery;
+	console.log("🚀 ~ rewrittenQuery:", rewrittenQuery)
+
+	// Create a new options object with the rewritten query
+	const updatedOptions = {
+		...options,
+		messages: [
+			...options.messages.slice(0, -1), // Keep all messages except the last one
+			{
+				role: 'user',
+				content: rewrittenQuery
+			}
+		],
+		name: 'ask-sourcegraph-docs',
+		lastMessageOnly: true,
+	};
+	console.log("🚀 ~ updatedOptions:", updatedOptions)
+
+	const { stream, threadId } = await langbase.pipes.run(updatedOptions);
 
 	return new Response(stream, {
 		status: 200,