📦 NEW: Add local embed support for git-sync memory (#43)

* 📦 NEW: Add embed support for git-sync memory

* 👌 IMPROVE: update hash in the end of deploy

* 👌 IMPROVE: improve flow of embed

Author: saqibameen

packages/baseai/src/deploy/index.ts

@@ -19,7 +19,10 @@ import color from 'picocolors';
 import type { MemoryI } from 'types/memory';
 import type { Pipe, PipeOld } from 'types/pipe';
 import { getStoredAuth } from './../auth/index';
-import { handleGitSyncMemories } from '@/utils/memory/git-sync/handle-git-sync-memories';
+import {
+	handleGitSyncMemories,
+	updateDeployedCommitHash
+} from '@/utils/memory/git-sync/handle-git-sync-memories';
 import { handleSingleDocDeploy } from './document';
 
 export interface Account {
@@ -615,6 +618,10 @@ export async function upsertMemory({
 		// Upload documents
 		const { name } = (await createResponse.json()) as MemoryI;
 		await uploadDocumentsToMemory({ documents, name, account });
+
+		if (isGitSync) {
+			await updateDeployedCommitHash(memory.name);
+		}
 	} catch (error) {
 		dlog('Error in createNewMemory:', error);
 		throw error;

packages/baseai/src/memory/embed.ts

@@ -1,7 +1,12 @@
 import { heading } from '@/utils/heading';
 import { checkMemoryExists } from '@/utils/memory/check-memory-exist';
 import { generateEmbeddings } from '@/utils/memory/generate-embeddings';
+import {
+	handleGitSyncMemories,
+	updateEmbeddedCommitHash
+} from '@/utils/memory/git-sync/handle-git-sync-memories';
 import { validateMemoryName } from '@/utils/memory/lib';
+import loadMemoryConfig from '@/utils/memory/load-memory-config';
 import { loadMemoryFiles } from '@/utils/memory/load-memory-files';
 import * as p from '@clack/prompts';
 import color from 'picocolors';
@@ -39,22 +44,45 @@ export async function embedMemory({
 
 		// 2- Load memory data.
 		s.start('Processing memory docs...');
-		const memoryFiles = await loadMemoryFiles(memoryName);
+		let memoryFiles = await loadMemoryFiles(memoryName);
 
 		if (memoryFiles.length === 0) {
 			p.cancel(`No valid documents found in memory '${memoryName}'.`);
 			process.exit(1);
 		}
 
-		// 3- Generate embeddings.
+		// 3- Get memory config.
+		const memoryConfig = await loadMemoryConfig(memoryName);
+
+		let filesToEmbed: string[] = [];
+
+		if (memoryConfig?.useGitRepo) {
+			filesToEmbed = await handleGitSyncMemories({
+				memoryName: memoryName,
+				config: memoryConfig
+			});
+
+			// Filter memory files to emebed
+			memoryFiles = memoryFiles.filter(doc =>
+				filesToEmbed.includes(doc.name)
+			);
+		}
+
+		// 4- Generate embeddings.
 		s.message('Generating embeddings...');
+		const shouldOverwrite = memoryConfig?.useGitRepo ? true : overwrite;
 		const result = await generateEmbeddings({
 			memoryFiles,
 			memoryName,
-			overwrite: overwrite || false,
+			overwrite: shouldOverwrite || false,
 			useLocalEmbeddings
 		});
 
+		if (memoryConfig?.useGitRepo) {
+			p.log.success('Synced memory files with git repository.');
+			await updateEmbeddedCommitHash(memoryName);
+		}
+
 		s.stop(result);
 	} catch (error: any) {
 		s.stop(`Stopped!`);

packages/baseai/src/utils/memory/generate-embeddings.ts

@@ -231,3 +231,13 @@ function generateEmbeddingSummary({
 
 	p.log.info(summary);
 }
+
+export async function listLocalEmbeddedMemoryDocuments({
+	memoryName
+}: {
+	memoryName: string;
+}) {
+	const memoryDb = await loadDb(memoryName);
+	const documents = Object.keys(memoryDb.data.documents);
+	return documents;
+}

packages/baseai/src/utils/memory/git-sync/get-changed-files-between-commits.ts

@@ -1,26 +1,23 @@
 import { execSync } from 'child_process';
-import path from 'path';
 
 /**
- * Get changed files between two Git commits
- * @param {Object} options - The options for the function
- * @param {string} options.oldCommit - The old already deployed commit
- * @param {string} options.latestCommit - The latest commit to deploy (default: 'HEAD')
- * @param {string} options.repoPath - The path to the Git repository (default: process.cwd())
- * @param {string[]} options.extensions - Array of file extensions to filter (default: all files)
- * @param {boolean} options.includeUntracked - Whether to include untracked files (default: false)
- * @returns {Promise<string[]>} - Array of changed file paths
+ * Retrieves a list of files that have changed between two Git commits within a specified directory.
+ *
+ * @param {Object} options - The options for the function.
+ * @param {string} options.oldCommit - The old commit reference to compare from.
+ * @param {string} [options.latestCommit='HEAD'] - The latest commit reference to compare to. Defaults to 'HEAD'.
+ * @param {string} options.dirToTrack - The directory to track for changes.
+ * @returns {Promise<string[]>} A promise that resolves to an array of changed file paths.
+ * @throws Will throw an error if the Git command fails or if the oldCommit is an empty string.
  */
 export async function getChangedFilesBetweenCommits({
 	oldCommit,
 	latestCommit = 'HEAD',
-	dirToTrack,
-	extensions = []
+	dirToTrack
 }: {
 	oldCommit: string;
 	latestCommit: string;
 	dirToTrack: string;
-	extensions: string[];
 }): Promise<string[]> {
 	try {
 		// Validate inputs

packages/baseai/src/utils/memory/git-sync/handle-git-sync-memories.ts

@@ -5,6 +5,8 @@ import { getChangedFilesBetweenCommits } from './get-changed-files-between-commi
 import type { MemoryConfigI } from 'types/memory';
 import { listMemoryDocuments, type Account } from '@/deploy';
 import { loadMemoryFilesFromCustomDir } from '../load-memory-files';
+import { listLocalEmbeddedMemoryDocuments } from '../generate-embeddings';
+import { saveEmbeddedCommitHashInMemoryConfig } from './save-embedded-commit-in-config';
 
 export async function handleGitSyncMemories({
 	memoryName,
@@ -13,7 +15,7 @@ export async function handleGitSyncMemories({
 }: {
 	memoryName: string;
 	config: MemoryConfigI;
-	account: Account;
+	account?: Account; // Undefined for local embed
 }): Promise<string[]> {
 	// Check for uncommitted changes
 	try {
@@ -30,13 +32,19 @@ export async function handleGitSyncMemories({
 
 	let filesToDeploy: string[] = [];
 
+	const isEmbed = !account;
+
 	// Step 1:
 	// Fetch the uploaded documents and compare with the local documents
 	// Handles new files that are not in the prodDocs due to extension and path updates
-	const prodDocs = await listMemoryDocuments({
-		account,
-		memoryName
-	});
+	const prodDocs = !isEmbed
+		? await listMemoryDocuments({
+				account,
+				memoryName
+			})
+		: await listLocalEmbeddedMemoryDocuments({
+				memoryName
+			}); // For local embedded docs are prod equivalent
 
 	const allFilesWithContent = await loadMemoryFilesFromCustomDir({
 		memoryName,
@@ -53,7 +61,11 @@ export async function handleGitSyncMemories({
 	// Step 2.1:
 	// If there's no deployedCommitHash, user is deploying for the first time
 	// Deploy all files in the directory
-	if (!config.deployedCommitHash) {
+	const lastHashUsed = isEmbed
+		? config.embeddedCommitHash
+		: config.deployedCommitHash;
+
+	if (!lastHashUsed) {
 		filesToDeploy = allFiles;
 		p.log.info(
 			`Found no previous deployed commit. Deploying all ${filesToDeploy.length} files in memory "${memoryName}":`
@@ -62,10 +74,9 @@ export async function handleGitSyncMemories({
 	// Step 2.2: Otherwise, get changed files between commits
 	else {
 		filesToDeploy = await getChangedFilesBetweenCommits({
-			oldCommit: config.deployedCommitHash,
+			oldCommit: lastHashUsed,
 			latestCommit: 'HEAD',
-			dirToTrack: config.dirToTrack,
-			extensions: config.extToTrack
+			dirToTrack: config.dirToTrack
 		});
 
 		if (filesToDeploy.length > 0) {
@@ -76,9 +87,16 @@ export async function handleGitSyncMemories({
 			// Print the changed file names TODO: Remove because it may clutter the terminal?
 			filesToDeploy.forEach(file => p.log.message(file));
 		} else {
-			p.log.info(
-				`No changes detected for memory "${memoryName}" since last deployment.`
-			);
+			const isEmbed = !account;
+			if (isEmbed) {
+				p.log.info(
+					`No changes detected for memory "${memoryName}" since last embedding.`
+				);
+			} else {
+				p.log.info(
+					`No changes detected for memory "${memoryName}" since last deployment.`
+				);
+			}
 		}
 	}
 
@@ -90,14 +108,21 @@ export async function handleGitSyncMemories({
 		return [];
 	}
 
-	// Step 4
-	// Update deployedCommitHash in memory config
-	// TODO: Should we update the deployedCommitHash after deploying?
+	return filesToDeploy;
+}
+
+export async function updateDeployedCommitHash(memoryName: string) {
 	const currentCommitHash = execSync('git rev-parse HEAD').toString().trim();
 	await saveDeployedCommitHashInMemoryConfig({
 		memoryName,
 		deployedCommitHash: currentCommitHash
 	});
+}
 
-	return filesToDeploy;
+export async function updateEmbeddedCommitHash(memoryName: string) {
+	const currentCommitHash = execSync('git rev-parse HEAD').toString().trim();
+	await saveEmbeddedCommitHashInMemoryConfig({
+		memoryName,
+		embeddedCommitHash: currentCommitHash
+	});
 }

packages/baseai/src/utils/memory/git-sync/save-embedded-commit-in-config.ts

@@ -0,0 +1,45 @@
+import fs from 'fs/promises';
+import * as p from '@clack/prompts';
+import path from 'path';
+
+export async function saveEmbeddedCommitHashInMemoryConfig({
+	memoryName,
+	embeddedCommitHash
+}: {
+	memoryName: string;
+	embeddedCommitHash: string;
+}): Promise<void> {
+	try {
+		const memoryDir = path.join(
+			process.cwd(),
+			'baseai',
+			'memory',
+			memoryName
+		);
+		const indexFilePath = path.join(memoryDir, 'index.ts');
+		let fileContents = await fs.readFile(indexFilePath, 'utf-8');
+
+		// Check if the embeddedCommitHash already exists in the config
+		if (fileContents.includes('embeddedCommitHash:')) {
+			// Update the existing embeddedCommitHash
+			fileContents = fileContents.replace(
+				/embeddedCommitHash:\s*['"].*['"]/,
+				`embeddedCommitHash: '${embeddedCommitHash}'`
+			);
+		} else {
+			// Add the embeddedCommitHash to the config
+			fileContents = fileContents.replace(
+				/config:\s*{/,
+				`config: {\n    embeddedCommitHash: '${embeddedCommitHash}',`
+			);
+		}
+
+		// Write the updated contents back to the file
+		await fs.writeFile(indexFilePath, fileContents, 'utf-8');
+
+		p.log.success(`Updated embeddedCommitHash for memory '${memoryName}'.`);
+	} catch (error) {
+		console.error(`Error saving latest commit hash: ${error}`);
+		throw error;
+	}
+}

packages/baseai/src/utils/memory/load-memory-config.ts

@@ -1,7 +1,7 @@
 import fs from 'fs/promises';
 import path from 'path';
 import * as p from '@clack/prompts';
-import type { MemoryConfigI } from 'types/memory';
+import { memoryConfigSchema, type MemoryConfigI } from 'types/memory';
 
 function parsePathJoin(joinArgs: string): string {
 	// Remove any quotes, split by comma, and trim each argument
@@ -24,6 +24,12 @@ function parseConfig(configString: string): MemoryConfigI {
 		/dirToTrack:(?:path\.(?:posix\.)?join\((.*?)\)|['"](.+?)['"])/
 	);
 	const extToTrackMatch = cleanConfig.match(/extToTrack:(\[.*?\])/);
+	const deployedCommitHashMatch = cleanConfig.match(
+		/deployedCommitHash:['"](.+?)['"]/
+	);
+	const embeddedCommitHashMatch = cleanConfig.match(
+		/embeddedCommitHash:['"](.+?)['"]/
+	);
 
 	if (!useGitRepoMatch || !dirToTrackMatch || !extToTrackMatch) {
 		throw new Error('Unable to parse config structure');
@@ -34,12 +40,34 @@ function parseConfig(configString: string): MemoryConfigI {
 		? dirToTrackMatch[2]
 		: parsePathJoin(dirToTrackMatch[1]);
 	const extToTrack = JSON.parse(extToTrackMatch[1].replace(/'/g, '"'));
+	const deployedCommitHash = deployedCommitHashMatch
+		? deployedCommitHashMatch[1]
+		: undefined;
+	const embeddedCommitHash = embeddedCommitHashMatch
+		? embeddedCommitHashMatch[1]
+		: undefined;
 
-	return {
+	const config: MemoryConfigI = {
 		useGitRepo,
 		dirToTrack,
 		extToTrack
 	};
+
+	if (deployedCommitHash) {
+		config.deployedCommitHash = deployedCommitHash;
+	}
+
+	if (embeddedCommitHash) {
+		config.embeddedCommitHash = embeddedCommitHash;
+	}
+
+	// Validate the parsed config against the schema
+	const result = memoryConfigSchema.safeParse(config);
+	if (!result.success) {
+		throw new Error(`Invalid config: ${result.error.message}`);
+	}
+
+	return config;
 }
 
 export default async function loadMemoryConfig(
@@ -73,14 +101,28 @@ export default async function loadMemoryConfig(
 		try {
 			const config = parseConfig(configMatch[1]);
 			return config;
-		} catch (parseError) {
-			p.cancel(`Unable to read config in '${memoryName}/index.ts'.`);
+		} catch (error) {
+			if (error instanceof Error) {
+				p.cancel(
+					`Unable to read config in '${memoryName}/index.ts': ${error.message}`
+				);
+			} else {
+				p.cancel(
+					`Unable to read config in '${memoryName}/index.ts': Unknown error occurred`
+				);
+			}
 			process.exit(1);
 		}
 	} catch (error) {
-		p.cancel(
-			`Memory '${memoryName}' does not exist or could not be loaded.`
-		);
+		if (error instanceof Error) {
+			p.cancel(
+				`Memory '${memoryName}' does not exist or could not be loaded: ${error.message}`
+			);
+		} else {
+			p.cancel(
+				`Memory '${memoryName}' does not exist or could not be loaded: Unknown error occurred`
+			);
+		}
 		process.exit(1);
 	}
 }

packages/baseai/types/memory.ts

@@ -42,7 +42,8 @@ export const memoryConfigSchema = z.object({
 			)
 			.min(1, 'At least one file extension must be specified')
 	]),
-	deployedCommitHash: z.string().optional()
+	deployedCommitHash: z.string().optional(),
+	embeddedCommitHash: z.string().optional()
 });
 
 export type MemoryConfigI = z.infer<typeof memoryConfigSchema>;