Skip to content

Commit c860fb3

Browse files
saqibameensaqibameen
authored
📦 NEW: Handle deleted files in git-sync (#44)
* k * 📦 NEW: Handle deleted files in git sync * Revert run
1 parent 8aefd6c commit c860fb3

File tree

6 files changed

+249
-64
lines changed

6 files changed

+249
-64
lines changed

‎packages/baseai/src/deploy/index.ts

Lines changed: 118 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import fs from 'fs/promises';
1616
import fetch from 'node-fetch';
1717
import path from 'path';
1818
import color from 'picocolors';
19-
import type { MemoryI } from 'types/memory';
19+
import { type MemoryI } from 'types/memory';
2020
import type { Pipe, PipeOld } from 'types/pipe';
2121
import { getStoredAuth } from './../auth/index';
2222
import {
@@ -484,17 +484,24 @@ export async function deployMemory({
484484
p.log.step(`Processing documents for memory: ${memoryNameWithoutExt}`);
485485

486486
let filesToDeploy: string[] = [];
487+
let filesToDelete: string[] = [];
487488
let memoryDocs: MemoryDocumentI[] = [];
488489

489490
// Git sync memories
490491
if (memoryObject.config?.useGitRepo) {
491492
// Get names of files to deploy, i.e., changed or new files
492-
filesToDeploy = await handleGitSyncMemories({
493+
const {
494+
filesToDeploy: gitFilesToDeploy,
495+
filesToDelete: gitFilesToDelete
496+
} = await handleGitSyncMemories({
493497
memoryName: memoryNameWithoutExt,
494498
config: memoryObject.config,
495499
account
496500
});
497501

502+
filesToDeploy = gitFilesToDeploy;
503+
filesToDelete = gitFilesToDelete;
504+
498505
// Load all documents contents for the memory
499506
memoryDocs = await loadMemoryFiles(memoryNameWithoutExt);
500507

@@ -513,7 +520,6 @@ export async function deployMemory({
513520
spinner.stop(
514521
`No documents to deploy for memory: ${memoryNameWithoutExt}. Skipping.`
515522
);
516-
return;
517523
}
518524

519525
spinner.stop(`Processed memory: ${memoryName.split('.')[0]}`);
@@ -525,7 +531,8 @@ export async function deployMemory({
525531
documents: memoryDocs,
526532
account,
527533
overwrite,
528-
isGitSync: memoryObject.config?.useGitRepo
534+
isGitSync: memoryObject.config?.useGitRepo,
535+
docsToDelete: filesToDelete
529536
});
530537
spinner.stop(`Deployment finished memory: ${memoryObject.name}`);
531538
} catch (error) {
@@ -549,13 +556,15 @@ export async function upsertMemory({
549556
documents,
550557
account,
551558
overwrite,
552-
isGitSync = false
559+
isGitSync = false,
560+
docsToDelete = []
553561
}: {
554562
memory: MemoryI;
555563
documents: MemoryDocumentI[];
556564
account: Account;
557565
overwrite: boolean;
558566
isGitSync?: boolean;
567+
docsToDelete?: string[];
559568
}): Promise<void> {
560569
const { createMemory } = getMemoryApiUrls({
561570
account,
@@ -603,6 +612,21 @@ export async function upsertMemory({
603612
documents,
604613
overwrite
605614
});
615+
616+
if (docsToDelete?.length > 0) {
617+
await deleteDocumentsFromMemory({
618+
documents: docsToDelete,
619+
name: memory.name,
620+
account
621+
});
622+
}
623+
624+
await updateDeployedCommitHash(memory.name);
625+
626+
p.log.info(
627+
`Updated deployed commit hash for memory: ${memory.name}`
628+
);
629+
606630
return;
607631
}
608632
}
@@ -620,7 +644,18 @@ export async function upsertMemory({
620644
await uploadDocumentsToMemory({ documents, name, account });
621645

622646
if (isGitSync) {
647+
if (docsToDelete?.length > 0) {
648+
await deleteDocumentsFromMemory({
649+
documents: docsToDelete,
650+
name: memory.name,
651+
account
652+
});
653+
}
654+
623655
await updateDeployedCommitHash(memory.name);
656+
p.log.info(
657+
`Updated deployed commit hash for memory: ${memory.name}`
658+
);
624659
}
625660
} catch (error) {
626661
dlog('Error in createNewMemory:', error);
@@ -657,6 +692,38 @@ export async function uploadDocumentsToMemory({
657692
}
658693
}
659694

695+
export async function deleteDocumentsFromMemory({
696+
documents,
697+
name,
698+
account
699+
}: {
700+
documents: string[];
701+
name: string;
702+
account: Account;
703+
}) {
704+
p.log.info(`Deleting documents from memory: ${name}`);
705+
706+
for (const doc of documents) {
707+
try {
708+
p.log.message(`Deleting document: ${doc} ....`);
709+
await new Promise(resolve => setTimeout(resolve, 800)); // To avoid rate limiting
710+
711+
const deleteResponse = await deleteDocument({
712+
documentName: doc,
713+
memoryName: name,
714+
account
715+
});
716+
717+
dlog(`Delete response status: ${deleteResponse.status}`);
718+
719+
p.log.message(`Deleted document: ${doc}`);
720+
} catch (error) {
721+
throw error;
722+
}
723+
}
724+
p.log.info(`Deleted documents from memory: ${name}`);
725+
}
726+
660727
export async function handleExistingMemoryDeploy({
661728
memory,
662729
account,
@@ -892,6 +959,44 @@ async function getSignedUploadUrl({
892959
}
893960
}
894961

962+
async function deleteDocument({
963+
documentName,
964+
memoryName,
965+
account
966+
}: {
967+
documentName: string;
968+
memoryName: string;
969+
account: Account;
970+
}) {
971+
const { deleteDocument } = getMemoryApiUrls({
972+
account,
973+
memoryName,
974+
documentName
975+
});
976+
977+
try {
978+
const response = await fetch(deleteDocument, {
979+
method: 'DELETE',
980+
headers: {
981+
'Content-Type': 'application/json',
982+
Authorization: `Bearer ${account.apiKey}`
983+
}
984+
});
985+
986+
if (!response.ok) {
987+
const errorData = (await response.json()) as ErrorResponse;
988+
throw new Error(
989+
`HTTP error! status: ${response.status}, message: ${errorData.error?.message}`
990+
);
991+
}
992+
993+
return response;
994+
} catch (error) {
995+
dlog('Error in deleteDocument:', error);
996+
throw error;
997+
}
998+
}
999+
8951000
async function uploadDocument(signedUrl: string, document: Blob) {
8961001
let mimeType = document.type;
8971002

@@ -930,16 +1035,19 @@ async function uploadDocument(signedUrl: string, document: Blob) {
9301035

9311036
export function getMemoryApiUrls({
9321037
account,
933-
memoryName
1038+
memoryName,
1039+
documentName
9341040
}: {
9351041
account: Account;
9361042
memoryName: string;
1043+
documentName?: string;
9371044
}) {
9381045
const isOrgAccount = account.apiKey.includes(':');
9391046
const ownerLogin = isOrgAccount
9401047
? account.apiKey.split(':')[0]
9411048
: account.login;
9421049
const baseUrl = `https://api.langbase.com/beta`;
1050+
const baseUrlV1 = `https://api.langbase.com/v1`;
9431051

9441052
// Create memory URL
9451053
const createUrlOrg = `${baseUrl}/org/${ownerLogin}/memorysets`;
@@ -955,9 +1063,13 @@ export function getMemoryApiUrls({
9551063
// Delete memory URL
9561064
const deleteMemory = `${baseUrl}/memorysets/${ownerLogin}/${memoryName}`;
9571065

1066+
// Delete document URL
1067+
const deleteDocument = `${baseUrlV1}/memory/${memoryName}/documents/${documentName}`;
1068+
9581069
return {
9591070
listDocuments,
9601071
deleteMemory,
1072+
deleteDocument,
9611073
createMemory: isOrgAccount ? createUrlOrg : createUrlUser,
9621074
uploadDocument: isOrgAccount ? uploadDocumentOrg : uploadDocumentUser
9631075
};

‎packages/baseai/src/memory/embed.ts

Lines changed: 62 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { heading } from '@/utils/heading';
22
import { checkMemoryExists } from '@/utils/memory/check-memory-exist';
3+
import { deleteDocumentFromDB, loadDb } from '@/utils/memory/db/lib';
34
import { generateEmbeddings } from '@/utils/memory/generate-embeddings';
45
import {
56
handleGitSyncMemories,
@@ -55,12 +56,17 @@ export async function embedMemory({
5556
const memoryConfig = await loadMemoryConfig(memoryName);
5657

5758
let filesToEmbed: string[] = [];
59+
let filesToDelete: string[] = [];
5860

5961
if (memoryConfig?.useGitRepo) {
60-
filesToEmbed = await handleGitSyncMemories({
61-
memoryName: memoryName,
62-
config: memoryConfig
63-
});
62+
const { filesToDeploy, filesToDelete: gitFilesToDelete } =
63+
await handleGitSyncMemories({
64+
memoryName: memoryName,
65+
config: memoryConfig
66+
});
67+
68+
filesToEmbed = filesToDeploy;
69+
filesToDelete = gitFilesToDelete;
6470

6571
// Filter memory files to emebed
6672
memoryFiles = memoryFiles.filter(doc =>
@@ -69,24 +75,66 @@ export async function embedMemory({
6975
}
7076

7177
// 4- Generate embeddings.
72-
s.message('Generating embeddings...');
73-
const shouldOverwrite = memoryConfig?.useGitRepo ? true : overwrite;
74-
const result = await generateEmbeddings({
75-
memoryFiles,
76-
memoryName,
77-
overwrite: shouldOverwrite || false,
78-
useLocalEmbeddings
79-
});
78+
let embedResult = 'Embeddings updated.';
79+
if (filesToEmbed && filesToEmbed.length > 0) {
80+
s.message('Generating embeddings...');
81+
const shouldOverwrite = memoryConfig?.useGitRepo ? true : overwrite;
82+
embedResult = await generateEmbeddings({
83+
memoryFiles,
84+
memoryName,
85+
overwrite: shouldOverwrite || false,
86+
useLocalEmbeddings
87+
});
88+
}
8089

8190
if (memoryConfig?.useGitRepo) {
82-
p.log.success('Synced memory files with git repository.');
91+
if (filesToDelete.length > 0) {
92+
await deleteDocumentsFromDB({
93+
memoryName,
94+
filesToDelete
95+
});
96+
}
8397
await updateEmbeddedCommitHash(memoryName);
98+
p.log.info('Updated embedded commit hash.');
99+
p.log.success('Synced memory files with git repository.');
84100
}
85101

86-
s.stop(result);
102+
s.stop(embedResult);
87103
} catch (error: any) {
88104
s.stop(`Stopped!`);
89105
p.cancel(`FAILED: ${error.message}`);
90106
process.exit(1);
91107
}
92108
}
109+
110+
export async function deleteDocumentsFromDB({
111+
memoryName,
112+
filesToDelete
113+
}: {
114+
memoryName: string;
115+
filesToDelete: string[];
116+
}) {
117+
const s = p.spinner();
118+
s.start('Detected files to delete. Deleting...');
119+
120+
try {
121+
const memoryDb = await loadDb(memoryName);
122+
123+
for (const docName of filesToDelete) {
124+
if (memoryDb.data.documents[docName]) {
125+
await deleteDocumentFromDB({ db: memoryDb, docName });
126+
p.log.info(`Deleted document: ${color.cyan(docName)}`);
127+
}
128+
}
129+
130+
s.stop(`Documents deleted from memory ${memoryName}.`);
131+
} catch (error) {
132+
s.stop('Stopped!');
133+
if (error instanceof Error) {
134+
p.cancel(`Failed to delete documents: ${error.message}`);
135+
} else {
136+
p.cancel(`Failed to delete documents. An unknown error occurred.`);
137+
}
138+
process.exit(1);
139+
}
140+
}

‎packages/baseai/src/utils/memory/db/lib.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ export function getDocument(
137137
return db.data.documents[docName];
138138
}
139139

140-
export async function deleteDocument({
140+
export async function deleteDocumentFromDB({
141141
db,
142142
docName
143143
}: {

‎packages/baseai/src/utils/memory/generate-embeddings.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MEMORYSETS } from './constants';
66
import {
77
addChunksBulk,
88
addDocument,
9-
deleteDocument,
9+
deleteDocumentFromDB,
1010
getDocument,
1111
loadDb
1212
} from './db/lib';
@@ -53,7 +53,7 @@ export async function generateEmbeddings({
5353
p.log.info(
5454
`Removing existing embeddings for DOC: ${color.cyan(name)}`
5555
);
56-
await deleteDocument({
56+
await deleteDocumentFromDB({
5757
db: memoryDb,
5858
docName: name
5959
});

0 commit comments

Comments
 (0)