Fixed the ingestion process from end to end

Author: preeesha

ai-assistant/src/endpoints/establishRelations.ts

@@ -54,7 +54,7 @@ export class EstablishRelationsEndpoint extends ApiEndpoint {
     makeBodies(
         content: any
     ): [EstablishRelationsEndpointRequestBody, EstablishRelationsEndpointResponseBody] {
-        const requestBody = JSON.parse(content) as EstablishRelationsEndpointRequestBody;
+        const requestBody = content as EstablishRelationsEndpointRequestBody;
         const responseBody: EstablishRelationsEndpointResponseBody = {
             status: 200,
         };

ingestion/src/main.ts

@@ -1,6 +1,7 @@
 import { exec } from "child_process"
 import { v4 as uuid } from "uuid"
 
+import { REPO_URI } from "./constants"
 import { insertDataIntoDB } from "./process/ingest/ingest"
 import { Codebase } from "./process/prepare/codebase"
 import { FileProcessor } from "./process/prepare/processor/file"

ingestion/src/process/ingest/ingest.ts

@@ -12,35 +12,42 @@ namespace Algorithms {
 			const res = await fetch(`${RC_APP_URI}/purgeDB`, {
 				method: "POST",
 				headers: {
+					accept: "application/json",
 					"Content-Type": "application/json",
 				},
 			})
 
 			return res.status === 200
 		} catch (e) {
-			console.log(e);
+			console.log(e)
 			return false
 		}
 	}
 
-	export async function insertBatch(batchID: string, nodes: DBNode[]): Promise<boolean> {
+	export async function insertBatch(
+		batchID: string,
+		nodes: DBNode[]
+	): Promise<boolean> {
 		try {
 			const res = await fetch(`${RC_APP_URI}/ingest`, {
 				method: "POST",
 				headers: {
+					accept: "application/json",
 					"Content-Type": "application/json",
 				},
 				body: JSON.stringify({ nodes, batchID }),
 			})
 
 			return res.status === 200
 		} catch (e) {
-			console.log(e);
+			console.log(e)
 			return false
 		}
 	}
 
-	export async function establishRelations(relations: DBNodeRelation[]): Promise<boolean> {
+	export async function establishRelations(
+		relations: DBNodeRelation[]
+	): Promise<boolean> {
 		try {
 			const res = await fetch(`${RC_APP_URI}/establishRelations`, {
 				method: "POST",
@@ -52,7 +59,7 @@ namespace Algorithms {
 
 			return res.status === 200
 		} catch (e) {
-			console.log(e);
+			console.log(e)
 			return false
 		}
 	}
@@ -87,11 +94,13 @@ export async function insertDataIntoDB(batchesDirPath: string) {
 			const nodes = Object.values(JSON.parse(data)) as DBNode[]
 
 			for (const node of nodes)
-				relations.push(...node.relations.map((relation) => ({
-					source: node.id,
-					target: relation.target,
-					relation: relation.relation,
-				})))
+				relations.push(
+					...node.relations.map((relation) => ({
+						source: node.id,
+						target: relation.target,
+						relation: relation.relation,
+					}))
+				)
 
 			const success = await Algorithms.insertBatch(batchID, nodes)
 			if (success) {
@@ -100,8 +109,7 @@ export async function insertDataIntoDB(batchesDirPath: string) {
 				errorBatches.add(batchID)
 			}
 		}
-		if (errorBatches.size > 0)
-			console.log("❌ Error batches", errorBatches)
+		if (errorBatches.size > 0) console.log("❌ Error batches", errorBatches)
 
 		// Establish relations
 		const success = await Algorithms.establishRelations(relations)

ingestion/src/process/prepare/codebase.ts

@@ -1,9 +1,4 @@
-import {
-	existsSync,
-	mkdirSync,
-	rmSync,
-	writeFileSync
-} from "fs"
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs"
 import { glob } from "glob"
 import path from "path"
 import { v4 as uuid } from "uuid"
@@ -55,7 +50,7 @@ export class Codebase {
 	}
 
 	private prepareFilesMetadata() {
-		const extensions = ["ts", "tsx", "js", "jsx"]
+		const extensions = ["ts", "js"]
 
 		console.log(`🕒 Preparing metadata for files: *.${extensions.join(", *.")}`)
 		{
@@ -107,7 +102,7 @@ export class Codebase {
 				const jobs = files.map((x) => this._fileProcessor.process(x, nodes))
 				await Promise.all(jobs)
 			} catch (e) {
-				console.log(e);
+				console.log(e)
 				console.error(`Error in processing ${start}-${end} files`)
 			}