Documentation nodes are now written to the file system as same as the code nodes so that they can be uploaded like them only

Author: preeesha

ingestion/src/main.ts

@@ -1,7 +1,11 @@
 import { exec } from "child_process"
 import { v4 as uuid } from "uuid"
 
+import { REPO_URI } from "./constants"
 import { Documentation } from "./process/documentation/documentation"
+import { insertDataIntoDB } from "./process/ingest/ingest"
+import { Codebase } from "./process/prepare/codebase"
+import { FileProcessor } from "./process/prepare/processor/file"
 
 namespace Algorithms {
 	export async function execCommand(command: string) {
@@ -24,16 +28,17 @@ namespace Algorithms {
 async function main() {
 	const sessionID = uuid()
 
-	// await Algorithms.execCommand(`git clone ${REPO_URI} ${sessionID}`)
-	// {
-	// 	const codebase = new Codebase(sessionID, new FileProcessor(), 1)
-	// 	await codebase.process()
-	// 	await insertDataIntoDB(codebase.dataDirPath)
-	// }
-	// await Algorithms.execCommand(`rm -rf ${sessionID}`)
+	await Algorithms.execCommand(`git clone ${REPO_URI} ${sessionID}`)
+	{
+		const codebase = new Codebase(sessionID, new FileProcessor(), 1)
+		await codebase.process()
 
-	const docs = new Documentation()
-	await docs.prepare()
+		const docs = new Documentation()
+		await docs.prepare(codebase.dataDirPath)
+
+		await insertDataIntoDB(codebase.dataDirPath)
+	}
+	await Algorithms.execCommand(`rm -rf ${sessionID}`)
 }
 
 main()

ingestion/src/process/documentation/documentation.ts

@@ -1,6 +1,6 @@
-import { writeFileSync } from "fs"
 import puppeteer from "puppeteer"
 
+import { writeFile } from "fs/promises"
 import { DOCUMENTATION_URL } from "../../constants"
 import { DevDocDBNode } from "../../core/devDocsDBNode"
 import { IDocumentation } from "./documentation.types"
@@ -54,8 +54,14 @@ export class Documentation implements IDocumentation {
 		return flattenedNodes
 	}
 
-	async prepare() {
+	async prepare(dataDirPath: string) {
 		const nodes = await this.prepareDevDocsNodes()
-		writeFileSync("nodes.json", JSON.stringify(nodes, null, 3))
+
+		const jobs = []
+		for (const node of nodes) {
+			jobs.push(
+				writeFile(`${dataDirPath}/docs-${node.id}.json`, JSON.stringify(node))
+			)
+		}
 	}
 }

ingestion/src/process/documentation/documentation.types.ts

@@ -1,3 +1,3 @@
 export interface IDocumentation {
-	prepare(): Promise<void>
+	prepare(dataDirPath: string): Promise<void>
 }