Refactored ingestion logic to utilize API functions for database operations, improving code organization and readability.

Author: yogesh-aggarwal

ingestion/src/lib/api/establish-relations.ts

@@ -0,0 +1,19 @@
+import { RC_APP_URI } from "../../../constants"
+import { DBNodeRelation } from "../../../core/dbNode"
+
+export async function establishRelations(relations: DBNodeRelation[]): Promise<boolean> {
+   try {
+      const res = await fetch(`${RC_APP_URI}/establishRelations`, {
+         method: "POST",
+         headers: {
+            "Content-Type": "application/json",
+         },
+         body: JSON.stringify({ relations }),
+      })
+
+      return res.status === 200
+   } catch (e) {
+      console.log(e)
+      return false
+   }
+} 

ingestion/src/lib/api/index.ts

@@ -0,0 +1,3 @@
+export { purgeDB } from './purge-db'
+export { insertBatch } from './insert-batch'
+export { establishRelations } from './establish-relations' 

ingestion/src/lib/api/insert-batch.ts

@@ -0,0 +1,28 @@
+import { RC_APP_URI } from "../../../constants"
+import { DBNode } from "../../../core/dbNode"
+
+export async function insertBatch(batchID: string, nodes: DBNode[]): Promise<boolean> {
+   let tries = 5
+   while (tries--) {
+      try {
+         const res = await fetch(`${RC_APP_URI}/ingest`, {
+            method: "POST",
+            headers: {
+               accept: "application/json",
+               "Content-Type": "application/json",
+            },
+            body: JSON.stringify({ nodes, batchID }),
+         })
+
+         if (res.status !== 200) {
+            console.log(res)
+            return false
+         }
+
+         return true
+      } catch (e) {
+         console.log(e)
+      }
+   }
+   return false
+} 

ingestion/src/lib/api/purge-db.ts

@@ -0,0 +1,18 @@
+import { RC_APP_URI } from "../../../constants"
+
+export async function purgeDB(): Promise<boolean> {
+   try {
+      const res = await fetch(`${RC_APP_URI}/purgeDB`, {
+         method: "POST",
+         headers: {
+            accept: "application/json",
+            "Content-Type": "application/json",
+         },
+      })
+
+      return res.status === 200
+   } catch (e) {
+      console.log(e)
+      return false
+   }
+} 

ingestion/src/process/ingest/ingest.ts

@@ -2,72 +2,9 @@ import { readFileSync, readdirSync } from "fs"
 import path from "path"
 import { v4 as uuid } from "uuid"
 
-import { RC_APP_URI } from "../../constants"
-import { DBNode, DBNodeRelation } from "../../core/dbNode"
-import { DevDocDBNodeRelation } from "../../core/devDocDBNode"
-
-namespace Algorithms {
-   export async function purgeDB(): Promise<boolean> {
-      try {
-         const res = await fetch(`${RC_APP_URI}/purgeDB`, {
-            method: "POST",
-            headers: {
-               accept: "application/json",
-               "Content-Type": "application/json",
-            },
-         })
-
-         return res.status === 200
-      } catch (e) {
-         console.log(e)
-         return false
-      }
-   }
-
-   export async function insertBatch(batchID: string, nodes: DBNode[]): Promise<boolean> {
-      let tries = 5
-      while (tries--) {
-         try {
-            const res = await fetch(`${RC_APP_URI}/ingest`, {
-               method: "POST",
-               headers: {
-                  accept: "application/json",
-                  "Content-Type": "application/json",
-               },
-               body: JSON.stringify({ nodes, batchID }),
-            })
-
-            if (res.status !== 200) {
-               console.log(res)
-               return false
-            }
-
-            return true
-            // return res.status === 200
-         } catch (e) {
-            console.log(e)
-         }
-      }
-      return false
-   }
-
-   export async function establishRelations(relations: DBNodeRelation[]): Promise<boolean> {
-      try {
-         const res = await fetch(`${RC_APP_URI}/establishRelations`, {
-            method: "POST",
-            headers: {
-               "Content-Type": "application/json",
-            },
-            body: JSON.stringify({ relations }),
-         })
-
-         return res.status === 200
-      } catch (e) {
-         console.log(e)
-         return false
-      }
-   }
-}
+import { DBNode, DBNodeRelation } from "@/core/dbNode"
+import { DevDocDBNodeRelation } from "@/core/devDocDBNode"
+import { purgeDB, insertBatch, establishRelations } from "@/lib/api"
 
 export async function insertDataIntoDB(batchesDirPath: string) {
    console.log("🕒 Inserting")
@@ -76,7 +13,7 @@ export async function insertDataIntoDB(batchesDirPath: string) {
 
    // /* Step 1: Empty DB */
    {
-      const success = await Algorithms.purgeDB()
+      const success = await purgeDB()
       if (!success) {
          console.log("❌ Error emptying db")
          return
@@ -117,7 +54,7 @@ export async function insertDataIntoDB(batchesDirPath: string) {
             }
          }
 
-         const success = await Algorithms.insertBatch(batchID, nodes)
+         const success = await insertBatch(batchID, nodes)
          if (success) {
             console.log(`📦 ${batchID} inserted`)
          } else {
@@ -135,7 +72,7 @@ export async function insertDataIntoDB(batchesDirPath: string) {
       // Establish relations
       const batchSize = 1000
       for (let i = 0; i < relations.length; i += batchSize) {
-         const success = await Algorithms.establishRelations(relations.slice(i, i + batchSize))
+         const success = await establishRelations(relations.slice(i, i + batchSize))
          if (success) {
             console.log(`🔗 Relations established ${i + 1000}/${relations.length}`)
          } else {