Implemented webhooks for the ingestion server

Author: preeesha

rocket-chatter-ingestion-server/src/routes/ingest.ts

@@ -65,13 +65,107 @@ async function startProcessJob(dirName: string): Promise<boolean> {
 	}
 }
 
+/**
+ * This function retries the given function until it returns true or the maximum number of retries is reached.
+ *
+ * @param maxRetries The maximum number of retries to be made before giving up. If set to -1, it will retry indefinitely
+ * @param retryInterval The time interval between each retry in milliseconds
+ * @param fn The function to be retried
+ * @returns true/false (boolean) representing the success or failure respectively
+ */
+async function keepRetrying(
+	maxRetries: number,
+	retryInterval: number,
+	fn: () => Promise<any>
+): Promise<boolean> {
+	if (maxRetries < 0) maxRetries = Number.MAX_SAFE_INTEGER
+
+	let retries = 0
+	while (retries < maxRetries) {
+		if (await fn()) return true
+
+		await new Promise((resolve) => setTimeout(resolve, retryInterval))
+		retries++
+	}
+
+	return false
+}
+
+/**
+ * This function sends the success response to the client
+ *
+ * @param successURL The URL where the success response is to be sent
+ */
+async function sendSuccessResponse(successURL: string, startTime: number) {
+	const success = await keepRetrying(100, 1000, async () => {
+		await fetch(successURL, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify({
+				status: 200,
+				message: "INGESTION_COMPLETED",
+				timeTaken: `${(Date.now() - startTime) / 1000}s`,
+			}),
+		})
+	})
+	if (!success) {
+		console.error("Failed to send success response")
+	}
+}
+
+/**
+ * This function sends the failure response to the client
+ *
+ * @param failureURL The URL where the failure response is to be sent
+ */
+async function sendFailureResponse(
+	failureURL: string,
+	reason: string,
+	startTime: number
+) {
+	const success = await keepRetrying(100, 1000, async () => {
+		await fetch(failureURL, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify({
+				status: 400,
+				message: "INGESTION_FAILED",
+				reason: reason,
+				timeTaken: `${(Date.now() - startTime) / 1000}s`,
+			}),
+		})
+	})
+	if (!success) {
+		console.error("Failed to send failure response")
+	}
+}
+
 /**
  *
  * @param _ request from the server to ingest code.
  * @param res response tells that ingestion is sucessfull or not
  *
  */
-export async function ingestRoute(_: Request, res: Response) {
+export async function ingestRoute(req: Request, res: Response) {
+	const { successURL, failureURL } = req.body
+	if (!successURL || !failureURL) {
+		return res.status(400).send({
+			status: 400,
+			message: "BAD_REQUEST",
+			reason: "URLS_MISSING",
+		})
+	}
+
+	// Send the response back to the client so that request is not kept waiting & timeout eventually
+	res.status(200).send({
+		status: 200,
+		message: "INGESTION INITIATED",
+	})
+
 	const sessionID = nanoid.customAlphabet(
 		"1234567890abcdefghijklmnopqrstuvwxyz"
 	)(10)
@@ -80,34 +174,19 @@ export async function ingestRoute(_: Request, res: Response) {
 	{
 		let success = false
 
+		/* Step 1: Fetch codebase to local storage */
 		success = fetchCodebaseFromRemote(
 			sessionID,
 			"https://github.com/RocketChat/Rocket.Chat"
 		)
-		if (!success) {
-			console.error("Error while fetching code.")
-			return res.status(500).send({
-				status: 500,
-				message: "FETCH_FAIL",
-				timeTaken: `0s`,
-			})
-		}
+		if (!success)
+			return await sendFailureResponse(failureURL, "FETCH_FAIL", startTime)
 
+		/* Step 2: Start the process job */
 		success = await startProcessJob(sessionID)
-		if (!success) {
-			console.error("Error in processing code.")
-			return res.status(500).send({
-				status: 500,
-				message: "PROCESS_JOB_FAIL",
-				timeTaken: `0s`,
-			})
-		}
+		if (!success)
+			return await sendFailureResponse(failureURL, "PROCESS_FAIL", startTime)
 	}
-	const endTime = Date.now()
 
-	res.status(200).send({
-		status: 200,
-		message: "INGESTION SUCCESSFUL",
-		timeTaken: `${(endTime - startTime) / 1000}s`,
-	})
+	await sendSuccessResponse(successURL, startTime)
 }