fixes

Author: Chetan11-dev

js/botasaurus-server-js/src/api-config.ts

@@ -69,27 +69,46 @@ function isWorker() {
 }
 
 /**
- * Check master endpoint health
+ * Check master endpoint health, retrying up to 3 times.
  */
-async function checkMasterHealth(masterEndpoint: string){
-    try {
-        const response = await fetch(`${masterEndpoint}/health`, {
-            method: 'GET',
-            signal: AbortSignal.timeout(10000) // 10 second timeout
-        });
-        
-        if (!response.ok) {
-            throw new Error(`Health check returned status ${response.status}`);
+async function checkMasterHealth(masterEndpoint: string) {
+    const MAX_RETRIES = 3;
+    let retryCount = 0;
+    while (retryCount < MAX_RETRIES) {
+        retryCount++;
+        try {
+            const response = await fetch(`${masterEndpoint}/health`, {
+                method: 'GET',
+                signal: AbortSignal.timeout(10000) // 10 second timeout
+            });
+
+            if (!response.ok) {
+                throw new Error(`Health check returned status ${response.status}`);
+            }
+            if (retryCount > 1) {
+                console.debug(`[K8s] Successfully reached master after retry (attempt ${retryCount}/${MAX_RETRIES}): ${masterEndpoint}`);
+            }
+            return true;
+        } catch (error: any) {
+            console.error(error);
+            console.debug(`[K8s] Health check error (attempt ${retryCount}/${MAX_RETRIES}): ${error instanceof Error ? error.message : error} - ${masterEndpoint}`);
+            if (retryCount < MAX_RETRIES) {
+                // Wait briefly before retrying
+                await sleep(3);
+                continue;
+            }
         }
-        return true;
-    } catch (error: any) {
-        console.error(`[K8s] Cannot reach master at ${masterEndpoint}`);
-        console.error(`[K8s] Error: ${error.message}`);
+    }
+
+    console.error(`[K8s] Failed to reach master after ${MAX_RETRIES} attempts: ${masterEndpoint}`);
+    if (!ApiConfig.isMasterNode()) {
+        // If not master node, exit the process
         process.exit(1);
-        return false;
     }
+    return false;
 }
 
+
 function addCorsHeaders(reply: any) {
     reply.header("Access-Control-Allow-Origin", "*");
     reply.header(
@@ -413,7 +432,11 @@ export function buildApp(
     routeAliases: any,
     enable_cache: boolean
 ): FastifyInstance {
-    const app = fastify({ logger: true });
+    const app = fastify({
+        logger: true,
+        // TODO: change appropriately as needed
+        bodyLimit: 500 * 1024 * 1024 // 500MB
+    });
 
     // Add CORS handling
     app.addHook("onRequest", (request, reply, done) => {

js/botasaurus-server-js/src/ndjson.ts

@@ -29,20 +29,13 @@ export class NDJSONWriteStream {
 
    performWrite(content: any): void | PromiseLike<void> {
     return new Promise((resolve) => {
-      const attemptWrite = (content: string) => {
-        if (!this.writeStream.write(content)) {
-
-          // Wait for the 'drain' event if write returns false
-          this.writeStream.once('drain', () => {
-            //   console.log('Buffer drained. Resuming writing.');
-            //   attemptWrite(); // Retry writing the data
-            resolve() // Resolve the promise if write is successful
-          })
-        } else {
+      if (!this.writeStream.write(content)) {
+        this.writeStream.once('drain', () => {
           resolve() // Resolve the promise if write is successful
-        }
+        })
+      } else {
+        resolve() // Resolve the promise if write is successful
       }
-      attemptWrite(content)
     })
   }