fixes

Author: Chetan11-dev

botasaurus-controls/src/index.ts

@@ -430,7 +430,36 @@ const parseListOfTexts = (value:any)=>{
   }
   return value;
 }
-
+function createHelpfulJsonError(e: any, trimmedValue: string) {
+  const errorMessage = e.message || 'Invalid JSON'
+
+  // Extract position info if available
+  const positionMatch = errorMessage.match(/position\s+(\d+)/i)
+  const position = positionMatch ? parseInt(positionMatch[1], 10) : null
+
+  let helpfulMessage = 'Invalid JSON: '
+
+  // Check for common mistakes
+  if (trimmedValue.includes("'") && !trimmedValue.includes('"')) {
+    helpfulMessage += "Use double quotes (\") instead of single quotes (') for strings."
+  } else if (/,\s*[}\]]/.test(trimmedValue)) {
+    helpfulMessage += "Trailing comma found. Remove the comma before the closing bracket."
+  } else if (errorMessage.includes('Unexpected token')) {
+    if (position !== null) {
+      const contextStart = Math.max(0, position - 10)
+      const contextEnd = Math.min(trimmedValue.length, position + 10)
+      const context = trimmedValue.substring(contextStart, contextEnd)
+      helpfulMessage += `Unexpected character near position ${position}: "...${context}..."`
+    } else {
+      helpfulMessage += errorMessage
+    }
+  } else if (errorMessage.includes('Unexpected end')) {
+    helpfulMessage += "JSON is incomplete. Check for missing closing brackets or quotes."
+  } else {
+    helpfulMessage += errorMessage
+  }
+  return helpfulMessage
+}
 function parseJSON(value: any): { parsed: any; error: string | null } {
   if (typeof value !== 'string') {
     return { parsed: value, error: null };
@@ -446,33 +475,7 @@ function parseJSON(value: any): { parsed: any; error: string | null } {
     return { parsed, error: null };
   } catch (e: any) {
     // Provide helpful error messages based on common JSON mistakes
-    const errorMessage = e.message || 'Invalid JSON';
-    
-    // Extract position info if available
-    const positionMatch = errorMessage.match(/position\s+(\d+)/i);
-    const position = positionMatch ? parseInt(positionMatch[1], 10) : null;
-    
-    let helpfulMessage = 'Invalid JSON: ';
-    
-    // Check for common mistakes
-    if (trimmedValue.includes("'") && !trimmedValue.includes('"')) {
-      helpfulMessage += "Use double quotes (\") instead of single quotes (') for strings.";
-    } else if (/,\s*[}\]]/.test(trimmedValue)) {
-      helpfulMessage += "Trailing comma found. Remove the comma before the closing bracket.";
-    } else if (errorMessage.includes('Unexpected token')) {
-      if (position !== null) {
-        const contextStart = Math.max(0, position - 10);
-        const contextEnd = Math.min(trimmedValue.length, position + 10);
-        const context = trimmedValue.substring(contextStart, contextEnd);
-        helpfulMessage += `Unexpected character near position ${position}: "...${context}..."`;
-      } else {
-        helpfulMessage += errorMessage;
-      }
-    } else if (errorMessage.includes('Unexpected end')) {
-      helpfulMessage += "JSON is incomplete. Check for missing closing brackets or quotes.";
-    } else {
-      helpfulMessage += errorMessage;
-    }
+    let helpfulMessage = createHelpfulJsonError(e, trimmedValue)
 
     return { parsed: null, error: helpfulMessage };
   }
@@ -1204,7 +1207,6 @@ private parse(data: any) {
   }
 }
 
-
 function isInvalidFileType(file: any, acceptedFileTypes: any) {
   const fileExtension = file.name.split(".").pop()?.toLowerCase()
   return !acceptedFileTypes.includes(fileExtension)
@@ -1250,4 +1252,5 @@ function createControls(input_js: string | Function) {
 }
 
 
-export { Controls, createControls, FileTypes }
+export { Controls, createControls, FileTypes }
+

js/botasaurus-server-js/src/api-config.ts

@@ -56,10 +56,6 @@ function isValidUrl(urlString: string): boolean {
     }
 }
 
-/**
- * Parse command line arguments for --master or --worker flags
- */
-
 /**
  * Check master endpoint health, retrying up to 3 times.
  */
@@ -452,9 +448,8 @@ export function buildApp(
     enable_cache: boolean
 ): FastifyInstance {
     const app = fastify({
-        logger: true,
-        // TODO: change appropriately as needed
-        bodyLimit: 500 * 1024 * 1024 // 500MB
+        logger: !isMaster,
+        bodyLimit: 250 * 1024 * 1024 // 250MB (2.5× headroom for 100MB chunks)
     });
 
     // Add CORS handling

js/botasaurus-server-js/src/download.ts

@@ -1,4 +1,4 @@
-import {writeExcelStreamed, writeJsonStreamed, writeJson, writeCsvStreamed, writeCsv, writeExcel } from './writer'; // Fixed import statement
+import {writeExcelStreamed, writeJsonStreamed, writeJson, writeCsvStreamed, writeCsv, writeExcel, writeNdjson } from './writer'; // Fixed import statement
 import writterHttp from './writer-http'; // Fixed import statement
 import { getPathToDownloadsDirectory } from './paths'
 import { TaskResults } from './task-results'
@@ -27,6 +27,9 @@ export function downloadResults(results: Array<Record<string, any>>, fmt: string
         } else {
             return writeExcel(results, filePath);
         }        
+    } else if (fmt === "ndjson") {
+        // Files are already stored as NDJSON, so just copy the file
+        return writeNdjson(TaskResults.generateTaskFilePath(taskId), filePath);
     }
 
     throw new Error("Unsupported format");
@@ -63,6 +66,12 @@ export async function downloadResultsHttp(reply: any, results: Array<Record<stri
         } else {
             return writterHttp.writeExcel(results, reply.raw);
         }     
+    } else if (fmt === "ndjson") {
+        headers["Content-Type"] = "application/x-ndjson";
+        headers["Content-Disposition"] = `attachment; filename="${filename}.ndjson"`;
+        reply.raw.writeHead(200, headers);
+        // Files are already stored as NDJSON, so stream directly
+        return writterHttp.writeNdjson(TaskResults.generateTaskFilePath(taskId), reply.raw);
     } else {
         throw new Error("Unsupported format");
     }

js/botasaurus-server-js/src/models.ts

@@ -181,6 +181,40 @@ export function serializeTaskForRunTask(obj: Task){
     result_count: obj.result_count,
   };
 }
+
+function sortDictByKeys(dictionary: Record<string, any>, keys: string[]): Record<string, any> {
+  const newDict: Record<string, any> = {};
+  const extraKeys: string[] = [];
+  
+  try {
+    // Create a Set from keys for O(1) lookups
+    const keysSet = new Set(keys);
+    
+    // Check for any keys in dictionary that aren't in the keys list
+    for (const key in dictionary) {
+      if (!keysSet.has(key)) {
+        extraKeys.push(key);
+      }
+    }
+    
+    // If we found extra keys, raise an error
+    if (extraKeys.length > 0) {
+      throw new Error(`Found keys in dictionary that weren't in keys list: ${extraKeys.join(', ')}`);
+    }
+    
+    // Add all keys from the provided keys list
+    for (const key of keys) {
+      newDict[key] = dictionary[key];
+    }
+  } catch (error) {
+    if (error instanceof Error) {
+      throw error;
+    }
+    throw new Error("Failed to sort dict by keys");
+  }
+  return newDict;
+}
+
 async function serializeTask(obj: Task, withResult: boolean): Promise<any> {
   const taskId = obj.id;
   const status = obj.status;
@@ -210,10 +244,12 @@ async function serializeTask(obj: Task, withResult: boolean): Promise<any> {
     updated_at: isoformat(obj.updated_at),
   };
 }
+export type StatusKind = typeof TaskStatus[keyof typeof TaskStatus]
+
 // Task model
 class Task {
     id!: number;
-    status!: string;
+    status!: StatusKind;
     sort_id!: number;
     task_name!: string;
     scraper_name!: string;
@@ -248,7 +284,7 @@ class Task {
 
     return new Task(x)
   }
-  export { getAutoincrementId, createTask, db, Task, TaskStatus, removeDuplicatesByKey, calculateDuration, isoformat, serializeUiOutputTask, serializeUiDisplayTask, serializeTask , initAutoIncrementDb};
+  export { getAutoincrementId, createTask, db, Task, TaskStatus, removeDuplicatesByKey, calculateDuration, isoformat, serializeUiOutputTask, serializeUiDisplayTask, serializeTask, sortDictByKeys, initAutoIncrementDb };