fixes

Author: Chetan11-dev

botasaurus/output.py

@@ -63,7 +63,7 @@ def read_json(filename):
 
 
 
-def write_temp_json(data, log=True):
+def write_temp_json(data, log=True, indent=4):
     filename = "temp"
 
     try:
@@ -73,7 +73,7 @@ def write_temp_json(data, log=True):
         if not filename.endswith(".json"):
             filename = filename + ".json"
 
-        _write_json(data, filename)
+        _write_json(data, filename, indent)
 
         if log:
             print(f"View written JSON file at {filename}")
@@ -99,7 +99,7 @@ def file_exists(filename):
         return False
 
 
-def write_json(data, filename, log=True):
+def write_json(data, filename, log=True, indent=4):
     # if type(data) is list and len(data) == 0:
     #     # if log:
     #     print("No JSON File written as data list is empty.")
@@ -112,7 +112,7 @@ def write_json(data, filename, log=True):
         if not filename.endswith(".json"):
             filename = filename + ".json"
 
-        _write_json(data, filename)
+        _write_json(data, filename, indent)
 
         if log:
             print(f"View written JSON file at {filename}")

docs/docs/botasaurus-desktop/enhancing-scrapers/input-controls.md

@@ -146,8 +146,9 @@ To use `searchMethod`, you need to define async handler functions and add them u
 ```ts
 import { Server } from 'botasaurus-server/server';
 import axios from 'axios';
+import { config } from '../../main/config';
 
-const baseUrl = "http://0.0.0.0:3000";
+const baseUrl = config.isDev ? "http://0.0.0.0:3000" : "https://api.my-app.com";
 
 async function getCityOptions(query: string, data: any) {
   const params = {

js/botasaurus-server-js/src/api-config.ts

@@ -78,7 +78,7 @@ function addScraperRoutes(app: FastifyInstance, apiBasePath: string) {
                 // If split_task exists, split the data
                 let dataItems: any[];
                 if (splitTask) {
-                    dataItems = splitTask(validatedData);
+                    dataItems = await splitTask(validatedData);
                 } else {
                     dataItems = [validatedData];
                 }

js/botasaurus-server-js/src/routes-db-logic.ts

@@ -366,7 +366,7 @@ async function createTasks(
 
   let tasksData: any[];
   if (split_task) {
-    tasksData = split_task(deepCloneDict(data));
+    tasksData = await split_task(deepCloneDict(data));
     if (tasksData.length === 0) {
       return [[], [], split_task];
     }

js/botasaurus-server-js/src/server.ts

@@ -513,11 +513,11 @@ class _Server {
     this.cache = true;
   }
 
-  createTasks({ scraperName, data, metadata }: {
+  async createTasks({ scraperName, data, metadata }: {
     scraperName: string;
     data: any;
     metadata: any;
-  }): [any[], boolean, boolean] {
+  }): Promise<[any[], boolean, boolean]> {
     const scraper = this.scrapers[scraperName];
 
     const tasks: any[] = [];
@@ -526,7 +526,7 @@ class _Server {
     const splitTask = scraper.split_task;
 
     if (splitTask) {
-      const splitData = splitTask(data);
+      const splitData = await splitTask(data);
       for (const item of splitData) {
         const taskName = scraper.get_task_name ? scraper.get_task_name(item) : 'Unnamed Task';
         tasks.push({ name: taskName, data: item, metadata });

setup.py

@@ -33,7 +33,7 @@ def get_description():
 setup(
     name="botasaurus",
     packages=["botasaurus"],
-    version='4.0.94',
+    version='4.0.95',
     license="MIT",
     project_urls={
         "Documentation": "https://omkar.cloud/botasaurus/",