Merge pull request #100 from brightdata/feat/polling-configuration

feat: add configurable timeout

Author: meirk-brd

README.md

@@ -398,13 +398,32 @@ Try the Web MCP without any setup:
         "PRO_MODE": "true",              // Enable all 60+ tools
         "RATE_LIMIT": "100/1h",          // Custom rate limiting
         "WEB_UNLOCKER_ZONE": "custom",   // Custom unlocker zone
-        "BROWSER_ZONE": "custom_browser" // Custom browser zone
+        "BROWSER_ZONE": "custom_browser", // Custom browser zone
+        "POLLING_TIMEOUT": "600"         // Polling timeout in seconds (default: 600)
       }
     }
   }
 }
 ```
 
+### Environment Variables
+
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `API_TOKEN` | Your Bright Data API token (required) | - | `your-token-here` |
+| `PRO_MODE` | Enable all 60+ tools | `false` | `true` |
+| `RATE_LIMIT` | Custom rate limiting | unlimited | `100/1h`, `50/30m` |
+| `WEB_UNLOCKER_ZONE` | Custom Web Unlocker zone name | `mcp_unlocker` | `my_custom_zone` |
+| `BROWSER_ZONE` | Custom Browser zone name | `mcp_browser` | `my_browser_zone` |
+| `POLLING_TIMEOUT` | Timeout for web_data_* tools polling (seconds) | `600` | `300`, `1200` |
+| `GROUPS` | Comma-separated tool group IDs | - | `ecommerce,browser` |
+| `TOOLS` | Comma-separated individual tool names | - | `extract,scrape_as_html` |
+
+**Notes:**
+- `POLLING_TIMEOUT` controls how long web_data_* tools wait for results. Each second = 1 polling attempt.
+- Lower values (e.g., 300) will fail faster on slow data collections.
+- Higher values (e.g., 1200) allow more time for complex scraping tasks.
+
 ---
 
 ## 📚 Documentation

server.js

@@ -14,6 +14,7 @@ const api_token = process.env.API_TOKEN;
 const unlocker_zone = process.env.WEB_UNLOCKER_ZONE || 'mcp_unlocker';
 const browser_zone = process.env.BROWSER_ZONE || 'mcp_browser';
 const pro_mode = process.env.PRO_MODE === 'true';
+const polling_timeout = parseInt(process.env.POLLING_TIMEOUT || '600', 10);
 const pro_mode_tools = ['search_engine', 'scrape_as_markdown',
     'search_engine_batch', 'scrape_batch'];
 const tool_groups = process.env.GROUPS ?
@@ -899,7 +900,7 @@ for (let {dataset_id, id, description, inputs, defaults = {}, fixed_values = {}}
             let snapshot_id = trigger_response.data.snapshot_id;
             console.error(`[${tool_name}] triggered collection with `
                 +`snapshot ID: ${snapshot_id}`);
-            let max_attempts = 600;
+            let max_attempts = polling_timeout;
             let attempts = 0;
             while (attempts < max_attempts)
             {

server.json

@@ -58,6 +58,13 @@
           "isRequired": false,
           "isSecret": false,
           "format": "string"
+        },
+        {
+          "name": "POLLING_TIMEOUT",
+          "description": "Polling timeout in seconds for web_data_* tools (default: 600)",
+          "isRequired": false,
+          "isSecret": false,
+          "format": "number"
         }
       ]
     }