|
2 | 2 |
|
3 | 3 | <div align="center"> |
4 | 4 |
|
5 | | -**Official Node.js/TypeScript Client for Thordata APIs** |
| 5 | +<img src="https://img.shields.io/badge/Thordata-AI%20Infrastructure-blue?style=for-the-badge" alt="Thordata Logo"> |
6 | 6 |
|
7 | | -_Proxy Network • SERP API • Web Unlocker • Web Scraper API_ |
| 7 | +**The Official Node.js/TypeScript Client for Thordata APIs** |
8 | 8 |
|
9 | | -[](https://www.npmjs.com/package/thordata-js-sdk) |
10 | | -[](LICENSE) |
| 9 | +*Proxy Network • SERP API • Web Unlocker • Web Scraper API* |
| 10 | + |
| 11 | +[](https://www.npmjs.com/package/thordata-js-sdk) |
| 12 | +[](LICENSE) |
| 13 | +[](https://github.com/Thordata/thordata-js-sdk/actions) |
11 | 14 |
|
12 | 15 | </div> |
13 | 16 |
|
14 | 17 | --- |
15 | 18 |
|
| 19 | +## 📖 Introduction |
| 20 | + |
| 21 | +A fully typed TypeScript SDK for Thordata, optimized for Node.js environments. It provides seamless integration with Thordata's proxy network and scraping APIs. |
| 22 | + |
| 23 | +**Key Features:** |
| 24 | +* **🔒 Type-Safe:** Written in TypeScript with complete definitions. |
| 25 | +* **🌐 Modern:** Uses `axios` and standard `https-proxy-agent` for reliable connectivity. |
| 26 | +* **⚡ Lazy Validation:** Zero-config initialization; only provide credentials for the features you use. |
| 27 | +* **🛡️ Proxy Support:** Full support for HTTPS and SOCKS5h protocols with authentication. |
| 28 | + |
| 29 | +--- |
| 30 | + |
16 | 31 | ## 📦 Installation |
17 | 32 |
|
18 | 33 | ```bash |
19 | 34 | npm install thordata-js-sdk |
| 35 | +# or |
| 36 | +yarn add thordata-js-sdk |
20 | 37 | ``` |
21 | 38 |
|
| 39 | +--- |
| 40 | + |
22 | 41 | ## 🔐 Configuration |
23 | 42 |
|
24 | | -Set environment variables: |
| 43 | +We recommend using `dotenv` to manage credentials. |
25 | 44 |
|
26 | 45 | ```bash |
27 | | -export THORDATA_SCRAPER_TOKEN="your_token" |
28 | | -export THORDATA_PUBLIC_TOKEN="your_public_token" |
29 | | -export THORDATA_PUBLIC_KEY="your_public_key" |
| 46 | +# .env file |
| 47 | +THORDATA_SCRAPER_TOKEN=your_token |
| 48 | +THORDATA_RESIDENTIAL_USERNAME=your_username |
| 49 | +THORDATA_RESIDENTIAL_PASSWORD=your_password |
| 50 | +THORDATA_PROXY_HOST=vpnXXXX.pr.thordata.net |
30 | 51 | ``` |
31 | 52 |
|
| 53 | +--- |
| 54 | + |
32 | 55 | ## 🚀 Quick Start |
33 | 56 |
|
| 57 | +### 1. SERP Search |
| 58 | + |
34 | 59 | ```typescript |
35 | | -import { Thordata } from "thordata-js-sdk"; |
| 60 | +import { ThordataClient, Engine } from "thordata-js-sdk"; |
36 | 61 |
|
37 | | -// Initialize (reads from env vars) |
38 | | -const client = new Thordata(); |
| 62 | +const client = new ThordataClient({}); // Auto-loads from env |
39 | 63 |
|
40 | | -async function main() { |
41 | | - // SERP Search |
42 | | - const results = await client.serpSearch({ |
43 | | - query: "nodejs", |
44 | | - engine: "google", |
| 64 | +async function search() { |
| 65 | + const result = await client.serpSearch({ |
| 66 | + query: "SpaceX launch", |
| 67 | + engine: Engine.GOOGLE_NEWS, |
45 | 68 | country: "us", |
| 69 | + num: 5 |
46 | 70 | }); |
47 | | - console.log(results.organic?.[0]?.title); |
| 71 | + |
| 72 | + console.log(result.news_results); |
48 | 73 | } |
49 | 74 |
|
50 | | -main(); |
| 75 | +search(); |
51 | 76 | ``` |
52 | 77 |
|
53 | | -## 📚 Core Features |
| 78 | +### 2. Universal Scrape (Web Unlocker) |
54 | 79 |
|
55 | | -### 🌐 Proxy Network |
| 80 | +```typescript |
| 81 | +async function scrape() { |
| 82 | + const html = await client.universalScrape({ |
| 83 | + url: "https://www.g2.com/products/thordata", |
| 84 | + jsRender: true, |
| 85 | + waitFor: ".reviews-list", |
| 86 | + country: "us" |
| 87 | + }); |
| 88 | + |
| 89 | + console.log("Page HTML length:", html.length); |
| 90 | +} |
| 91 | +``` |
56 | 92 |
|
57 | | -Build proxy URLs for `axios`, `fetch`, `puppeteer`, etc. |
| 93 | +### 3. Using the Proxy Network |
58 | 94 |
|
59 | 95 | ```typescript |
60 | 96 | import { Thordata } from "thordata-js-sdk"; |
61 | 97 |
|
62 | | -// Create proxy config |
63 | | -const proxy = Thordata.Proxy.residentialFromEnv().country("jp").city("tokyo").sticky(30); // 30 min session |
| 98 | +// Create a targeted proxy config |
| 99 | +const proxy = Thordata.Proxy.residentialFromEnv() |
| 100 | + .country("gb") |
| 101 | + .city("london") |
| 102 | + .sticky(10); // 10 minutes session |
64 | 103 |
|
65 | | -// Get URL string |
66 | | -console.log(proxy.toProxyUrl()); |
| 104 | +const client = new Thordata(); |
67 | 105 |
|
68 | | -// Use with internal client |
69 | | -const response = await client.request("https://httpbin.org/ip", { proxy }); |
| 106 | +// Request uses the proxy automatically |
| 107 | +const response = await client.request("https://ipinfo.io/json", { proxy }); |
70 | 108 | console.log(response); |
71 | 109 | ``` |
72 | 110 |
|
73 | | -### 🔍 SERP API |
74 | | - |
75 | | -```typescript |
76 | | -import { Engine } from "thordata-js-sdk"; |
77 | | - |
78 | | -const news = await client.serpSearch({ |
79 | | - query: "SpaceX", |
80 | | - engine: Engine.GOOGLE_NEWS, |
81 | | - num: 20, |
82 | | - country: "us", |
83 | | - language: "en", |
84 | | -}); |
85 | | -``` |
86 | | - |
87 | | -### 🔓 Universal Scraping API (Web Unlocker) |
| 111 | +--- |
88 | 112 |
|
89 | | -```typescript |
90 | | -const html = await client.universalScrape({ |
91 | | - url: "https://example.com/spa", |
92 | | - jsRender: true, |
93 | | - waitFor: ".loaded-content", |
94 | | - blockResources: "image,media", |
95 | | -}); |
96 | | -``` |
| 113 | +## ⚙️ Advanced Usage |
97 | 114 |
|
98 | | -### 🕷️ Web Scraper API (Tasks) |
| 115 | +### Task Management (Async) |
99 | 116 |
|
100 | 117 | ```typescript |
101 | | -// 1. Create Task |
| 118 | +// Create a scraping task |
102 | 119 | const taskId = await client.createScraperTask({ |
103 | | - fileName: "task_1", |
| 120 | + fileName: "task_001", |
104 | 121 | spiderId: "universal", |
105 | 122 | spiderName: "universal", |
106 | | - parameters: { url: "https://example.com" }, |
| 123 | + parameters: { url: "https://example.com" } |
107 | 124 | }); |
108 | 125 |
|
109 | | -// 2. Wait |
| 126 | +console.log(`Task ${taskId} created. Waiting...`); |
| 127 | + |
| 128 | +// Poll for completion |
110 | 129 | const status = await client.waitForTask(taskId); |
111 | 130 |
|
112 | | -// 3. Download |
113 | 131 | if (status === "ready") { |
114 | | - const url = await client.getTaskResult(taskId); |
115 | | - console.log(url); |
116 | | -} |
117 | | -``` |
118 | | - |
119 | | -### 📊 Account Management |
120 | | - |
121 | | -```typescript |
122 | | -// Usage Stats |
123 | | -const stats = await client.getUsageStatistics("2024-01-01", "2024-01-31"); |
124 | | - |
125 | | -// Manage Whitelist |
126 | | -await client.addWhitelistIp("1.2.3.4"); |
127 | | - |
128 | | -// Check ISP Proxies |
129 | | -const servers = await client.listProxyServers(1); // 1=ISP |
130 | | -``` |
131 | | - |
132 | | -## ⚙️ Advanced Usage |
133 | | - |
134 | | -### Error Handling |
135 | | - |
136 | | -The SDK throws typed errors for better control. |
137 | | - |
138 | | -```typescript |
139 | | -import { ThordataRateLimitError, ThordataAuthError } from "thordata-js-sdk"; |
140 | | - |
141 | | -try { |
142 | | - await client.serpSearch({ ... }); |
143 | | -} catch (e) { |
144 | | - if (e instanceof ThordataRateLimitError) { |
145 | | - console.log(`Rate limited! Retry after ${e.retryAfter}s`); |
146 | | - } else if (e instanceof ThordataAuthError) { |
147 | | - console.log("Check your tokens!"); |
148 | | - } |
| 132 | + const downloadUrl = await client.getTaskResult(taskId); |
| 133 | + console.log("Result:", downloadUrl); |
149 | 134 | } |
150 | 135 | ``` |
151 | 136 |
|
152 | | -### Configuration Options |
153 | | - |
154 | | -```typescript |
155 | | -const client = new ThordataClient({ |
156 | | - scraperToken: "...", |
157 | | - timeoutMs: 60000, |
158 | | - maxRetries: 3, // Auto-retry on 429/5xx |
159 | | -}); |
160 | | -``` |
| 137 | +--- |
161 | 138 |
|
162 | 139 | ## 📄 License |
163 | 140 |
|
164 | | -MIT License |
| 141 | +MIT License. |
0 commit comments