Skip to content

Commit c42c3a5

Browse files
coder-hxlcoder-hxl
authored
Merge pull request #101 from coder-hxl/embracingAI
Embracing AI
2 parents 3a2a0d0 + 357b2b3 commit c42c3a5

File tree

165 files changed

+11278
-6819
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

165 files changed

+11278
-6819
lines changed

‎.eslintrc.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
module.exports = {
1+
export default {
22
env: {
33
node: true,
44
es2021: true

‎.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,8 @@ coverage
66

77
upload
88

9-
test/start/index.js
9+
test/dev/index.js
1010
test/server/index.js
11+
12+
docs/.vitepress/cache
13+
docs/.vitepress/dist

‎README.md

Lines changed: 50 additions & 2093 deletions
Large diffs are not rendered by default.

‎assets/example.png

780 KB
Loading

‎assets/run-example.gif

-2.52 MB
Binary file not shown.

‎docs/.vitepress/config.ts

Lines changed: 381 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,381 @@
1+
import { defineConfig } from 'vitepress'
2+
3+
// https://vitepress.dev/reference/site-config
4+
export default defineConfig({
5+
title: 'x-crawl',
6+
7+
head: [['link', { rel: 'icon', type: 'image/svg+xml', href: '/logo.svg' }]],
8+
9+
cleanUrls: true,
10+
lastUpdated: true,
11+
12+
themeConfig: {
13+
// https://vitepress.dev/reference/default-theme-config
14+
logo: '/logo.svg',
15+
16+
search: {
17+
provider: 'local'
18+
},
19+
20+
socialLinks: [
21+
{ icon: 'github', link: 'https://github.com/coder-hxl/x-crawl' },
22+
{ icon: 'discord', link: 'https://discord.gg/SF7aaebg4E' }
23+
]
24+
},
25+
26+
locales: {
27+
root: {
28+
label: 'English',
29+
lang: 'en',
30+
description: 'Flexible Node.js AI-assisted crawler library',
31+
32+
themeConfig: {
33+
nav: [
34+
{ text: 'Guide', link: '/guide/index' },
35+
{ text: 'API', link: '/api/index' },
36+
{ text: 'Type', link: '/type/index' },
37+
{
38+
text: 'About',
39+
items: [
40+
{ text: 'FAQ', link: '/about/faq' },
41+
{ text: 'Community', link: '/about/community' },
42+
{ text: 'Releases', link: '/about/releases' },
43+
{ text: 'Old documents', link: '/about/old-docs' },
44+
{ text: 'Issues', link: '/about/issues' },
45+
{ text: 'Announcements', link: '/about/announcements' }
46+
]
47+
}
48+
],
49+
50+
search: {
51+
provider: 'local'
52+
},
53+
54+
sidebar: {
55+
'/guide': [
56+
{
57+
text: 'Getting Started',
58+
items: [
59+
{ text: 'Introduction', link: '/guide/index' },
60+
{ text: 'Quick Start', link: '/guide/quick-start' }
61+
]
62+
},
63+
{
64+
text: 'Essentials',
65+
items: [
66+
{
67+
text: 'Create Application',
68+
link: '/guide/create-crawl-application'
69+
},
70+
{ text: 'Crawl Page', link: '/guide/crawl-page' },
71+
{ text: 'Crawl HTML', link: '/guide/crawl-html' },
72+
{ text: 'Crawl Data', link: '/guide/crawl-data' },
73+
{ text: 'Crawl File', link: '/guide/crawl-file' },
74+
{ text: 'Interval Time', link: '/guide/interval' },
75+
{ text: 'Retry', link: '/guide/retry' },
76+
{ text: 'Proxy', link: '/guide/proxy' },
77+
{ text: 'Priority Crawl', link: '/guide/priority' },
78+
{ text: 'Terminal Prompt', link: '/guide/reporters' },
79+
{ text: 'About the Results', link: '/guide/results' },
80+
{ text: 'TypeScript', link: '/guide/typescript' }
81+
]
82+
},
83+
{
84+
text: 'AI Assisted',
85+
items: [
86+
{
87+
text: 'Creating AI applications',
88+
link: '/guide/create-ai-application'
89+
},
90+
{
91+
text: 'Intelligent on-demand analysis elements',
92+
link: '/guide/parse-elements'
93+
},
94+
{
95+
text: 'Smartly generated element selectors',
96+
link: '/guide/get-element-selectors'
97+
},
98+
{
99+
text: 'Intelligent reply to crawler questions',
100+
link: '/guide/crawl-openai-help'
101+
},
102+
{
103+
text: 'User-defined AI functions',
104+
link: '/guide/crawl-openai-custom'
105+
}
106+
]
107+
},
108+
{
109+
text: 'Advance',
110+
items: [
111+
{ text: 'Crawl mode', link: '/guide/crawl-mode' },
112+
{
113+
text: 'device fingerprinting',
114+
link: '/guide/fingerprint'
115+
},
116+
{ text: 'configuration', link: '/guide/config' }
117+
]
118+
}
119+
],
120+
121+
'/api': [
122+
{
123+
text: 'crawler',
124+
items: [
125+
{ text: 'createCrawl', link: '/api/index' },
126+
{ text: 'crawlPage', link: '/api/crawl-page' },
127+
{ text: 'crawlHTML', link: '/api/crawl-html' },
128+
{ text: 'crawlData', link: '/api/crawl-data' },
129+
{ text: 'crawlFile', link: '/api/crawl-file' }
130+
]
131+
},
132+
{
133+
text: 'AI',
134+
items: [
135+
{
136+
text: 'createCrawlOpenAI',
137+
link: '/api/create-crawl-openai'
138+
},
139+
{ text: 'parseElements', link: '/api/parse-elements' },
140+
{
141+
text: 'getElementSelectors',
142+
link: '/api/get-element-selectors'
143+
},
144+
{ text: 'help', link: '/api/help' },
145+
{ text: 'custom', link: '/api/custom' }
146+
]
147+
}
148+
],
149+
150+
'/type': [
151+
{
152+
text: 'crawler',
153+
items: [
154+
{ text: 'createCrawl', link: '/type/index' },
155+
{ text: 'crawlPage', link: '/type/crawl-page' },
156+
{ text: 'crawlHTML', link: '/type/crawl-html' },
157+
{ text: 'crawlData', link: '/type/crawl-data' },
158+
{ text: 'crawlFile', link: '/type/crawl-file' },
159+
{
160+
text: 'CrawlOtherConfig',
161+
link: '/type/crawl-other-config'
162+
}
163+
]
164+
},
165+
{
166+
text: 'AI',
167+
items: [
168+
{
169+
text: 'createCrawlOpenAI',
170+
link: '/type/create-crawl-openai'
171+
},
172+
{ text: 'parseElements', link: '/type/parse-elements' },
173+
{
174+
text: 'getElementSelectors',
175+
link: '/type/get-element-selectors'
176+
},
177+
{
178+
text: 'CrawlOpenaiOtherConfig',
179+
link: '/type/crawl-openai-other-config'
180+
}
181+
]
182+
}
183+
]
184+
},
185+
186+
footer: {
187+
message: 'Released under the MIT license',
188+
copyright: 'Copyright © 2024-present CoderHXL. All rights reserved'
189+
},
190+
191+
editLink: {
192+
pattern: 'https://github.com/coder-hxl/x-crawl/edit/main/docs/:path',
193+
text: 'Suggest changes to this page'
194+
}
195+
}
196+
},
197+
198+
cn: {
199+
label: '简体中文',
200+
lang: 'cn',
201+
link: '/cn/',
202+
description: '灵活的 Node.js AI 辅助爬虫库',
203+
204+
themeConfig: {
205+
// https://vitepress.dev/reference/default-theme-config
206+
nav: [
207+
{ text: '指南', link: '/cn/guide/index' },
208+
{ text: 'API', link: '/cn/api/index' },
209+
{ text: 'Type', link: '/cn/type/index' },
210+
{
211+
text: '关于',
212+
items: [
213+
{ text: '常见问题', link: '/cn/about/faq' },
214+
{ text: '社区', link: '/cn/about/community' },
215+
{ text: '版本发布', link: '/cn/about/releases' },
216+
{ text: '旧版本文档', link: '/cn/about/old-docs' },
217+
{ text: 'Issues', link: '/cn/about/issues' },
218+
{ text: '注意事项', link: '/cn/about/announcements' }
219+
]
220+
}
221+
],
222+
223+
search: {
224+
provider: 'local',
225+
options: {
226+
translations: {
227+
button: {
228+
buttonText: '搜索文档',
229+
buttonAriaLabel: '搜索文档'
230+
},
231+
modal: {
232+
noResultsText: '无法找到相关结果',
233+
resetButtonTitle: '清除查询条件',
234+
footer: {
235+
selectText: '选择',
236+
navigateText: '切换'
237+
}
238+
}
239+
}
240+
}
241+
},
242+
243+
sidebar: {
244+
'/cn/guide': [
245+
{
246+
text: '开始',
247+
items: [
248+
{ text: '简介', link: '/cn/guide/index' },
249+
{ text: '快速上手', link: '/cn/guide/quick-start' }
250+
]
251+
},
252+
{
253+
text: '基础',
254+
items: [
255+
{
256+
text: '创建应用',
257+
link: '/cn/guide/create-crawl-application'
258+
},
259+
{ text: '爬取页面', link: '/cn/guide/crawl-page' },
260+
{ text: '爬取 HTML', link: '/cn/guide/crawl-html' },
261+
{ text: '爬取接口', link: '/cn/guide/crawl-data' },
262+
{ text: '爬取文件', link: '/cn/guide/crawl-file' },
263+
{ text: '间隔时间', link: '/cn/guide/interval' },
264+
{ text: '失败重试', link: '/cn/guide/retry' },
265+
{ text: '轮换代理', link: '/cn/guide/proxy' },
266+
{ text: '优先队列', link: '/cn/guide/priority' },
267+
{ text: '终端信息', link: '/cn/guide/reporters' },
268+
{ text: '关于结果', link: '/cn/guide/results' },
269+
{ text: 'TypeScript', link: '/cn/guide/typescript' }
270+
]
271+
},
272+
{
273+
text: 'AI 辅助',
274+
items: [
275+
{
276+
text: '创建 AI 应用',
277+
link: '/cn/guide/create-ai-application'
278+
},
279+
{ text: '智能按需分析元素', link: '/cn/guide/parse-elements' },
280+
{
281+
text: '智能生成元素选择器',
282+
link: '/cn/guide/get-element-selectors'
283+
},
284+
{
285+
text: '智能回复爬虫问题',
286+
link: '/cn/guide/crawl-openai-help'
287+
},
288+
{
289+
text: '用户自定义 AI 功能',
290+
link: '/cn/guide/crawl-openai-custom'
291+
}
292+
]
293+
},
294+
{
295+
text: '进阶',
296+
items: [
297+
{ text: '爬取模式', link: '/cn/guide/crawl-mode' },
298+
{ text: '设备指纹', link: '/cn/guide/fingerprint' },
299+
{ text: '配置', link: '/cn/guide/config' }
300+
]
301+
}
302+
],
303+
304+
'/cn/api': [
305+
{
306+
text: '爬虫',
307+
items: [
308+
{ text: 'createCrawl', link: '/cn/api/index' },
309+
{ text: 'crawlPage', link: '/cn/api/crawl-page' },
310+
{ text: 'crawlHTML', link: '/cn/api/crawl-html' },
311+
{ text: 'crawlData', link: '/cn/api/crawl-data' },
312+
{ text: 'crawlFile', link: '/cn/api/crawl-file' }
313+
]
314+
},
315+
{
316+
text: 'AI',
317+
items: [
318+
{
319+
text: 'createCrawlOpenAI',
320+
link: '/cn/api/create-crawl-openai'
321+
},
322+
{ text: 'parseElements', link: '/cn/api/parse-elements' },
323+
{
324+
text: 'getElementSelectors',
325+
link: '/cn/api/get-element-selectors'
326+
},
327+
{ text: 'help', link: '/cn/api/help' },
328+
{ text: 'custom', link: '/cn/api/custom' }
329+
]
330+
}
331+
],
332+
333+
'/cn/type': [
334+
{
335+
text: '爬虫',
336+
items: [
337+
{ text: 'createCrawl', link: '/cn/type/index' },
338+
{ text: 'crawlPage', link: '/cn/type/crawl-page' },
339+
{ text: 'crawlHTML', link: '/cn/type/crawl-html' },
340+
{ text: 'crawlData', link: '/cn/type/crawl-data' },
341+
{ text: 'crawlFile', link: '/cn/type/crawl-file' },
342+
{
343+
text: 'CrawlOtherConfig',
344+
link: '/cn/type/crawl-other-config'
345+
}
346+
]
347+
},
348+
{
349+
text: 'AI',
350+
items: [
351+
{
352+
text: 'createCrawlOpenAI',
353+
link: '/cn/type/create-crawl-openai'
354+
},
355+
{ text: 'parseElements', link: '/cn/type/parse-elements' },
356+
{
357+
text: 'getElementSelectors',
358+
link: '/cn/type/get-element-selectors'
359+
},
360+
{
361+
text: 'CrawlOpenaiOtherConfig',
362+
link: '/cn/type/crawl-openai-other-config'
363+
}
364+
]
365+
}
366+
]
367+
},
368+
369+
footer: {
370+
message: '基于 MIT 许可发布',
371+
copyright: '版权所有 © 2024-present CoderHXL'
372+
},
373+
374+
editLink: {
375+
pattern: 'https://github.com/coder-hxl/x-crawl/edit/main/docs/:path',
376+
text: '为此页提供修改建议'
377+
}
378+
}
379+
}
380+
}
381+
})

0 commit comments

Comments
 (0)