1
1
import OpenAI , { ClientOptions } from 'openai'
2
+ import ora from 'ora'
2
3
3
4
import {
4
5
PARSE_ELEMENTS_CONTEXT ,
5
6
GET_ELEMENT_SELECTORS_CONTEXT ,
6
7
HELP_CONTEXT
7
8
} from './context'
8
- import { isObject , log , logStart , logSuccess } from '../shared'
9
+ import { isObject } from '../shared'
9
10
10
11
type OpenAIChatModel =
11
12
| 'gpt-4-0125-preview'
@@ -26,75 +27,75 @@ type OpenAIChatModel =
26
27
| 'gpt-3.5-turbo-0125'
27
28
| 'gpt-3.5-turbo-16k-0613'
28
29
29
- interface CreateXCrawlOpenAIConfig {
30
+ interface CreateCrawlOpenAIConfig {
30
31
defaultModel ?: {
31
32
chatModel : OpenAIChatModel
32
33
}
33
34
clientOptions ?: ClientOptions
34
35
}
35
36
36
- interface XCrawlOpenAICommonAPIOtherOption {
37
+ interface CrawlOpenAICommonAPIOtherOption {
37
38
model ?: OpenAIChatModel
38
39
}
39
40
40
- interface XCrawlOpenAIRunChatOption {
41
+ interface CrawlOpenAIRunChatOption {
41
42
model : OpenAIChatModel | undefined
42
43
context : string
43
44
HTMLContent : string
44
45
userContent : string
45
46
responseFormatType : 'text' | 'json_object'
46
47
}
47
48
48
- interface XCrawlOpenAIParseElementsContentOptions {
49
+ interface CrawlOpenAIParseElementsContentOptions {
49
50
message : string
50
51
}
51
52
52
- interface XCrawlOpenAIGetElementSelectorsContentOptions {
53
+ interface CrawlOpenAIGetElementSelectorsContentOptions {
53
54
message : string
54
55
pathMode : 'default' | 'strict'
55
56
}
56
57
57
- interface XCrawlOpenAIGetElementSelectorsResult {
58
+ interface CrawlOpenAIGetElementSelectorsResult {
58
59
selectors : string
59
60
type : 'single' | 'multiple' | 'none'
60
61
}
61
62
62
- interface XCrawlOpenAIParseElementsResult < T extends Record < string , string > > {
63
+ interface CrawlOpenAIParseElementsResult < T extends Record < string , string > > {
63
64
elements : T [ ]
64
65
type : 'single' | 'multiple' | 'none'
65
66
}
66
67
67
- interface XCrawlOpenAIApp {
68
+ interface CrawlOpenAIApp {
68
69
parseElements < T extends Record < string , string > > (
69
70
HTML : string ,
70
- content : string | XCrawlOpenAIParseElementsContentOptions ,
71
- option ?: XCrawlOpenAICommonAPIOtherOption
72
- ) : Promise < XCrawlOpenAIParseElementsResult < T > >
71
+ content : string | CrawlOpenAIParseElementsContentOptions ,
72
+ option ?: CrawlOpenAICommonAPIOtherOption
73
+ ) : Promise < CrawlOpenAIParseElementsResult < T > >
73
74
74
75
getElementSelectors (
75
76
HTML : string ,
76
- content : string | XCrawlOpenAIGetElementSelectorsContentOptions ,
77
- option ?: XCrawlOpenAICommonAPIOtherOption
78
- ) : Promise < XCrawlOpenAIGetElementSelectorsResult >
77
+ content : string | CrawlOpenAIGetElementSelectorsContentOptions ,
78
+ option ?: CrawlOpenAICommonAPIOtherOption
79
+ ) : Promise < CrawlOpenAIGetElementSelectorsResult >
79
80
80
81
help (
81
82
content : string ,
82
- option ?: XCrawlOpenAICommonAPIOtherOption
83
+ option ?: CrawlOpenAICommonAPIOtherOption
83
84
) : Promise < string >
84
85
85
86
custom ( ) : OpenAI
86
87
}
87
88
88
- export function createXCrawlOpenAI (
89
- config : CreateXCrawlOpenAIConfig = { }
90
- ) : XCrawlOpenAIApp {
89
+ export function createCrawlOpenAI (
90
+ config : CreateCrawlOpenAIConfig = { }
91
+ ) : CrawlOpenAIApp {
91
92
const { defaultModel, clientOptions } = config
92
93
93
94
const openai = new OpenAI ( clientOptions )
94
95
const chatDefaultModel : OpenAIChatModel =
95
96
defaultModel ?. chatModel ?? 'gpt-3.5-turbo'
96
97
97
- async function runChat < T > ( option : XCrawlOpenAIRunChatOption ) : Promise < T > {
98
+ async function runChat < T > ( option : CrawlOpenAIRunChatOption ) : Promise < T > {
98
99
const {
99
100
model = chatDefaultModel ,
100
101
context,
@@ -103,7 +104,9 @@ export function createXCrawlOpenAI(
103
104
responseFormatType
104
105
} = option
105
106
106
- log ( logStart ( `AI is answering your question, please wait a moment` ) )
107
+ const spinner = ora (
108
+ `AI is answering your question, please wait a moment`
109
+ ) . start ( )
107
110
const completion = await openai . chat . completions . create ( {
108
111
model,
109
112
messages : [
@@ -114,7 +117,7 @@ export function createXCrawlOpenAI(
114
117
response_format : { type : responseFormatType } ,
115
118
temperature : 0.1
116
119
} )
117
- log ( logSuccess ( `AI has completed your question` ) )
120
+ spinner . succeed ( `AI has completed your question` )
118
121
119
122
const content = completion . choices [ 0 ] . message . content
120
123
const result =
@@ -123,25 +126,25 @@ export function createXCrawlOpenAI(
123
126
return result
124
127
}
125
128
126
- const app : XCrawlOpenAIApp = {
129
+ const app : CrawlOpenAIApp = {
127
130
async parseElements < T extends Record < string , string > > (
128
131
HTML : string ,
129
- content : string | XCrawlOpenAIParseElementsContentOptions ,
130
- option : XCrawlOpenAICommonAPIOtherOption = { }
131
- ) : Promise < XCrawlOpenAIParseElementsResult < T > > {
132
+ content : string | CrawlOpenAIParseElementsContentOptions ,
133
+ option : CrawlOpenAICommonAPIOtherOption = { }
134
+ ) : Promise < CrawlOpenAIParseElementsResult < T > > {
132
135
const { model } = option
133
136
134
137
let coderContent : string = ''
135
138
if ( isObject ( content ) ) {
136
139
coderContent = JSON . stringify ( content )
137
140
} else {
138
- const obj : XCrawlOpenAIParseElementsContentOptions = {
141
+ const obj : CrawlOpenAIParseElementsContentOptions = {
139
142
message : content
140
143
}
141
144
coderContent = JSON . stringify ( obj )
142
145
}
143
146
144
- const result = await runChat < XCrawlOpenAIParseElementsResult < T > > ( {
147
+ const result = await runChat < CrawlOpenAIParseElementsResult < T > > ( {
145
148
model,
146
149
context : PARSE_ELEMENTS_CONTEXT ,
147
150
HTMLContent : HTML ,
@@ -154,23 +157,23 @@ export function createXCrawlOpenAI(
154
157
155
158
async getElementSelectors (
156
159
HTML : string ,
157
- content : string | XCrawlOpenAIGetElementSelectorsContentOptions ,
158
- option : XCrawlOpenAICommonAPIOtherOption = { }
159
- ) : Promise < XCrawlOpenAIGetElementSelectorsResult > {
160
+ content : string | CrawlOpenAIGetElementSelectorsContentOptions ,
161
+ option : CrawlOpenAICommonAPIOtherOption = { }
162
+ ) : Promise < CrawlOpenAIGetElementSelectorsResult > {
160
163
const { model } = option
161
164
162
165
let coderContent : string = ''
163
166
if ( isObject ( content ) ) {
164
167
coderContent = JSON . stringify ( content )
165
168
} else {
166
- const obj : XCrawlOpenAIGetElementSelectorsContentOptions = {
169
+ const obj : CrawlOpenAIGetElementSelectorsContentOptions = {
167
170
message : content ,
168
171
pathMode : 'default'
169
172
}
170
173
coderContent = JSON . stringify ( obj )
171
174
}
172
175
173
- const result = await runChat < XCrawlOpenAIGetElementSelectorsResult > ( {
176
+ const result = await runChat < CrawlOpenAIGetElementSelectorsResult > ( {
174
177
model,
175
178
context : GET_ELEMENT_SELECTORS_CONTEXT ,
176
179
HTMLContent : HTML ,
@@ -183,7 +186,7 @@ export function createXCrawlOpenAI(
183
186
184
187
async help (
185
188
content : string ,
186
- option : XCrawlOpenAICommonAPIOtherOption = { }
189
+ option : CrawlOpenAICommonAPIOtherOption = { }
187
190
) : Promise < string > {
188
191
const { model } = option
189
192
0 commit comments