|
1 | | -import OpenAI, { ClientOptions } from 'openai' |
2 | | -import ora from 'ora' |
3 | | - |
4 | | -import { |
5 | | - PARSE_ELEMENTS_CONTEXT, |
6 | | - GET_ELEMENT_SELECTORS_CONTEXT, |
7 | | - HELP_CONTEXT |
8 | | -} from './context' |
9 | | -import { isObject, logStart, logSuccess } from '../shared' |
10 | | - |
11 | | -type OpenAIChatModel = |
12 | | - | 'gpt-4o' |
13 | | - | 'gpt-4o-2024-05-13' |
14 | | - | 'gpt-4-turbo' |
15 | | - | 'gpt-4-turbo-2024-04-09' |
16 | | - | 'gpt-4-0125-preview' |
17 | | - | 'gpt-4-turbo-preview' |
18 | | - | 'gpt-4-1106-preview' |
19 | | - | 'gpt-4-vision-preview' |
20 | | - | 'gpt-4' |
21 | | - | 'gpt-4-0314' |
22 | | - | 'gpt-4-0613' |
23 | | - | 'gpt-4-32k' |
24 | | - | 'gpt-4-32k-0314' |
25 | | - | 'gpt-4-32k-0613' |
26 | | - | 'gpt-3.5-turbo' |
27 | | - | 'gpt-3.5-turbo-16k' |
28 | | - | 'gpt-3.5-turbo-0301' |
29 | | - | 'gpt-3.5-turbo-0613' |
30 | | - | 'gpt-3.5-turbo-1106' |
31 | | - | 'gpt-3.5-turbo-0125' |
32 | | - | 'gpt-3.5-turbo-16k-0613' |
33 | | - |
34 | | -interface CreateCrawlOpenAIConfig { |
35 | | - defaultModel?: { |
36 | | - chatModel: (string & {}) | OpenAIChatModel |
37 | | - } |
38 | | - clientOptions?: ClientOptions |
39 | | -} |
40 | | - |
41 | | -interface CrawlOpenAICommonAPIOtherOption { |
42 | | - model?: (string & {}) | OpenAIChatModel |
43 | | -} |
44 | | - |
45 | | -interface CrawlOpenAIRunChatOption { |
46 | | - model: (string & {}) | OpenAIChatModel | undefined |
47 | | - context: string |
48 | | - HTMLContent: string |
49 | | - userContent: string |
50 | | - responseFormatType: 'text' | 'json_object' |
51 | | -} |
52 | | - |
53 | | -interface CrawlOpenAIParseElementsContentOptions { |
54 | | - message: string |
55 | | -} |
56 | | - |
57 | | -interface CrawlOpenAIGetElementSelectorsContentOptions { |
58 | | - message: string |
59 | | - pathMode: 'default' | 'strict' |
60 | | -} |
61 | | - |
62 | | -interface CrawlOpenAIGetElementSelectorsResult { |
63 | | - selectors: string |
64 | | - type: 'single' | 'multiple' | 'none' |
65 | | -} |
66 | | - |
67 | | -interface CrawlOpenAIParseElementsResult<T extends Record<string, string>> { |
68 | | - elements: T[] |
69 | | - type: 'single' | 'multiple' | 'none' |
70 | | -} |
71 | | - |
72 | | -interface CrawlOpenAIApp { |
73 | | - parseElements<T extends Record<string, string>>( |
74 | | - HTML: string, |
75 | | - content: string | CrawlOpenAIParseElementsContentOptions, |
76 | | - option?: CrawlOpenAICommonAPIOtherOption |
77 | | - ): Promise<CrawlOpenAIParseElementsResult<T>> |
78 | | - |
79 | | - getElementSelectors( |
80 | | - HTML: string, |
81 | | - content: string | CrawlOpenAIGetElementSelectorsContentOptions, |
82 | | - option?: CrawlOpenAICommonAPIOtherOption |
83 | | - ): Promise<CrawlOpenAIGetElementSelectorsResult> |
84 | | - |
85 | | - help( |
86 | | - content: string, |
87 | | - option?: CrawlOpenAICommonAPIOtherOption |
88 | | - ): Promise<string> |
89 | | - |
90 | | - custom(): OpenAI |
91 | | -} |
92 | | - |
93 | | -export function createCrawlOpenAI( |
94 | | - config: CreateCrawlOpenAIConfig = {} |
95 | | -): CrawlOpenAIApp { |
96 | | - const { defaultModel, clientOptions } = config |
97 | | - |
98 | | - const openai = new OpenAI(clientOptions) |
99 | | - const chatDefaultModel: (string & {}) | OpenAIChatModel = |
100 | | - defaultModel?.chatModel ?? 'gpt-3.5-turbo' |
101 | | - |
102 | | - async function runChat<T>(option: CrawlOpenAIRunChatOption): Promise<T> { |
103 | | - const { |
104 | | - model = chatDefaultModel, |
105 | | - context, |
106 | | - HTMLContent, |
107 | | - userContent, |
108 | | - responseFormatType |
109 | | - } = option |
110 | | - |
111 | | - const spinner = ora( |
112 | | - logStart(`AI is answering your question, please wait a moment`) |
113 | | - ).start() |
114 | | - const completion = await openai.chat.completions.create({ |
115 | | - model, |
116 | | - messages: [ |
117 | | - { role: 'system', content: context }, |
118 | | - { role: 'user', name: 'x-crawl', content: HTMLContent }, |
119 | | - { role: 'user', name: 'coder', content: userContent } |
120 | | - ], |
121 | | - response_format: { type: responseFormatType }, |
122 | | - temperature: 0.1 |
123 | | - }) |
124 | | - spinner.succeed(logSuccess(`AI has completed your question`)) |
125 | | - |
126 | | - const content = completion.choices[0].message.content |
127 | | - const result = |
128 | | - responseFormatType === 'json_object' ? JSON.parse(content!) : content |
129 | | - |
130 | | - return result |
131 | | - } |
132 | | - |
133 | | - const app: CrawlOpenAIApp = { |
134 | | - async parseElements<T extends Record<string, string>>( |
135 | | - HTML: string, |
136 | | - content: string | CrawlOpenAIParseElementsContentOptions, |
137 | | - option: CrawlOpenAICommonAPIOtherOption = {} |
138 | | - ): Promise<CrawlOpenAIParseElementsResult<T>> { |
139 | | - const { model } = option |
140 | | - |
141 | | - let coderContent: string = '' |
142 | | - if (isObject(content)) { |
143 | | - coderContent = JSON.stringify(content) |
144 | | - } else { |
145 | | - const obj: CrawlOpenAIParseElementsContentOptions = { |
146 | | - message: content |
147 | | - } |
148 | | - coderContent = JSON.stringify(obj) |
149 | | - } |
150 | | - |
151 | | - const result = await runChat<CrawlOpenAIParseElementsResult<T>>({ |
152 | | - model, |
153 | | - context: PARSE_ELEMENTS_CONTEXT, |
154 | | - HTMLContent: HTML, |
155 | | - userContent: coderContent, |
156 | | - responseFormatType: 'json_object' |
157 | | - }) |
158 | | - |
159 | | - return result |
160 | | - }, |
161 | | - |
162 | | - async getElementSelectors( |
163 | | - HTML: string, |
164 | | - content: string | CrawlOpenAIGetElementSelectorsContentOptions, |
165 | | - option: CrawlOpenAICommonAPIOtherOption = {} |
166 | | - ): Promise<CrawlOpenAIGetElementSelectorsResult> { |
167 | | - const { model } = option |
168 | | - |
169 | | - let coderContent: string = '' |
170 | | - if (isObject(content)) { |
171 | | - coderContent = JSON.stringify(content) |
172 | | - } else { |
173 | | - const obj: CrawlOpenAIGetElementSelectorsContentOptions = { |
174 | | - message: content, |
175 | | - pathMode: 'default' |
176 | | - } |
177 | | - coderContent = JSON.stringify(obj) |
178 | | - } |
179 | | - |
180 | | - const result = await runChat<CrawlOpenAIGetElementSelectorsResult>({ |
181 | | - model, |
182 | | - context: GET_ELEMENT_SELECTORS_CONTEXT, |
183 | | - HTMLContent: HTML, |
184 | | - userContent: coderContent, |
185 | | - responseFormatType: 'json_object' |
186 | | - }) |
187 | | - |
188 | | - return result |
189 | | - }, |
190 | | - |
191 | | - async help( |
192 | | - content: string, |
193 | | - option: CrawlOpenAICommonAPIOtherOption = {} |
194 | | - ): Promise<string> { |
195 | | - const { model } = option |
196 | | - |
197 | | - const result = await runChat<string>({ |
198 | | - model, |
199 | | - context: HELP_CONTEXT, |
200 | | - HTMLContent: '', |
201 | | - userContent: content, |
202 | | - responseFormatType: 'text' |
203 | | - }) |
204 | | - |
205 | | - return result |
206 | | - }, |
207 | | - |
208 | | - custom() { |
209 | | - return openai |
210 | | - } |
211 | | - } |
212 | | - |
213 | | - return app |
214 | | -} |
| 1 | +export * from './openai' |
| 2 | +export * from './ollama' |
0 commit comments