@@ -28,6 +28,63 @@ Before diving into the implementation, let's understand why local testing is val
28
28
6 . ** Quick Iteration** : Test the entire pipeline without API costs
29
29
7 . ** Prompt Development** : Iterate quickly on prompts and logic using smaller models
30
30
31
+ ## Project Setup and Dependencies
32
+
33
+ Before we start implementing our research system, let's set up the project and install the necessary dependencies:
34
+
35
+ 1 . ** Create a new Nitric project** :
36
+
37
+ ``` bash
38
+ # Install Nitric CLI if you haven't already
39
+ npm install -g @nitric/cli
40
+
41
+ # Create a new Nitric project
42
+ nitric new deep-research
43
+ cd deep-research
44
+ ```
45
+
46
+ 2 . ** Configure dependencies in deno.json** :
47
+
48
+ Create or update the ` deno.json ` file in your project root:
49
+
50
+ ``` json title:deno.json
51
+ {
52
+ "imports" : {
53
+ "@nitric/sdk" : " npm:@nitric/sdk" ,
54
+ "openai" : " npm:openai" ,
55
+ "duck-duck-scrape" : " npm:duck-duck-scrape" ,
56
+ "cheerio" : " npm:cheerio" ,
57
+ "turndown" : " npm:turndown"
58
+ },
59
+ "tasks" : {
60
+ "start" : " deno run --allow-net --allow-env --allow-read main.ts"
61
+ }
62
+ }
63
+ ```
64
+
65
+ 3 . ** Install dependencies** :
66
+
67
+ ``` bash
68
+ # Install dependencies using Deno
69
+ deno install
70
+ ```
71
+
72
+ 4 . ** Project structure** :
73
+
74
+ ```
75
+ deep-research/
76
+ ├── deno.json # Deno configuration and dependencies
77
+ ├── .env # Environment variables
78
+ ├── utils/
79
+ │ └── search.ts # Search functionality
80
+ ├── prompts/
81
+ │ ├── query.ts # Query generation prompt
82
+ │ ├── summarizer.ts # Content summarization prompt
83
+ │ └── reflect.ts # Research reflection prompt
84
+ └── services/
85
+ └── api.ts # Main API implementation
86
+ ```
87
+
31
88
## The Architecture
32
89
33
90
Our research system will be built as a Nitric API with several key components:
@@ -91,7 +148,7 @@ The LLM integration handles the "Summarization" and "Reflection" steps:
91
148
The system can be configured to use different LLM providers through environment variables:
92
149
93
150
``` typescript title:services/api.ts
94
- import openai from ' npm: openai'
151
+ import { OpenAI } from ' openai'
95
152
96
153
// Configuration from environment variables
97
154
const LLM_CONFIG = {
@@ -100,7 +157,7 @@ const LLM_CONFIG = {
100
157
model: Deno .env .get (' LLM_MODEL' ) || ' llama3.2:3b' ,
101
158
}
102
159
103
- const OAI = new openai ({
160
+ const OAI = new OpenAI ({
104
161
baseURL: LLM_CONFIG .baseURL ,
105
162
apiKey: LLM_CONFIG .apiKey ,
106
163
})
@@ -258,8 +315,17 @@ interface ReflectTopicMessage extends ResearchTopicMessage<'reflect'> {
258
315
The system includes HTML cleaning and markdown conversion:
259
316
260
317
``` typescript title:services/api.ts
318
+ import { load as cheerioLoad } from ' cheerio'
319
+ import { TurndownService } from ' turndown'
320
+
321
+ // Initialize TurndownService for HTML to Markdown conversion
322
+ const turndownService = new TurndownService ({
323
+ headingStyle: ' atx' ,
324
+ codeBlockStyle: ' fenced' ,
325
+ })
326
+
261
327
function cleanHtml(html : string ): string {
262
- const $ = cheerio . load (html )
328
+ const $ = cheerioLoad (html )
263
329
264
330
// Remove script and style tags
265
331
$ (' script, style, noscript, iframe, embed, object' ).remove ()
@@ -292,14 +358,22 @@ function cleanHtml(html: string): string {
292
358
The main API implementation ties everything together:
293
359
294
360
``` typescript title:services/api.ts
295
- import { api , topic , bucket } from ' npm: @nitric/sdk'
361
+ import { api , topic , bucket } from ' @nitric/sdk'
296
362
import search from ' ../utils/search.ts'
297
- import openai from ' npm: openai'
363
+ import { OpenAI } from ' openai'
298
364
import queryPrompt from ' ../prompts/query.ts'
299
365
import summarizerPrompt from ' ../prompts/summarizer.ts'
300
366
import reflectionPrompt from ' ../prompts/reflect.ts'
367
+ import { load as cheerioLoad } from ' cheerio'
368
+ import { TurndownService } from ' turndown'
369
+
370
+ // Initialize TurndownService for HTML to Markdown conversion
371
+ const turndownService = new TurndownService ({
372
+ headingStyle: ' atx' ,
373
+ codeBlockStyle: ' fenced' ,
374
+ })
301
375
302
- const OAI = new openai ({
376
+ const OAI = new OpenAI ({
303
377
baseURL: ' http://localhost:11434/v1' ,
304
378
apiKey: ' ollama' ,
305
379
})
0 commit comments