|
1 | 1 | """ |
2 | | -Basic example of scraping pipeline using ScriptCreatorGraph |
| 2 | +Basic example of scraping pipeline using SmartScraper |
3 | 3 | """ |
4 | 4 |
|
5 | 5 | import os |
| 6 | +import json |
6 | 7 | from dotenv import load_dotenv |
7 | | -from scrapegraphai.graphs import ScriptCreatorGraph |
| 8 | +from scrapegraphai.graphs import SmartScraperGraph |
8 | 9 | from scrapegraphai.utils import prettify_exec_info |
9 | 10 |
|
10 | 11 | load_dotenv() |
|
13 | 14 | # Define the configuration for the graph |
14 | 15 | # ************************************************ |
15 | 16 |
|
16 | | -openai_key = os.getenv("OPENAI_APIKEY") |
17 | 17 |
|
18 | 18 | graph_config = { |
19 | 19 | "llm": { |
20 | | - "api_key": openai_key, |
21 | | - "model": "openai/gpt-4o", |
| 20 | + "api_key": os.getenv("OPENAI_API_KEY"), |
| 21 | + "model": "gpt-4o", |
22 | 22 | }, |
23 | | - "library": "beautifulsoup" |
| 23 | + "verbose": True, |
| 24 | + "headless": False, |
24 | 25 | } |
25 | 26 |
|
26 | 27 | # ************************************************ |
27 | | -# Create the ScriptCreatorGraph instance and run it |
| 28 | +# Create the SmartScraperGraph instance and run it |
28 | 29 | # ************************************************ |
29 | 30 |
|
30 | | -script_creator_graph = ScriptCreatorGraph( |
31 | | - prompt="List me all the projects with their description.", |
32 | | - # also accepts a string with the already downloaded HTML code |
33 | | - source="https://perinim.github.io/projects", |
| 31 | +smart_scraper_graph = SmartScraperGraph( |
| 32 | + prompt="List me what does the company do, the name and a contact email.", |
| 33 | + source="https://scrapegraphai.com/", |
34 | 34 | config=graph_config |
35 | 35 | ) |
36 | 36 |
|
37 | | -result = script_creator_graph.run() |
38 | | -print(result) |
| 37 | +result = smart_scraper_graph.run() |
| 38 | +print(json.dumps(result, indent=4)) |
39 | 39 |
|
40 | 40 | # ************************************************ |
41 | 41 | # Get graph execution info |
42 | 42 | # ************************************************ |
43 | 43 |
|
44 | | -graph_exec_info = script_creator_graph.get_execution_info() |
| 44 | +graph_exec_info = smart_scraper_graph.get_execution_info() |
45 | 45 | print(prettify_exec_info(graph_exec_info)) |
46 | | - |
0 commit comments