Skip to content

Commit 8553d89

Browse files
VinciGit00VinciGit00
authored
Merge pull request #698 from vedovati-matteo/pre/beta
Code Generator functionalities and graph
2 parents f42a95f + fb87901 commit 8553d89

37 files changed

+2364
-18
lines changed

‎examples/anthropic/code_generator_graph_anthropic.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"""
2+
Basic example of scraping pipeline using Code Generator with schema
3+
"""
4+
5+
import os, json
6+
from typing import List
7+
from dotenv import load_dotenv
8+
from langchain_core.pydantic_v1 import BaseModel, Field
9+
from scrapegraphai.graphs import CodeGeneratorGraph
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Define the output schema for the graph
15+
# ************************************************
16+
17+
class Project(BaseModel):
18+
title: str = Field(description="The title of the project")
19+
description: str = Field(description="The description of the project")
20+
21+
class Projects(BaseModel):
22+
projects: List[Project]
23+
24+
# ************************************************
25+
# Define the configuration for the graph
26+
# ************************************************
27+
28+
anthropic_key = os.getenv("ANTHROPIC_API_KEY")
29+
30+
graph_config = {
31+
"llm": {
32+
"api_key":anthropic_key,
33+
"model": "anthropic/claude-3-haiku-20240307",
34+
},
35+
"verbose": True,
36+
"headless": False,
37+
"reduction": 2,
38+
"max_iterations": {
39+
"overall": 10,
40+
"syntax": 3,
41+
"execution": 3,
42+
"validation": 3,
43+
"semantic": 3
44+
},
45+
"output_file_name": "extracted_data.py"
46+
}
47+
48+
# ************************************************
49+
# Create the SmartScraperGraph instance and run it
50+
# ************************************************
51+
52+
code_generator_graph = CodeGeneratorGraph(
53+
prompt="List me all the projects with their description",
54+
source="https://perinim.github.io/projects/",
55+
schema=Projects,
56+
config=graph_config
57+
)
58+
59+
result = code_generator_graph.run()
60+
print(result)

‎examples/azure/code_generator_graph_azure.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
"""
2+
Basic example of scraping pipeline using Code Generator with schema
3+
"""
4+
5+
import os, json
6+
from typing import List
7+
from dotenv import load_dotenv
8+
from langchain_core.pydantic_v1 import BaseModel, Field
9+
from scrapegraphai.graphs import CodeGeneratorGraph
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Define the output schema for the graph
15+
# ************************************************
16+
17+
class Project(BaseModel):
18+
title: str = Field(description="The title of the project")
19+
description: str = Field(description="The description of the project")
20+
21+
class Projects(BaseModel):
22+
projects: List[Project]
23+
24+
# ************************************************
25+
# Define the configuration for the graph
26+
# ************************************************
27+
28+
graph_config = {
29+
"llm": {
30+
"api_key": os.environ["AZURE_OPENAI_KEY"],
31+
"model": "azure_openai/gpt-3.5-turbo",
32+
},
33+
"verbose": True,
34+
"headless": False,
35+
"reduction": 2,
36+
"max_iterations": {
37+
"overall": 10,
38+
"syntax": 3,
39+
"execution": 3,
40+
"validation": 3,
41+
"semantic": 3
42+
},
43+
"output_file_name": "extracted_data.py"
44+
}
45+
46+
# ************************************************
47+
# Create the SmartScraperGraph instance and run it
48+
# ************************************************
49+
50+
code_generator_graph = CodeGeneratorGraph(
51+
prompt="List me all the projects with their description",
52+
source="https://perinim.github.io/projects/",
53+
schema=Projects,
54+
config=graph_config
55+
)
56+
57+
result = code_generator_graph.run()
58+
print(result)

‎examples/bedrock/code_generator_graph_bedrock.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"""
2+
Basic example of scraping pipeline using Code Generator with schema
3+
"""
4+
5+
import os, json
6+
from typing import List
7+
from dotenv import load_dotenv
8+
from langchain_core.pydantic_v1 import BaseModel, Field
9+
from scrapegraphai.graphs import CodeGeneratorGraph
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Define the output schema for the graph
15+
# ************************************************
16+
17+
class Project(BaseModel):
18+
title: str = Field(description="The title of the project")
19+
description: str = Field(description="The description of the project")
20+
21+
class Projects(BaseModel):
22+
projects: List[Project]
23+
24+
# ************************************************
25+
# Define the configuration for the graph
26+
# ************************************************
27+
28+
29+
graph_config = {
30+
"llm": {
31+
"client": "client_name",
32+
"model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
33+
"temperature": 0.0
34+
},
35+
"verbose": True,
36+
"headless": False,
37+
"reduction": 2,
38+
"max_iterations": {
39+
"overall": 10,
40+
"syntax": 3,
41+
"execution": 3,
42+
"validation": 3,
43+
"semantic": 3
44+
},
45+
"output_file_name": "extracted_data.py"
46+
}
47+
48+
# ************************************************
49+
# Create the SmartScraperGraph instance and run it
50+
# ************************************************
51+
52+
code_generator_graph = CodeGeneratorGraph(
53+
prompt="List me all the projects with their description",
54+
source="https://perinim.github.io/projects/",
55+
schema=Projects,
56+
config=graph_config
57+
)
58+
59+
result = code_generator_graph.run()
60+
print(result)

‎examples/deepseek/code_generator_graph_deepseek.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"""
2+
Basic example of scraping pipeline using Code Generator with schema
3+
"""
4+
5+
import os, json
6+
from typing import List
7+
from dotenv import load_dotenv
8+
from langchain_core.pydantic_v1 import BaseModel, Field
9+
from scrapegraphai.graphs import CodeGeneratorGraph
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Define the output schema for the graph
15+
# ************************************************
16+
17+
class Project(BaseModel):
18+
title: str = Field(description="The title of the project")
19+
description: str = Field(description="The description of the project")
20+
21+
class Projects(BaseModel):
22+
projects: List[Project]
23+
24+
# ************************************************
25+
# Define the configuration for the graph
26+
# ************************************************
27+
28+
deepseek_key = os.getenv("DEEPSEEK_APIKEY")
29+
30+
graph_config = {
31+
"llm": {
32+
"model": "deepseek/deepseek-chat",
33+
"api_key": deepseek_key,
34+
},
35+
"verbose": True,
36+
"headless": False,
37+
"reduction": 2,
38+
"max_iterations": {
39+
"overall": 10,
40+
"syntax": 3,
41+
"execution": 3,
42+
"validation": 3,
43+
"semantic": 3
44+
},
45+
"output_file_name": "extracted_data.py"
46+
}
47+
48+
# ************************************************
49+
# Create the SmartScraperGraph instance and run it
50+
# ************************************************
51+
52+
code_generator_graph = CodeGeneratorGraph(
53+
prompt="List me all the projects with their description",
54+
source="https://perinim.github.io/projects/",
55+
schema=Projects,
56+
config=graph_config
57+
)
58+
59+
result = code_generator_graph.run()
60+
print(result)

‎examples/ernie/code_generator_graph_ernie.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
"""
2+
Basic example of scraping pipeline using Code Generator with schema
3+
"""
4+
5+
import os, json
6+
from typing import List
7+
from dotenv import load_dotenv
8+
from langchain_core.pydantic_v1 import BaseModel, Field
9+
from scrapegraphai.graphs import CodeGeneratorGraph
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Define the output schema for the graph
15+
# ************************************************
16+
17+
class Project(BaseModel):
18+
title: str = Field(description="The title of the project")
19+
description: str = Field(description="The description of the project")
20+
21+
class Projects(BaseModel):
22+
projects: List[Project]
23+
24+
# ************************************************
25+
# Define the configuration for the graph
26+
# ************************************************
27+
28+
openai_key = os.getenv("OPENAI_APIKEY")
29+
30+
graph_config = {
31+
"llm": {
32+
"model": "ernie/ernie-bot-turbo",
33+
"ernie_client_id": "<ernie_client_id>",
34+
"ernie_client_secret": "<ernie_client_secret>",
35+
"temperature": 0.1
36+
},
37+
"verbose": True,
38+
"headless": False,
39+
"reduction": 2,
40+
"max_iterations": {
41+
"overall": 10,
42+
"syntax": 3,
43+
"execution": 3,
44+
"validation": 3,
45+
"semantic": 3
46+
},
47+
"output_file_name": "extracted_data.py"
48+
}
49+
50+
# ************************************************
51+
# Create the SmartScraperGraph instance and run it
52+
# ************************************************
53+
54+
code_generator_graph = CodeGeneratorGraph(
55+
prompt="List me all the projects with their description",
56+
source="https://perinim.github.io/projects/",
57+
schema=Projects,
58+
config=graph_config
59+
)
60+
61+
result = code_generator_graph.run()
62+
print(result)

‎examples/fireworks/code_generator_graph_fireworks.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
"""
2+
Basic example of scraping pipeline using Code Generator with schema
3+
"""
4+
5+
import os, json
6+
from typing import List
7+
from dotenv import load_dotenv
8+
from langchain_core.pydantic_v1 import BaseModel, Field
9+
from scrapegraphai.graphs import CodeGeneratorGraph
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Define the output schema for the graph
15+
# ************************************************
16+
17+
class Project(BaseModel):
18+
title: str = Field(description="The title of the project")
19+
description: str = Field(description="The description of the project")
20+
21+
class Projects(BaseModel):
22+
projects: List[Project]
23+
24+
# ************************************************
25+
# Define the configuration for the graph
26+
# ************************************************
27+
28+
fireworks_api_key = os.getenv("FIREWORKS_APIKEY")
29+
30+
graph_config = {
31+
"llm": {
32+
"api_key": fireworks_api_key,
33+
"model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct"
34+
},
35+
"verbose": True,
36+
"headless": False,
37+
"reduction": 2,
38+
"max_iterations": {
39+
"overall": 10,
40+
"syntax": 3,
41+
"execution": 3,
42+
"validation": 3,
43+
"semantic": 3
44+
},
45+
"output_file_name": "extracted_data.py"
46+
}
47+
48+
# ************************************************
49+
# Create the SmartScraperGraph instance and run it
50+
# ************************************************
51+
52+
code_generator_graph = CodeGeneratorGraph(
53+
prompt="List me all the projects with their description",
54+
source="https://perinim.github.io/projects/",
55+
schema=Projects,
56+
config=graph_config
57+
)
58+
59+
result = code_generator_graph.run()
60+
print(result)

0 commit comments

Comments
 (0)