Skip to content

Commit 431b495

Browse files
VinciGit00VinciGit00
committed
add examples for azure
1 parent 75a4042 commit 431b495

File tree

5 files changed

+445
-0
lines changed

5 files changed

+445
-0
lines changed

‎examples/azure/inputs/books.xml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
<?xml version="1.0"?>
2+
<catalog>
3+
<book id="bk101">
4+
<author>Gambardella, Matthew</author>
5+
<title>XML Developer's Guide</title>
6+
<genre>Computer</genre>
7+
<price>44.95</price>
8+
<publish_date>2000-10-01</publish_date>
9+
<description>An in-depth look at creating applications
10+
with XML.</description>
11+
</book>
12+
<book id="bk102">
13+
<author>Ralls, Kim</author>
14+
<title>Midnight Rain</title>
15+
<genre>Fantasy</genre>
16+
<price>5.95</price>
17+
<publish_date>2000-12-16</publish_date>
18+
<description>A former architect battles corporate zombies,
19+
an evil sorceress, and her own childhood to become queen
20+
of the world.</description>
21+
</book>
22+
<book id="bk103">
23+
<author>Corets, Eva</author>
24+
<title>Maeve Ascendant</title>
25+
<genre>Fantasy</genre>
26+
<price>5.95</price>
27+
<publish_date>2000-11-17</publish_date>
28+
<description>After the collapse of a nanotechnology
29+
society in England, the young survivors lay the
30+
foundation for a new society.</description>
31+
</book>
32+
<book id="bk104">
33+
<author>Corets, Eva</author>
34+
<title>Oberon's Legacy</title>
35+
<genre>Fantasy</genre>
36+
<price>5.95</price>
37+
<publish_date>2001-03-10</publish_date>
38+
<description>In post-apocalypse England, the mysterious
39+
agent known only as Oberon helps to create a new life
40+
for the inhabitants of London. Sequel to Maeve
41+
Ascendant.</description>
42+
</book>
43+
<book id="bk105">
44+
<author>Corets, Eva</author>
45+
<title>The Sundered Grail</title>
46+
<genre>Fantasy</genre>
47+
<price>5.95</price>
48+
<publish_date>2001-09-10</publish_date>
49+
<description>The two daughters of Maeve, half-sisters,
50+
battle one another for control of England. Sequel to
51+
Oberon's Legacy.</description>
52+
</book>
53+
<book id="bk106">
54+
<author>Randall, Cynthia</author>
55+
<title>Lover Birds</title>
56+
<genre>Romance</genre>
57+
<price>4.95</price>
58+
<publish_date>2000-09-02</publish_date>
59+
<description>When Carla meets Paul at an ornithology
60+
conference, tempers fly as feathers get ruffled.</description>
61+
</book>
62+
<book id="bk107">
63+
<author>Thurman, Paula</author>
64+
<title>Splish Splash</title>
65+
<genre>Romance</genre>
66+
<price>4.95</price>
67+
<publish_date>2000-11-02</publish_date>
68+
<description>A deep sea diver finds true love twenty
69+
thousand leagues beneath the sea.</description>
70+
</book>
71+
<book id="bk108">
72+
<author>Knorr, Stefan</author>
73+
<title>Creepy Crawlies</title>
74+
<genre>Horror</genre>
75+
<price>4.95</price>
76+
<publish_date>2000-12-06</publish_date>
77+
<description>An anthology of horror stories about roaches,
78+
centipedes, scorpions and other insects.</description>
79+
</book>
80+
<book id="bk109">
81+
<author>Kress, Peter</author>
82+
<title>Paradox Lost</title>
83+
<genre>Science Fiction</genre>
84+
<price>6.95</price>
85+
<publish_date>2000-11-02</publish_date>
86+
<description>After an inadvertant trip through a Heisenberg
87+
Uncertainty Device, James Salway discovers the problems
88+
of being quantum.</description>
89+
</book>
90+
<book id="bk110">
91+
<author>O'Brien, Tim</author>
92+
<title>Microsoft .NET: The Programming Bible</title>
93+
<genre>Computer</genre>
94+
<price>36.95</price>
95+
<publish_date>2000-12-09</publish_date>
96+
<description>Microsoft's .NET initiative is explored in
97+
detail in this deep programmer's reference.</description>
98+
</book>
99+
<book id="bk111">
100+
<author>O'Brien, Tim</author>
101+
<title>MSXML3: A Comprehensive Guide</title>
102+
<genre>Computer</genre>
103+
<price>36.95</price>
104+
<publish_date>2000-12-01</publish_date>
105+
<description>The Microsoft MSXML3 parser is covered in
106+
detail, with attention to XML DOM interfaces, XSLT processing,
107+
SAX and more.</description>
108+
</book>
109+
<book id="bk112">
110+
<author>Galos, Mike</author>
111+
<title>Visual Studio 7: A Comprehensive Guide</title>
112+
<genre>Computer</genre>
113+
<price>49.95</price>
114+
<publish_date>2001-04-16</publish_date>
115+
<description>Microsoft Visual Studio 7 is explored in depth,
116+
looking at how Visual Basic, Visual C++, C#, and ASP+ are
117+
integrated into a comprehensive development
118+
environment.</description>
119+
</book>
120+
</catalog>

‎examples/azure/inputs/example.json‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
{
2+
"kind":"youtube#searchListResponse",
3+
"etag":"q4ibjmYp1KA3RqMF4jFLl6PBwOg",
4+
"nextPageToken":"CAUQAA",
5+
"regionCode":"NL",
6+
"pageInfo":{
7+
"totalResults":1000000,
8+
"resultsPerPage":5
9+
},
10+
"items":[
11+
{
12+
"kind":"youtube#searchResult",
13+
"etag":"QCsHBifbaernVCbLv8Cu6rAeaDQ",
14+
"id":{
15+
"kind":"youtube#video",
16+
"videoId":"TvWDY4Mm5GM"
17+
},
18+
"snippet":{
19+
"publishedAt":"2023-07-24T14:15:01Z",
20+
"channelId":"UCwozCpFp9g9x0wAzuFh0hwQ",
21+
"title":"3 Football Clubs Kylian Mbappe Should Avoid Signing ✍️❌⚽️ #football #mbappe #shorts",
22+
"description":"",
23+
"thumbnails":{
24+
"default":{
25+
"url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/default.jpg",
26+
"width":120,
27+
"height":90
28+
},
29+
"medium":{
30+
"url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/mqdefault.jpg",
31+
"width":320,
32+
"height":180
33+
},
34+
"high":{
35+
"url":"https://i.ytimg.com/vi/TvWDY4Mm5GM/hqdefault.jpg",
36+
"width":480,
37+
"height":360
38+
}
39+
},
40+
"channelTitle":"FC Motivate",
41+
"liveBroadcastContent":"none",
42+
"publishTime":"2023-07-24T14:15:01Z"
43+
}
44+
},
45+
{
46+
"kind":"youtube#searchResult",
47+
"etag":"0NG5QHdtIQM_V-DBJDEf-jK_Y9k",
48+
"id":{
49+
"kind":"youtube#video",
50+
"videoId":"aZM_42CcNZ4"
51+
},
52+
"snippet":{
53+
"publishedAt":"2023-07-24T16:09:27Z",
54+
"channelId":"UCM5gMM_HqfKHYIEJ3lstMUA",
55+
"title":"Which Football Club Could Cristiano Ronaldo Afford To Buy? 💰",
56+
"description":"Sign up to Sorare and get a FREE card: https://sorare.pxf.io/NellisShorts Give Soraredata a go for FREE: ...",
57+
"thumbnails":{
58+
"default":{
59+
"url":"https://i.ytimg.com/vi/aZM_42CcNZ4/default.jpg",
60+
"width":120,
61+
"height":90
62+
},
63+
"medium":{
64+
"url":"https://i.ytimg.com/vi/aZM_42CcNZ4/mqdefault.jpg",
65+
"width":320,
66+
"height":180
67+
},
68+
"high":{
69+
"url":"https://i.ytimg.com/vi/aZM_42CcNZ4/hqdefault.jpg",
70+
"width":480,
71+
"height":360
72+
}
73+
},
74+
"channelTitle":"John Nellis",
75+
"liveBroadcastContent":"none",
76+
"publishTime":"2023-07-24T16:09:27Z"
77+
}
78+
},
79+
{
80+
"kind":"youtube#searchResult",
81+
"etag":"WbBz4oh9I5VaYj91LjeJvffrBVY",
82+
"id":{
83+
"kind":"youtube#video",
84+
"videoId":"wkP3XS3aNAY"
85+
},
86+
"snippet":{
87+
"publishedAt":"2023-07-24T16:00:50Z",
88+
"channelId":"UC4EP1dxFDPup_aFLt0ElsDw",
89+
"title":"PAULO DYBALA vs THE WORLD'S LONGEST FREEKICK WALL",
90+
"description":"Can Paulo Dybala curl a football around the World's longest free kick wall? We met up with the World Cup winner and put him to ...",
91+
"thumbnails":{
92+
"default":{
93+
"url":"https://i.ytimg.com/vi/wkP3XS3aNAY/default.jpg",
94+
"width":120,
95+
"height":90
96+
},
97+
"medium":{
98+
"url":"https://i.ytimg.com/vi/wkP3XS3aNAY/mqdefault.jpg",
99+
"width":320,
100+
"height":180
101+
},
102+
"high":{
103+
"url":"https://i.ytimg.com/vi/wkP3XS3aNAY/hqdefault.jpg",
104+
"width":480,
105+
"height":360
106+
}
107+
},
108+
"channelTitle":"Shoot for Love",
109+
"liveBroadcastContent":"none",
110+
"publishTime":"2023-07-24T16:00:50Z"
111+
}
112+
},
113+
{
114+
"kind":"youtube#searchResult",
115+
"etag":"juxv_FhT_l4qrR05S1QTrb4CGh8",
116+
"id":{
117+
"kind":"youtube#video",
118+
"videoId":"rJkDZ0WvfT8"
119+
},
120+
"snippet":{
121+
"publishedAt":"2023-07-24T10:00:39Z",
122+
"channelId":"UCO8qj5u80Ga7N_tP3BZWWhQ",
123+
"title":"TOP 10 DEFENDERS 2023",
124+
"description":"SoccerKingz https://soccerkingz.nl Use code: 'ILOVEHOF' to get 10% off. TOP 10 DEFENDERS 2023 Follow us! • Instagram ...",
125+
"thumbnails":{
126+
"default":{
127+
"url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/default.jpg",
128+
"width":120,
129+
"height":90
130+
},
131+
"medium":{
132+
"url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/mqdefault.jpg",
133+
"width":320,
134+
"height":180
135+
},
136+
"high":{
137+
"url":"https://i.ytimg.com/vi/rJkDZ0WvfT8/hqdefault.jpg",
138+
"width":480,
139+
"height":360
140+
}
141+
},
142+
"channelTitle":"Home of Football",
143+
"liveBroadcastContent":"none",
144+
"publishTime":"2023-07-24T10:00:39Z"
145+
}
146+
},
147+
{
148+
"kind":"youtube#searchResult",
149+
"etag":"wtuknXTmI1txoULeH3aWaOuXOow",
150+
"id":{
151+
"kind":"youtube#video",
152+
"videoId":"XH0rtu4U6SE"
153+
},
154+
"snippet":{
155+
"publishedAt":"2023-07-21T16:30:05Z",
156+
"channelId":"UCwozCpFp9g9x0wAzuFh0hwQ",
157+
"title":"3 Things You Didn't Know About Erling Haaland ⚽️🇳🇴 #football #haaland #shorts",
158+
"description":"",
159+
"thumbnails":{
160+
"default":{
161+
"url":"https://i.ytimg.com/vi/XH0rtu4U6SE/default.jpg",
162+
"width":120,
163+
"height":90
164+
},
165+
"medium":{
166+
"url":"https://i.ytimg.com/vi/XH0rtu4U6SE/mqdefault.jpg",
167+
"width":320,
168+
"height":180
169+
},
170+
"high":{
171+
"url":"https://i.ytimg.com/vi/XH0rtu4U6SE/hqdefault.jpg",
172+
"width":480,
173+
"height":360
174+
}
175+
},
176+
"channelTitle":"FC Motivate",
177+
"liveBroadcastContent":"none",
178+
"publishTime":"2023-07-21T16:30:05Z"
179+
}
180+
}
181+
]
182+
}

‎examples/azure/inputs/username.csv‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Username; Identifier;First name;Last name
2+
booker12;9012;Rachel;Booker
3+
grey07;2070;Laura;Grey
4+
johnson81;4081;Craig;Johnson
5+
jenkins46;9346;Mary;Jenkins
6+
smith79;5079;Jamie;Smith
7+

‎examples/azure/json_scraper_azure.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key
3+
"""
4+
5+
import os
6+
from dotenv import load_dotenv
7+
from langchain_openai import AzureChatOpenAI
8+
from langchain_openai import AzureOpenAIEmbeddings
9+
from scrapegraphai.graphs import JSONScraperGraph
10+
from scrapegraphai.utils import prettify_exec_info
11+
12+
13+
# required environment variable in .env
14+
# AZURE_OPENAI_ENDPOINT
15+
# AZURE_OPENAI_CHAT_DEPLOYMENT_NAME
16+
# MODEL_NAME
17+
# AZURE_OPENAI_API_KEY
18+
# OPENAI_API_TYPE
19+
# AZURE_OPENAI_API_VERSION
20+
# AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME
21+
load_dotenv()
22+
23+
24+
FILE_NAME = "inputs/example.json"
25+
curr_dir = os.path.dirname(os.path.realpath(__file__))
26+
file_path = os.path.join(curr_dir, FILE_NAME)
27+
28+
with open(file_path, 'r', encoding="utf-8") as file:
29+
text = file.read()
30+
31+
# ************************************************
32+
# Initialize the model instances
33+
# ************************************************
34+
35+
llm_model_instance = AzureChatOpenAI(
36+
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
37+
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
38+
)
39+
40+
embedder_model_instance = AzureOpenAIEmbeddings(
41+
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
42+
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
43+
)
44+
45+
# ************************************************
46+
# Create the JSONScraperGraph instance and run it
47+
# ************************************************
48+
49+
graph_config = {
50+
"llm": {"model_instance": llm_model_instance},
51+
"embeddings": {"model_instance": embedder_model_instance}
52+
}
53+
54+
smart_scraper_graph = JSONScraperGraph(
55+
prompt="List me all the authors, title and genres of the books",
56+
source=text, # Pass the content of the file, not the file object
57+
config=graph_config
58+
)
59+
60+
result = smart_scraper_graph.run()
61+
print(result)
62+
63+
# ************************************************
64+
# Get graph execution info
65+
# ************************************************
66+
67+
graph_exec_info = smart_scraper_graph.get_execution_info()
68+
print(prettify_exec_info(graph_exec_info))

0 commit comments

Comments
 (0)