Skip to content

Commit ba782a6

Browse files
VinciGit00VinciGit00
committed
add compatibility for versions
1 parent 765b548 commit ba782a6

File tree

5 files changed

+111
-12
lines changed

5 files changed

+111
-12
lines changed

‎pyproject.toml‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,17 @@
11
[project]
22
name = "scrapegraphai"
3-
4-
53
version = "1.8.1b1"
6-
7-
84
description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
95
authors = [
106
{ name = "Marco Vinciguerra", email = "[email protected]" },
117
{ name = "Marco Perini", email = "[email protected]" },
128
{ name = "Lorenzo Padoan", email = "[email protected]" }
139
]
1410
dependencies = [
15-
"langchain==0.1.15",
16-
"langchain-openai==0.1.6",
17-
"langchain-google-genai==1.0.3",
18-
"langchain-google-vertexai==1.0.6",
11+
"langchain",
12+
"langchain-google-genai",
13+
"langchain-google-vertexai",
14+
"langchain-openai",
1915
"langchain-groq==0.1.3",
2016
"langchain-aws==0.1.3",
2117
"langchain-anthropic==0.1.11",
@@ -91,5 +87,3 @@ dev-dependencies = [
9187
[tool.rye.scripts]
9288
pylint-local = "pylint scrapegraphai/**/*.py"
9389
pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
94-
pylint-score-ci = "pylint --disable=all --enable=metrics --output-format=text scrapegraphai/**/.py"
95-

‎requirements-dev.lock‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ distro==1.9.0
9494
# via openai
9595
dnspython==2.6.1
9696
# via email-validator
97+
docstring-parser==0.16
98+
# via google-cloud-aiplatform
9799
docutils==0.19
98100
# via sphinx
99101
email-validator==2.1.1
@@ -136,6 +138,11 @@ google-ai-generativelanguage==0.6.4
136138
google-api-core==2.19.0
137139
# via google-ai-generativelanguage
138140
# via google-api-python-client
141+
# via google-cloud-aiplatform
142+
# via google-cloud-bigquery
143+
# via google-cloud-core
144+
# via google-cloud-resource-manager
145+
# via google-cloud-storage
139146
# via google-generativeai
140147
google-api-python-client==2.130.0
141148
# via google-generativeai
@@ -144,13 +151,37 @@ google-auth==2.29.0
144151
# via google-api-core
145152
# via google-api-python-client
146153
# via google-auth-httplib2
154+
# via google-cloud-aiplatform
155+
# via google-cloud-bigquery
156+
# via google-cloud-core
157+
# via google-cloud-resource-manager
158+
# via google-cloud-storage
147159
# via google-generativeai
148160
google-auth-httplib2==0.2.0
149161
# via google-api-python-client
162+
google-cloud-aiplatform==1.58.0
163+
# via langchain-google-vertexai
164+
google-cloud-bigquery==3.25.0
165+
# via google-cloud-aiplatform
166+
google-cloud-core==2.4.1
167+
# via google-cloud-bigquery
168+
# via google-cloud-storage
169+
google-cloud-resource-manager==1.12.3
170+
# via google-cloud-aiplatform
171+
google-cloud-storage==2.17.0
172+
# via google-cloud-aiplatform
173+
# via langchain-google-vertexai
174+
google-crc32c==1.5.0
175+
# via google-cloud-storage
176+
# via google-resumable-media
150177
google-generativeai==0.5.4
151178
# via langchain-google-genai
179+
google-resumable-media==2.7.1
180+
# via google-cloud-bigquery
181+
# via google-cloud-storage
152182
googleapis-common-protos==1.63.0
153183
# via google-api-core
184+
# via grpc-google-iam-v1
154185
# via grpcio-status
155186
graphviz==0.20.3
156187
# via burr
@@ -159,8 +190,12 @@ greenlet==3.0.3
159190
# via playwright
160191
groq==0.8.0
161192
# via langchain-groq
193+
grpc-google-iam-v1==0.13.1
194+
# via google-cloud-resource-manager
162195
grpcio==1.64.0
163196
# via google-api-core
197+
# via googleapis-common-protos
198+
# via grpc-google-iam-v1
164199
# via grpcio-status
165200
grpcio-status==1.62.2
166201
# via google-api-core
@@ -243,13 +278,16 @@ langchain-core==0.1.52
243278
# via langchain-community
244279
# via langchain-fireworks
245280
# via langchain-google-genai
281+
# via langchain-google-vertexai
246282
# via langchain-groq
247283
# via langchain-openai
248284
# via langchain-text-splitters
249285
langchain-fireworks==0.1.3
250286
# via scrapegraphai
251287
langchain-google-genai==1.0.3
252288
# via scrapegraphai
289+
langchain-google-vertexai==1.0.4
290+
# via scrapegraphai
253291
langchain-groq==0.1.3
254292
# via scrapegraphai
255293
langchain-openai==0.1.6
@@ -301,6 +339,7 @@ numpy==1.26.4
301339
# via pyarrow
302340
# via pydeck
303341
# via sf-hamilton
342+
# via shapely
304343
# via streamlit
305344
openai==1.30.3
306345
# via burr
@@ -311,6 +350,8 @@ orjson==3.10.3
311350
# via langsmith
312351
packaging==23.2
313352
# via altair
353+
# via google-cloud-aiplatform
354+
# via google-cloud-bigquery
314355
# via huggingface-hub
315356
# via langchain-core
316357
# via marshmallow
@@ -337,11 +378,16 @@ pluggy==1.5.0
337378
proto-plus==1.23.0
338379
# via google-ai-generativelanguage
339380
# via google-api-core
381+
# via google-cloud-aiplatform
382+
# via google-cloud-resource-manager
340383
protobuf==4.25.3
341384
# via google-ai-generativelanguage
342385
# via google-api-core
386+
# via google-cloud-aiplatform
387+
# via google-cloud-resource-manager
343388
# via google-generativeai
344389
# via googleapis-common-protos
390+
# via grpc-google-iam-v1
345391
# via grpcio-status
346392
# via proto-plus
347393
# via streamlit
@@ -358,6 +404,7 @@ pydantic==2.7.1
358404
# via fastapi
359405
# via fastapi-pagination
360406
# via fireworks-ai
407+
# via google-cloud-aiplatform
361408
# via google-generativeai
362409
# via groq
363410
# via langchain
@@ -384,6 +431,7 @@ pytest-mock==3.14.0
384431
python-dateutil==2.9.0.post0
385432
# via botocore
386433
# via dateparser
434+
# via google-cloud-bigquery
387435
# via htmldate
388436
# via matplotlib
389437
# via pandas
@@ -411,6 +459,8 @@ requests==2.32.2
411459
# via burr
412460
# via free-proxy
413461
# via google-api-core
462+
# via google-cloud-bigquery
463+
# via google-cloud-storage
414464
# via huggingface-hub
415465
# via langchain
416466
# via langchain-community
@@ -433,6 +483,8 @@ semchunk==1.0.1
433483
# via scrapegraphai
434484
sf-hamilton==1.63.0
435485
# via burr
486+
shapely==2.0.4
487+
# via google-cloud-aiplatform
436488
shellingham==1.5.4
437489
# via typer
438490
six==1.16.0

‎requirements.lock‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ distro==1.9.0
6262
# via anthropic
6363
# via groq
6464
# via openai
65+
docstring-parser==0.16
66+
# via google-cloud-aiplatform
6567
exceptiongroup==1.2.1
6668
# via anyio
6769
faiss-cpu==1.8.0
@@ -84,6 +86,11 @@ google-ai-generativelanguage==0.6.4
8486
google-api-core==2.19.0
8587
# via google-ai-generativelanguage
8688
# via google-api-python-client
89+
# via google-cloud-aiplatform
90+
# via google-cloud-bigquery
91+
# via google-cloud-core
92+
# via google-cloud-resource-manager
93+
# via google-cloud-storage
8794
# via google-generativeai
8895
google-api-python-client==2.130.0
8996
# via google-generativeai
@@ -92,22 +99,50 @@ google-auth==2.29.0
9299
# via google-api-core
93100
# via google-api-python-client
94101
# via google-auth-httplib2
102+
# via google-cloud-aiplatform
103+
# via google-cloud-bigquery
104+
# via google-cloud-core
105+
# via google-cloud-resource-manager
106+
# via google-cloud-storage
95107
# via google-generativeai
96108
google-auth-httplib2==0.2.0
97109
# via google-api-python-client
110+
google-cloud-aiplatform==1.58.0
111+
# via langchain-google-vertexai
112+
google-cloud-bigquery==3.25.0
113+
# via google-cloud-aiplatform
114+
google-cloud-core==2.4.1
115+
# via google-cloud-bigquery
116+
# via google-cloud-storage
117+
google-cloud-resource-manager==1.12.3
118+
# via google-cloud-aiplatform
119+
google-cloud-storage==2.17.0
120+
# via google-cloud-aiplatform
121+
# via langchain-google-vertexai
122+
google-crc32c==1.5.0
123+
# via google-cloud-storage
124+
# via google-resumable-media
98125
google-generativeai==0.5.4
99126
# via langchain-google-genai
127+
google-resumable-media==2.7.1
128+
# via google-cloud-bigquery
129+
# via google-cloud-storage
100130
googleapis-common-protos==1.63.0
101131
# via google-api-core
132+
# via grpc-google-iam-v1
102133
# via grpcio-status
103134
graphviz==0.20.3
104135
# via scrapegraphai
105136
greenlet==3.0.3
106137
# via playwright
107138
groq==0.8.0
108139
# via langchain-groq
140+
grpc-google-iam-v1==0.13.1
141+
# via google-cloud-resource-manager
109142
grpcio==1.64.0
110143
# via google-api-core
144+
# via googleapis-common-protos
145+
# via grpc-google-iam-v1
111146
# via grpcio-status
112147
grpcio-status==1.62.2
113148
# via google-api-core
@@ -163,13 +198,16 @@ langchain-core==0.1.52
163198
# via langchain-community
164199
# via langchain-fireworks
165200
# via langchain-google-genai
201+
# via langchain-google-vertexai
166202
# via langchain-groq
167203
# via langchain-openai
168204
# via langchain-text-splitters
169205
langchain-fireworks==0.1.3
170206
# via scrapegraphai
171207
langchain-google-genai==1.0.3
172208
# via scrapegraphai
209+
langchain-google-vertexai==1.0.4
210+
# via scrapegraphai
173211
langchain-groq==0.1.3
174212
# via scrapegraphai
175213
langchain-openai==0.1.6
@@ -203,12 +241,15 @@ numpy==1.26.4
203241
# via langchain-aws
204242
# via langchain-community
205243
# via pandas
244+
# via shapely
206245
openai==1.30.3
207246
# via langchain-fireworks
208247
# via langchain-openai
209248
orjson==3.10.3
210249
# via langsmith
211250
packaging==23.2
251+
# via google-cloud-aiplatform
252+
# via google-cloud-bigquery
212253
# via huggingface-hub
213254
# via langchain-core
214255
# via marshmallow
@@ -222,11 +263,16 @@ playwright==1.43.0
222263
proto-plus==1.23.0
223264
# via google-ai-generativelanguage
224265
# via google-api-core
266+
# via google-cloud-aiplatform
267+
# via google-cloud-resource-manager
225268
protobuf==4.25.3
226269
# via google-ai-generativelanguage
227270
# via google-api-core
271+
# via google-cloud-aiplatform
272+
# via google-cloud-resource-manager
228273
# via google-generativeai
229274
# via googleapis-common-protos
275+
# via grpc-google-iam-v1
230276
# via grpcio-status
231277
# via proto-plus
232278
pyasn1==0.6.0
@@ -237,6 +283,7 @@ pyasn1-modules==0.4.0
237283
pydantic==2.7.1
238284
# via anthropic
239285
# via fireworks-ai
286+
# via google-cloud-aiplatform
240287
# via google-generativeai
241288
# via groq
242289
# via langchain
@@ -252,6 +299,7 @@ pyparsing==3.1.2
252299
python-dateutil==2.9.0.post0
253300
# via botocore
254301
# via dateparser
302+
# via google-cloud-bigquery
255303
# via htmldate
256304
# via pandas
257305
python-dotenv==1.0.1
@@ -270,6 +318,8 @@ regex==2024.5.15
270318
requests==2.32.2
271319
# via free-proxy
272320
# via google-api-core
321+
# via google-cloud-bigquery
322+
# via google-cloud-storage
273323
# via huggingface-hub
274324
# via langchain
275325
# via langchain-community
@@ -282,6 +332,8 @@ s3transfer==0.10.1
282332
# via boto3
283333
semchunk==1.0.1
284334
# via scrapegraphai
335+
shapely==2.0.4
336+
# via google-cloud-aiplatform
285337
six==1.16.0
286338
# via python-dateutil
287339
sniffio==1.3.1

‎tests/graphs/.env.example‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
OPENAI_API_KEY="YOUR OPENAI API KEY"
1+
OPENAI_API_KEY="YOUR OPENAI API KEY"
2+
FIREWORKS_APIKEY="YOOUR FIREWORK KEY"

‎tests/graphs/smart_scraper_fireworks_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
@pytest.fixture
1515
def graph_config():
1616
"""Configuration of the graph"""
17-
fireworks_api_key = os.getenv("FIREWORKS_APIKEY")
17+
fireworks_api_key = os.getenv("FIREWORKS_APIKEY")
1818
return {
1919
"llm": {
2020
"api_key": fireworks_api_key,

0 commit comments

Comments
 (0)