Skip to content

Commit a6e1813

Browse files
committed
fix(fetch_node): bug in handling local files
1 parent fcb3abb commit a6e1813

File tree

11 files changed

+34
-135
lines changed

11 files changed

+34
-135
lines changed

examples/openai/custom_graph_openai copy.py

Lines changed: 0 additions & 113 deletions
This file was deleted.

examples/openai/omni_scraper_openai.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
graph_config = {
2020
"llm": {
2121
"api_key": openai_key,
22-
"model": "gpt-4o",
22+
"model": "gpt-4-turbo",
2323
},
2424
"verbose": True,
2525
"headless": True,

scrapegraphai/graphs/csv_scraper_graph.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ def _create_graph(self):
3030
Creates the graph of nodes representing the workflow for web scraping.
3131
"""
3232
fetch_node = FetchNode(
33-
input="csv",
34-
output=["doc"],
33+
input="csv | csv_dir",
34+
output=["doc", "link_urls", "img_urls"],
3535
)
3636
parse_node = ParseNode(
3737
input="doc",

scrapegraphai/graphs/deep_scraper_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def _create_graph(self) -> BaseGraph:
6161
"""
6262
fetch_node = FetchNode(
6363
input="url | local_dir",
64-
output=["doc"]
64+
output=["doc", "link_urls", "img_urls"]
6565
)
6666
parse_node = ParseNode(
6767
input="doc",

scrapegraphai/graphs/json_scraper_graph.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ def _create_graph(self) -> BaseGraph:
5454
"""
5555

5656
fetch_node = FetchNode(
57-
input="json",
58-
output=["doc"],
57+
input="json | json_dir",
58+
output=["doc", "link_urls", "img_urls"],
5959
)
6060
parse_node = ParseNode(
6161
input="doc",

scrapegraphai/graphs/pdf_scraper_graph.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ def _create_graph(self) -> BaseGraph:
5656
"""
5757

5858
fetch_node = FetchNode(
59-
input='pdf',
60-
output=["doc"],
59+
input='pdf | pdf_dir',
60+
output=["doc", "link_urls", "img_urls"],
6161
)
6262
parse_node = ParseNode(
6363
input="doc",

scrapegraphai/graphs/script_creator_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ def _create_graph(self) -> BaseGraph:
5959

6060
fetch_node = FetchNode(
6161
input="url | local_dir",
62-
output=["doc"],
62+
output=["doc", "link_urls", "img_urls"],
6363
)
6464
parse_node = ParseNode(
6565
input="doc",

scrapegraphai/graphs/smart_scraper_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ def _create_graph(self) -> BaseGraph:
5757
"""
5858
fetch_node = FetchNode(
5959
input="url | local_dir",
60-
output=["doc"],
60+
output=["doc", "link_urls", "img_urls"],
6161
node_config={
6262
"loader_kwargs": self.config.get("loader_kwargs", {}),
6363
}

scrapegraphai/graphs/speech_graph.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def _create_graph(self) -> BaseGraph:
5656

5757
fetch_node = FetchNode(
5858
input="url | local_dir",
59-
output=["doc"]
59+
output=["doc", "link_urls", "img_urls"]
6060
)
6161
parse_node = ParseNode(
6262
input="doc",

scrapegraphai/graphs/xml_scraper_graph.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ def _create_graph(self) -> BaseGraph:
5656
"""
5757

5858
fetch_node = FetchNode(
59-
input="xml",
60-
output=["doc"]
59+
input="xml | xml_dir",
60+
output=["doc", "link_urls", "img_urls"]
6161
)
6262
parse_node = ParseNode(
6363
input="doc",

0 commit comments

Comments
 (0)