fix: Removed link_urls and img_ulrs from FetchNode output

LorenzoPaleari · LorenzoPaleari · commit 57337a0a8c86 · 2024-09-02T12:53:40.000+02:00
diff --git a/examples/anthropic/custom_graph_haiku.py b/examples/anthropic/custom_graph_haiku.py
@@ -40,7 +40,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/bedrock/custom_graph_bedrock.py b/examples/bedrock/custom_graph_bedrock.py
@@ -55,7 +55,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/ernie/custom_graph_ernie.py b/examples/ernie/custom_graph_ernie.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/fireworks/custom_graph_fireworks.py b/examples/fireworks/custom_graph_fireworks.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/groq/custom_graph_groq.py b/examples/groq/custom_graph_groq.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/huggingfacehub/custom_graph_huggingfacehub.py b/examples/huggingfacehub/custom_graph_huggingfacehub.py
@@ -55,7 +55,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/local_models/custom_graph_ollama.py b/examples/local_models/custom_graph_ollama.py
@@ -44,7 +44,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/mistral/custom_graph_mistral.py b/examples/mistral/custom_graph_mistral.py
@@ -42,7 +42,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/mixed_models/custom_graph_groq_openai.py b/examples/mixed_models/custom_graph_groq_openai.py
@@ -51,7 +51,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/nemotron/custom_graph_nemotron.py b/examples/nemotron/custom_graph_nemotron.py
@@ -42,7 +42,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/oneapi/custom_graph_oneapi.py b/examples/oneapi/custom_graph_oneapi.py
@@ -38,7 +38,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/examples/openai/custom_graph_openai.py b/examples/openai/custom_graph_openai.py
@@ -43,7 +43,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
diff --git a/scrapegraphai/graphs/deep_scraper_graph.py b/scrapegraphai/graphs/deep_scraper_graph.py
@@ -69,7 +69,7 @@ def _create_repeated_graph(self) -> BaseGraph:
         """
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc", "link_urls", "img_urls"]
+            output=["doc"]
         )
         parse_node = ParseNode(
             input="doc",
diff --git a/scrapegraphai/graphs/json_scraper_graph.py b/scrapegraphai/graphs/json_scraper_graph.py
@@ -56,7 +56,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="json | json_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
         )
 
         generate_answer_node = GenerateAnswerNode(
diff --git a/scrapegraphai/graphs/script_creator_graph.py b/scrapegraphai/graphs/script_creator_graph.py
@@ -62,7 +62,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "loader_kwargs": self.config.get("loader_kwargs", {}),
diff --git a/scrapegraphai/graphs/search_link_graph.py b/scrapegraphai/graphs/search_link_graph.py
@@ -52,7 +52,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="url| local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "force": self.config.get("force", False),
diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py
@@ -61,7 +61,7 @@ def _create_graph(self) -> BaseGraph:
         """
         fetch_node = FetchNode(
             input="url| local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "force": self.config.get("force", False),
diff --git a/scrapegraphai/graphs/speech_graph.py b/scrapegraphai/graphs/speech_graph.py
@@ -62,7 +62,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="url | local_dir",
-            output=["doc", "link_urls", "img_urls"]
+            output=["doc"]
         )
         parse_node = ParseNode(
             input="doc",
diff --git a/scrapegraphai/graphs/xml_scraper_graph.py b/scrapegraphai/graphs/xml_scraper_graph.py
@@ -60,7 +60,7 @@ def _create_graph(self) -> BaseGraph:
 
         fetch_node = FetchNode(
             input="xml | xml_dir",
-            output=["doc", "link_urls", "img_urls"]
+            output=["doc"]
         )
 
         generate_answer_node = GenerateAnswerNode(
diff --git a/tests/graphs/abstract_graph_test.py b/tests/graphs/abstract_graph_test.py
@@ -22,7 +22,7 @@ def __init__(self, prompt: str, config: dict):
     def _create_graph(self) -> BaseGraph:
         fetch_node = FetchNode(
             input="url| local_dir",
-            output=["doc", "link_urls", "img_urls"],
+            output=["doc"],
             node_config={
                 "llm_model": self.llm_model,
                 "force": self.config.get("force", False),
