refactoring of nodes

Author: VinciGit00

examples/openai/custom_graph_openai.py

@@ -6,7 +6,7 @@
 from dotenv import load_dotenv
 from scrapegraphai.models import OpenAI
 from scrapegraphai.graphs import BaseGraph
-from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode
+from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode
 load_dotenv()
 
 # ************************************************
@@ -31,6 +31,12 @@
 llm_model = OpenAI(graph_config["llm"])
 
 # define the nodes for the graph
+robot_node = RobotsNode(
+    input="url",
+    output=["is_scrapable"],
+    node_config={"llm": llm_model}
+)
+
 fetch_node = FetchNode(
     input="url | local_dir",
     output=["doc"],
@@ -57,12 +63,14 @@
 
 graph = BaseGraph(
     nodes={
+        robot_node,
         fetch_node,
         parse_node,
         rag_node,
         generate_answer_node,
     },
     edges={
+        (robot_node, fetch_node),
         (fetch_node, parse_node),
         (parse_node, rag_node),
         (rag_node, generate_answer_node)

tests/node_test.py renamed to examples/single_node/robot_node.py

@@ -0,0 +1,34 @@
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <commit_message>"
+    exit 1
+fi
+
+cd ..
+
+# Extract the commit message from the argument
+commit_message="$1"
+
+# Run Pylint on the specified Python files
+pylint pylint scrapegraphai/**/*.py scrapegraphai/*.py tests/**/*.py
+
+cd tests
+
+# Run pytest
+if ! pytest; then
+    echo "Pytest failed. Aborting commit and push."
+    exit 1
+fi
+
+cd ..
+
+# Make the pull
+git pull
+
+# Add the modified files to the Git repository
+git add .
+
+# Commit the changes with the provided message
+git commit -m "$commit_message"
+
+# Push the changes to the remote repository
+git push

manual deployment/commit_and_push_with_tests.sh

@@ -82,8 +82,6 @@ def execute(self, initial_state: dict) -> dict:
 
             with get_openai_callback() as cb:
                 result = current_node.execute(state)
-                # ADd the check for the node RObots
-
                 node_exec_time = time.time() - curr_time
                 total_exec_time += node_exec_time
 

scrapegraphai/graphs/base_graph.py

@@ -137,7 +137,8 @@ def execute(self, state):
             if "no" in is_scrapable:
                 print("\033[33mScraping this website is not allowed\033[0m")
                 if not self.force_scraping:
-                    return {"update": "block the scraping phase"}
+                    raise ValueError(
+                        'The website you selected is not scrapable')
             else:
                 print("\033[92mThe path is scrapable\033[0m")
 

scrapegraphai/nodes/robots_node.py

@@ -6,5 +6,5 @@ Remember to activating Ollama and having installed the LLM on your pc
 
 For running the tests run the command:
 ```python
-pytests
+pytest
 ```