feat(indexify-node): add example

Author: PeriniM

examples/integrations/indexify_node_example.py

@@ -0,0 +1,72 @@
+""" 
+Basic example of scraping pipeline using SmartScraper with schema
+"""
+
+import os, json
+from typing import List
+
+from dotenv import load_dotenv
+load_dotenv()
+
+from pydantic import BaseModel, Field
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.integrations import IndexifyNode
+
+
+# ************************************************
+# Define the output schema for the graph
+# ************************************************
+
+class Image(BaseModel):
+    url: str = Field(description="The url of the image")
+
+class Images(BaseModel):
+    images: List[Image]
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key":openai_key,
+        "model": "gpt-3.5-turbo",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+# ************************************************
+# Define the custom nodes for the graph
+# ************************************************
+
+indexify_node = IndexifyNode(
+    input="answer & img_urls",
+    output=["is_indexed"],
+    node_config={
+        "verbose": True
+    }
+)
+
+# ************************************************
+# Create the SmartScraperGraph instance
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me all the images with their url",
+    source="https://giphy.com/",
+    schema=Images,
+    config=graph_config
+)
+
+# Add the custom node to the graph
+smart_scraper_graph.append_node(indexify_node)
+
+# ************************************************
+# Run the SmartScraperGraph
+# ************************************************
+
+result = smart_scraper_graph.run()
+print(json.dumps(result, indent=2))

scrapegraphai/integrations/__init__.py

@@ -2,4 +2,5 @@
 Init file for integrations module
 """
 
-from .burr_bridge import BurrBridge
+from .burr_bridge import BurrBridge
+from .indexify_node import IndexifyNode

scrapegraphai/integrations/indexify_node.py

@@ -0,0 +1,79 @@
+"""
+IndexifyNode Module
+"""
+
+from typing import List, Optional
+
+from ..utils.logging import get_logger
+from ..nodes.base_node import BaseNode
+
+# try:
+#     import indexify
+# except ImportError:
+#     raise ImportError("indexify package is not installed. Please install it with 'pip install scrapegraphai[indexify]'")
+
+
+class IndexifyNode(BaseNode):
+    """
+    A node responsible for indexing the content present in the state.
+
+    Attributes:
+        verbose (bool): A flag indicating whether to show print statements during execution.
+
+    Args:
+        input (str): Boolean expression defining the input keys needed from the state.
+        output (List[str]): List of output keys to be updated in the state.
+        node_config (dict): Additional configuration for the node.
+        node_name (str): The unique identifier name for the node, defaulting to "Parse".
+    """
+
+    def __init__(
+        self,
+        input: str,
+        output: List[str],
+        node_config: Optional[dict] = None,
+        node_name: str = "Indexify",
+    ):
+        super().__init__(node_name, "node", input, output, 2, node_config)
+
+        self.verbose = (
+            False if node_config is None else node_config.get("verbose", False)
+        )
+
+    def execute(self, state: dict) -> dict:
+        """
+        Executes the node's logic to index the content present in the state.
+
+        Args:
+            state (dict): The current state of the graph. The input keys will be used to fetch the
+                            correct data from the state.
+
+        Returns:
+            dict: The updated state with the output key containing the parsed content chunks.
+
+        Raises:
+            KeyError: If the input keys are not found in the state, indicating that the
+                        necessary information for parsing the content is missing.
+        """
+
+        self.logger.info(f"--- Executing {self.node_name} Node ---")
+
+        # Interpret input keys based on the provided input expression
+        # input_keys length matches the min_input_len parameter in the __init__ method
+        # e.g. "answer & parsed_doc" or "answer | img_urls"
+        
+        input_keys = self.get_input_keys(state)
+
+        # Fetching data from the state based on the input keys
+        input_data = [state[key] for key in input_keys]
+
+        answer = input_data[0]
+        img_urls = input_data[1]
+
+        # Indexify the content
+        # ...
+
+        isIndexified = True
+        state.update({self.output[0]: isIndexified})
+
+        return state