chore: pandas package is now optional

Author: PeriniM

examples/anthropic/csv_scraper_anthropic.py

@@ -3,9 +3,8 @@
 """
 import os
 from dotenv import load_dotenv
-import pandas as pd
 from scrapegraphai.graphs import CSVScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 
@@ -17,7 +16,8 @@
 curr_dir = os.path.dirname(os.path.realpath(__file__))
 file_path = os.path.join(curr_dir, FILE_NAME)
 
-text = pd.read_csv(file_path)
+with open(file_path, 'r') as file:
+    text = file.read()
 
 # ************************************************
 # Define the configuration for the graph
@@ -41,7 +41,7 @@
 
 csv_scraper_graph = CSVScraperGraph(
     prompt="List me all the last names",
-    source=str(text),  # Pass the content of the file, not the file object
+    source=text,  # Pass the content of the file
     config=graph_config
 )
 
@@ -53,8 +53,4 @@
 # ************************************************
 
 graph_exec_info = csv_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")
+print(prettify_exec_info(graph_exec_info))

examples/anthropic/csv_scraper_graph_multi_anthropic.py

@@ -3,9 +3,8 @@
 """
 import os
 from dotenv import load_dotenv
-import pandas as pd
 from scrapegraphai.graphs import CSVScraperMultiGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 # ************************************************
@@ -16,7 +15,8 @@
 curr_dir = os.path.dirname(os.path.realpath(__file__))
 file_path = os.path.join(curr_dir, FILE_NAME)
 
-text = pd.read_csv(file_path)
+with open(file_path, 'r') as file:
+    text = file.read()
 
 # ************************************************
 # Define the configuration for the graph
@@ -48,7 +48,3 @@
 
 graph_exec_info = csv_scraper_graph.get_execution_info()
 print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")

examples/openai/smart_scraper_openai.py

@@ -28,7 +28,7 @@
 # ************************************************
 
 smart_scraper_graph = SmartScraperGraph(
-    prompt="Extract me all the articles",
+    prompt="Extract me the first article",
     source="https://www.wired.com",
     config=graph_config
 )

pyproject.toml

@@ -19,7 +19,6 @@ dependencies = [
     "mistral-common>=1.4.0",
     "html2text>=2024.2.26",
     "beautifulsoup4>=4.12.3",
-    "pandas>=2.2.2",
     "python-dotenv>=1.0.1",
     "tiktoken>=0.7",
     "tqdm>=4.66.4",
@@ -28,9 +27,10 @@ dependencies = [
     "playwright>=1.43.0",
     "undetected-playwright>=0.3.0",
     "langchain-ollama>=0.1.3",
+    "semchunk>=2.2.0",
     "qdrant-client>=1.11.3",
     "fastembed>=0.3.6",
-    "semchunk>=2.2.0",
+    
     "transformers>=4.44.2",
     "googlesearch-python>=1.2.5",
     "async-timeout>=4.0.3",

scrapegraphai/nodes/fetch_node.py

@@ -4,7 +4,6 @@
 import json
 from typing import List, Optional
 from langchain_openai import ChatOpenAI, AzureChatOpenAI
-import pandas as pd
 import requests
 from langchain_community.document_loaders import PyPDFLoader
 from langchain_core.documents import Document
@@ -199,6 +198,10 @@ def load_file_content(self, source, input_type):
             loader = PyPDFLoader(source)
             return loader.load()
         elif input_type == "csv":
+            try:
+                import pandas as pd
+            except ImportError:
+                raise ImportError("pandas is not installed. Please install it using `pip install pandas`.")
             return [
                 Document(
                     page_content=str(pd.read_csv(source)), metadata={"source": "csv"}

scrapegraphai/utils/prettify_exec_info.py

@@ -1,25 +1,45 @@
 """
 Prettify the execution information of the graph.
 """
-import pandas as pd
+from typing import Union
 
-def prettify_exec_info(complete_result: list[dict]) -> pd.DataFrame:
+def prettify_exec_info(complete_result: list[dict], as_string: bool = True) -> Union[str, list[dict]]:
     """
-    Transforms the execution information of a graph into a DataFrame for enhanced visualization.
+    Formats the execution information of a graph showing node statistics.
 
     Args:
-        complete_result (list[dict]): The complete execution information of the graph.
+        complete_result (list[dict]): The execution information containing node statistics.
+        as_string (bool, optional): If True, returns a formatted string table. 
+                                  If False, returns the original list. Defaults to True.
 
     Returns:
-        pd.DataFrame: A DataFrame that organizes the execution information 
-        for better readability and analysis.
-
-    Example:
-        >>> prettify_exec_info([{'node': 'A', 'status': 'success'},
-          {'node': 'B', 'status': 'failure'}])
-        DataFrame with columns 'node' and 'status' showing execution results for each node.
+        Union[str, list[dict]]: A formatted string table if as_string=True, 
+        otherwise the original list of dictionaries.
     """
+    if not as_string:
+        return complete_result
+
+    if not complete_result:
+        return "Empty result"
+
+    # Format the table
+    lines = []
+    lines.append("Node Statistics:")
+    lines.append("-" * 100)
+    lines.append(f"{'Node':<20} {'Tokens':<10} {'Prompt':<10} {'Compl.':<10} {'Requests':<10} {'Cost ($)':<10} {'Time (s)':<10}")
+    lines.append("-" * 100)
+
+    for item in complete_result:
+        node = item['node_name']
+        tokens = item['total_tokens']
+        prompt = item['prompt_tokens']
+        completion = item['completion_tokens']
+        requests = item['successful_requests']
+        cost = f"{item['total_cost_USD']:.4f}"
+        time = f"{item['exec_time']:.2f}"
 
-    df_nodes = pd.DataFrame(complete_result)
+        lines.append(
+            f"{node:<20} {tokens:<10} {prompt:<10} {completion:<10} {requests:<10} {cost:<10} {time:<10}"
+        )
 
-    return df_nodes
+    return "\n".join(lines)

tests/graphs/scrape_graph_test.py

@@ -4,10 +4,8 @@
 
 import os
 import pytest
-import pandas as pd
 from dotenv import load_dotenv
 from scrapegraphai.graphs import ScrapeGraph
-from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 

tests/graphs/smart_scraper_fireworks_test.py

@@ -4,10 +4,8 @@
 
 import os
 import pytest
-import pandas as pd
 from dotenv import load_dotenv
 from scrapegraphai.graphs import SmartScraperGraph
-from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 

tests/graphs/smart_scraper_multi_lite_graph_openai_test.py

@@ -4,10 +4,8 @@
 
 import os
 import pytest
-import pandas as pd
 from dotenv import load_dotenv
 from scrapegraphai.graphs import SmartScraperMultiLiteGraph
-from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 

tests/graphs/smart_scraper_openai_test.py

@@ -4,10 +4,8 @@
 
 import os
 import pytest
-import pandas as pd
 from dotenv import load_dotenv
 from scrapegraphai.graphs import SmartScraperGraph
-from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()