Merge pull request #161 from cemkod/main

Support for Anthropic Claude 3 models

Author: VinciGit00

examples/anthropic/smart_scraper_haiku.py

@@ -0,0 +1,59 @@
+""" 
+Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+from langchain_community.llms import HuggingFaceEndpoint
+from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
+
+
+# required environment variables in .env
+# HUGGINGFACEHUB_API_TOKEN
+# ANTHROPIC_API_KEY
+load_dotenv()
+
+HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+# ************************************************
+# Initialize the model instances
+# ************************************************
+
+
+embedder_model_instance = HuggingFaceInferenceAPIEmbeddings(
+    api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2"
+)
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "claude-3-haiku-20240307",
+        "max_tokens": 4000},
+    "embeddings": {"model_instance": embedder_model_instance}
+}
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="""Don't say anything else. Output JSON only. List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, 
+    event_end_date, event_end_time, location, event_mode, event_category, 
+    third_party_redirect, no_of_days, 
+    time_in_hours, hosted_or_attending, refreshments_type, 
+    registration_available, registration_link""",
+    # also accepts a string with the already downloaded HTML code
+    source="https://www.hmhco.com/event",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

requirements.txt

@@ -1,6 +1,7 @@
 langchain==0.1.14
 langchain-openai==0.1.1
 langchain-google-genai==1.0.1
+langchain-anthropic==0.1.11
 html2text==2020.1.16
 faiss-cpu==1.8.0
 beautifulsoup4==4.12.3

scrapegraphai/graphs/abstract_graph.py

@@ -6,9 +6,7 @@
 from langchain_openai import AzureOpenAIEmbeddings, OpenAIEmbeddings
 from langchain_community.embeddings import HuggingFaceHubEmbeddings, OllamaEmbeddings
 from ..helpers import models_tokens
-from ..models import AzureOpenAI, Bedrock, Gemini, Groq, HuggingFace, Ollama, OpenAI, Claude
-from langchain_aws.embeddings.bedrock import BedrockEmbeddings
-from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from ..models import AzureOpenAI, Bedrock, Gemini, Groq, HuggingFace, Ollama, OpenAI, Anthropic
 
 
 class AbstractGraph(ABC):
@@ -200,6 +198,9 @@ def _create_llm(self, llm_config: dict, chat=False) -> object:
                     "temperature": llm_params["temperature"],
                 }
             })
+        elif "claude-3-" in llm_params["model"]:
+            self.model_token = models_tokens["claude"]["claude3"]
+            return Anthropic(llm_params)
         else:
             raise ValueError(
                 "Model provided by the configuration not supported")

scrapegraphai/graphs/base_graph.py

@@ -7,6 +7,7 @@
 from langchain_community.callbacks import get_openai_callback
 from typing import Tuple
 
+
 class BaseGraph:
     """
     BaseGraph manages the execution flow of a graph composed of interconnected nodes.
@@ -82,7 +83,7 @@ def execute(self, initial_state: dict) -> Tuple[dict, list]:
         Returns:
             Tuple[dict, list]: A tuple containing the final state and a list of execution info.
         """
-        
+
         current_node_name = self.nodes[0]
         state = initial_state
 

scrapegraphai/models/__init__.py

@@ -11,4 +11,4 @@
 from .hugging_face import HuggingFace
 from .groq import Groq
 from .bedrock import Bedrock
-from .claude import Claude
+from .anthropic import Anthropic

scrapegraphai/models/anthropic.py

@@ -0,0 +1,17 @@
+""" 
+Anthropic Module
+"""
+from langchain_anthropic import ChatAnthropic
+
+
+class Anthropic(ChatAnthropic):
+    """
+    A wrapper for the ChatAnthropic class that provides default configuration
+    and could be extended with additional methods if needed.
+
+    Args:
+        llm_config (dict): Configuration parameters for the language model.
+    """
+
+    def __init__(self, llm_config: dict):
+        super().__init__(**llm_config)