Resolve merge conflicts

Author: shkamboj1

CHANGELOG.md

@@ -1,3 +1,47 @@
+## [0.8.0-beta.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.7.0...v0.8.0-beta.1) (2024-05-03)
+
+
+### Features
+
+* add pdf scraper ([10a9453](https://github.com/VinciGit00/Scrapegraph-ai/commit/10a94530e3fd4dfde933ecfa96cb3e21df72e606))
+
+
+### CI
+
+* **release:** 0.7.0-beta.3 [skip ci] ([fbb06ab](https://github.com/VinciGit00/Scrapegraph-ai/commit/fbb06ab551fac9cc9824ad567f042e55450277bd))
+
+## [0.7.0](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.6.2...v0.7.0) (2024-05-03)
+
+### Features
+
+* add base_node to __init__.py ([cb1cb61](https://github.com/VinciGit00/Scrapegraph-ai/commit/cb1cb616b7998d3624bf57b19b5f1b1945fea4ef))
+* Azure implementation + embeddings refactoring ([aa9271e](https://github.com/VinciGit00/Scrapegraph-ai/commit/aa9271e7bc4daa54860499d0615580b17550ff58))
+
+
+### Refactor
+
+* Changed the way embedding model is created in AbstractGraph class and removed handling of embedding model creation from RAGNode. Now AbstractGraph will call a dedicated method for embedding models instead of _create_llm. This makes it easy to use any LLM with any supported embedding model. ([819cbcd](https://github.com/VinciGit00/Scrapegraph-ai/commit/819cbcd3be1a8cb195de0b44c6b6d4d824e2a42a))
+
+
+### CI
+
+* **release:** 0.7.0-beta.1 [skip ci] ([98dec36](https://github.com/VinciGit00/Scrapegraph-ai/commit/98dec36c60d1dc8b072482e8d514c3869a45a3f8))
+* **release:** 0.7.0-beta.2 [skip ci] ([42fa02e](https://github.com/VinciGit00/Scrapegraph-ai/commit/42fa02e65a3a81796bd66e55cf9dd1d1b692cb89))
+
+
+## [0.7.0-beta.3](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.7.0-beta.2...v0.7.0-beta.3) (2024-05-03)
+## [0.7.0-beta.2](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.7.0-beta.1...v0.7.0-beta.2) (2024-05-03)
+
+
+### Features
+
+* Azure implementation + embeddings refactoring ([aa9271e](https://github.com/VinciGit00/Scrapegraph-ai/commit/aa9271e7bc4daa54860499d0615580b17550ff58))
+* add pdf scraper ([10a9453](https://github.com/VinciGit00/Scrapegraph-ai/commit/10a94530e3fd4dfde933ecfa96cb3e21df72e606))
+
+### Refactor
+
+* Changed the way embedding model is created in AbstractGraph class and removed handling of embedding model creation from RAGNode. Now AbstractGraph will call a dedicated method for embedding models instead of _create_llm. This makes it easy to use any LLM with any supported embedding model. ([819cbcd](https://github.com/VinciGit00/Scrapegraph-ai/commit/819cbcd3be1a8cb195de0b44c6b6d4d824e2a42a))
+
 ## [0.7.0-beta.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.6.2...v0.7.0-beta.1) (2024-05-03)
 
 

examples/groq/smart_scraper_groq_openai.py

@@ -25,7 +25,7 @@
     },
     "embeddings": {
         "api_key": openai_key,
-        "model": "gpt-3.5-turbo",
+        "model": "openai",
     },
     "headless": False
 }

examples/openai/smart_scraper_openai.py

@@ -21,7 +21,7 @@
         "api_key": openai_key,
         "model": "gpt-3.5-turbo",
     },
-    "verbose":False,
+    "verbose": True,
 }
 
 # ************************************************

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "scrapegraphai"
 
-version = "0.7.0b1"
+version = "0.8.0b1"
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [

scrapegraphai/graphs/__init__.py

@@ -10,3 +10,4 @@
 from .xml_scraper_graph import XMLScraperGraph
 from .json_scraper_graph import JSONScraperGraph
 from .csv_scraper_graph import CSVScraperGraph
+from .pdf_scraper_graph import PDFScraperGraph

scrapegraphai/graphs/abstract_graph.py

@@ -5,8 +5,12 @@
 from abc import ABC, abstractmethod
 from typing import Optional
 
-from ..models import OpenAI, Gemini, Ollama, AzureOpenAI, HuggingFace, Groq, Bedrock
+from langchain_aws.embeddings.bedrock import BedrockEmbeddings
+from langchain_community.embeddings import HuggingFaceHubEmbeddings, OllamaEmbeddings
+from langchain_openai import AzureOpenAIEmbeddings, OpenAIEmbeddings
+
 from ..helpers import models_tokens
+from ..models import AzureOpenAI, Bedrock, Gemini, Groq, HuggingFace, Ollama, OpenAI
 
 
 class AbstractGraph(ABC):
@@ -43,7 +47,8 @@ def __init__(self, prompt: str, config: dict, source: Optional[str] = None):
         self.source = source
         self.config = config
         self.llm_model = self._create_llm(config["llm"], chat=True)
-        self.embedder_model = self.llm_model if "embeddings" not in config else self._create_llm(
+        self.embedder_model = self._create_default_embedder(    
+            ) if "embeddings" not in config else self._create_embedder(
             config["embeddings"])
 
         # Set common configuration parameters
@@ -172,6 +177,85 @@ def _create_llm(self, llm_config: dict, chat=False) -> object:
         else:
             raise ValueError(
                 "Model provided by the configuration not supported")
+    
+    def _create_default_embedder(self) -> object:
+        """
+        Create an embedding model instance based on the chosen llm model.
+
+        Returns:
+            object: An instance of the embedding model client.
+
+        Raises:
+            ValueError: If the model is not supported.
+        """
+
+        if isinstance(self.llm_model, OpenAI):
+            return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key)
+        elif isinstance(self.llm_model, AzureOpenAIEmbeddings):
+            return self.llm_model
+        elif isinstance(self.llm_model, AzureOpenAI):
+            return AzureOpenAIEmbeddings()
+        elif isinstance(self.llm_model, Ollama):
+            # unwrap the kwargs from the model whihc is a dict
+            params = self.llm_model._lc_kwargs
+            # remove streaming and temperature
+            params.pop("streaming", None)
+            params.pop("temperature", None)
+
+            return OllamaEmbeddings(**params)
+        elif isinstance(self.llm_model, HuggingFace):
+            return HuggingFaceHubEmbeddings(model=self.llm_model.model)
+        elif isinstance(self.llm_model, Bedrock):
+            return BedrockEmbeddings(client=None, model_id=self.llm_model.model_id)
+        else:
+            raise ValueError("Embedding Model missing or not supported")
+        
+    def _create_embedder(self, embedder_config: dict) -> object:
+        """
+        Create an embedding model instance based on the configuration provided.
+
+        Args:
+            embedder_config (dict): Configuration parameters for the embedding model.
+
+        Returns:
+            object: An instance of the embedding model client.
+
+        Raises:
+            KeyError: If the model is not supported.
+        """
+        
+        # Instantiate the embedding model based on the model name
+        if "openai" in embedder_config["model"]:
+            return OpenAIEmbeddings(api_key=embedder_config["api_key"])
+
+        elif "azure" in embedder_config["model"]:
+            return AzureOpenAIEmbeddings()
+
+        elif "ollama" in embedder_config["model"]:
+            embedder_config["model"] = embedder_config["model"].split("/")[-1]
+            try:
+                models_tokens["ollama"][embedder_config["model"]]
+            except KeyError:
+                raise KeyError("Model not supported")
+            return OllamaEmbeddings(**embedder_config)
+        
+        elif "hugging_face" in embedder_config["model"]:
+            try:
+                models_tokens["hugging_face"][embedder_config["model"]]
+            except KeyError:
+                raise KeyError("Model not supported")
+            return HuggingFaceHubEmbeddings(model=embedder_config["model"])
+        
+        elif "bedrock" in embedder_config["model"]:
+            embedder_config["model"] = embedder_config["model"].split("/")[-1]
+            try:
+                models_tokens["bedrock"][embedder_config["model"]]
+            except KeyError:
+                raise KeyError("Model not supported")
+            return BedrockEmbeddings(client=None, model_id=embedder_config["model"])
+        else:
+            raise ValueError(
+                "Model provided by the configuration not supported") 
 
     def get_state(self, key=None) -> dict:
         """""

scrapegraphai/graphs/pdf_scraper_graph.py

@@ -0,0 +1,118 @@
+"""
+PDFScraperGraph Module
+"""
+
+from .base_graph import BaseGraph
+from ..nodes import (
+    FetchNode,
+    ParseNode,
+    RAGNode,
+    GenerateAnswerNode
+)
+from .abstract_graph import AbstractGraph
+
+
+class PDFScraperGraph(AbstractGraph):
+    """
+    PDFScraperGraph is a scraping pipeline that extracts information from pdf files using a natural
+    language model to interpret and answer prompts.
+
+    Attributes:
+        prompt (str): The prompt for the graph.
+        source (str): The source of the graph.
+        config (dict): Configuration parameters for the graph.
+        llm_model: An instance of a language model client, configured for generating answers.
+        embedder_model: An instance of an embedding model client, 
+        configured for generating embeddings.
+        verbose (bool): A flag indicating whether to show print statements during execution.
+        headless (bool): A flag indicating whether to run the graph in headless mode.
+        model_token (int): The token limit for the language model.
+
+    Args:
+        prompt (str): The prompt for the graph.
+        source (str): The source of the graph.
+        config (dict): Configuration parameters for the graph.
+
+    Example:
+        >>> pdf_scraper = PDFScraperGraph(
+        ...     "List me all the attractions in Chioggia.",
+        ...     "data/chioggia.pdf",
+        ...     {"llm": {"model": "gpt-3.5-turbo"}}
+        ... )
+        >>> result = pdf_scraper.run()
+    """
+
+    def __init__(self, prompt: str, source: str, config: dict):
+        super().__init__(prompt, config, source)
+
+        self.input_key = "pdf" if source.endswith("pdf") else "pdf_dir"
+
+    def _create_graph(self) -> BaseGraph:
+        """
+        Creates the graph of nodes representing the workflow for web scraping.
+
+        Returns:
+            BaseGraph: A graph instance representing the web scraping workflow.
+        """
+
+        fetch_node = FetchNode(
+            input="pdf_dir",
+            output=["doc"],
+            node_config={
+                "headless": self.headless,
+                "verbose": self.verbose
+            }
+        )
+        parse_node = ParseNode(
+            input="doc",
+            output=["parsed_doc"],
+            node_config={
+                "chunk_size": self.model_token,
+                "verbose": self.verbose
+            }
+        )
+        rag_node = RAGNode(
+            input="user_prompt & (parsed_doc | doc)",
+            output=["relevant_chunks"],
+            node_config={
+                "llm": self.llm_model,
+                "embedder_model": self.embedder_model,
+                "verbose": self.verbose
+            }
+        )
+        generate_answer_node = GenerateAnswerNode(
+            input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+            output=["answer"],
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
+        )
+
+        return BaseGraph(
+            nodes=[
+                fetch_node,
+                parse_node,
+                rag_node,
+                generate_answer_node,
+            ],
+            edges=[
+                (fetch_node, parse_node),
+                (parse_node, rag_node),
+                (rag_node, generate_answer_node)
+            ],
+            entry_point=fetch_node
+        )
+
+    def run(self) -> str:
+        """
+        Executes the web scraping process and returns the answer to the prompt.
+
+        Returns:
+            str: The answer to the prompt.
+        """
+
+        inputs = {"user_prompt": self.prompt, self.input_key: self.source}
+        self.final_state, self.execution_info = self.graph.execute(inputs)
+
+        return self.final_state.get("answer", "No answer found.")

scrapegraphai/nodes/__init__.py

@@ -16,3 +16,4 @@
 from .search_link_node import SearchLinkNode
 from .robots_node import RobotsNode
 from .generate_answer_csv_node import GenerateAnswerCSVNode
+from .generate_answer_pdf_node import GenerateAnswerPDFNode