removed unused imports and comments + removed dead code

Author: VinciGit00

scrapegraphai/models/deepseek.py

@@ -3,7 +3,6 @@
 """
 from langchain_openai import ChatOpenAI
 
-
 class DeepSeek(ChatOpenAI):
     """
     A wrapper for the ChatOpenAI class (DeepSeek uses an OpenAI-like API) that

scrapegraphai/models/oneapi.py

@@ -3,7 +3,6 @@
 """
 from langchain_openai import ChatOpenAI
 
-
 class OneApi(ChatOpenAI):
     """
     A wrapper for the OneApi class that provides default configuration

scrapegraphai/models/openai_itt.py

@@ -1,11 +1,9 @@
 """
 OpenAIImageToText Module
 """
-
 from langchain_openai import ChatOpenAI
 from langchain_core.messages import HumanMessage
 
-
 class OpenAIImageToText(ChatOpenAI):
     """
     A wrapper for the OpenAIImageToText class that provides default configuration
@@ -43,6 +41,5 @@ def run(self, image_url: str) -> str:
             ]
         )
 
-        # Use the invoke method from the superclass (ChatOpenAI)
         result = self.invoke([message]).content
         return result

scrapegraphai/models/openai_tts.py

@@ -1,10 +1,8 @@
 """
 OpenAITextToSpeech Module
 """
-
 from openai import OpenAI
 
-
 class OpenAITextToSpeech:
     """
     Implements a text-to-speech model using the OpenAI API.
@@ -20,7 +18,6 @@ class OpenAITextToSpeech:
 
     def __init__(self, tts_config: dict):
 
-        # convert model_name to model
         self.client = OpenAI(api_key=tts_config.get("api_key"), 
                              base_url=tts_config.get("base_url", None))
         self.model = tts_config.get("model", "tts-1")

scrapegraphai/utils/copy.py

@@ -12,7 +12,6 @@ class DeepCopyError(Exception):
 
     pass
 
-
 def is_boto3_client(obj):
     """ 
     Function for understanding if the script is using boto3 or not
@@ -30,7 +29,6 @@ def is_boto3_client(obj):
             return False
     return False
 
-
 def safe_deepcopy(obj: Any) -> Any:
     """
     Attempts to create a deep copy of the object using `copy.deepcopy`

scrapegraphai/utils/custom_callback.py

@@ -8,15 +8,12 @@
 import threading
 from typing import Any, Dict, List, Optional
 from contextvars import ContextVar
-
 from langchain_core.callbacks import BaseCallbackHandler
 from langchain_core.messages import AIMessage
 from langchain_core.outputs import ChatGeneration, LLMResult
 from langchain_core.tracers.context import register_configure_hook
-
 from .model_costs import MODEL_COST_PER_1K_TOKENS_INPUT, MODEL_COST_PER_1K_TOKENS_OUTPUT
 
-
 def get_token_cost_for_model(
     model_name: str, num_tokens: int, is_completion: bool = False
 ) -> float:
@@ -36,7 +33,6 @@ def get_token_cost_for_model(
         return 0.0
     if is_completion:
         return MODEL_COST_PER_1K_TOKENS_OUTPUT[model_name] * (num_tokens / 1000)
-        
     return MODEL_COST_PER_1K_TOKENS_INPUT[model_name] * (num_tokens / 1000)
 
 
@@ -154,4 +150,4 @@ def get_custom_callback(llm_model_name: str):
     cb = CustomCallbackHandler(llm_model_name)
     custom_callback.set(cb)
     yield cb
-    custom_callback.set(None)
+    custom_callback.set(None)

scrapegraphai/utils/llm_callback_manager.py

@@ -3,14 +3,16 @@
 """
 import threading
 from contextlib import contextmanager
-from .custom_callback import get_custom_callback
-
 from langchain_community.callbacks import get_openai_callback
 from langchain_community.callbacks.manager import get_bedrock_anthropic_callback
 from langchain_openai import ChatOpenAI, AzureChatOpenAI
 from langchain_aws import ChatBedrock
+from .custom_callback import get_custom_callback
 
 class CustomLLMCallbackManager:
+    """
+    custom LLLM calback class
+    """
     _lock = threading.Lock()
 
     @contextmanager
@@ -22,7 +24,8 @@ def exclusive_get_callback(self, llm_model, llm_model_name):
                         yield cb
                 finally:
                     CustomLLMCallbackManager._lock.release()
-            elif isinstance(llm_model, ChatBedrock) and llm_model_name is not None and "claude" in llm_model_name:
+            elif isinstance(llm_model, ChatBedrock) and \
+                  llm_model_name is not None and "claude" in llm_model_name:
                 try:
                     with get_bedrock_anthropic_callback() as cb:
                         yield cb
@@ -35,4 +38,4 @@ def exclusive_get_callback(self, llm_model, llm_model_name):
                 finally:
                     CustomLLMCallbackManager._lock.release()
         else:
-            yield None
+            yield None

scrapegraphai/utils/logging.py

@@ -1,11 +1,9 @@
 """
 A centralized logging system for any library.
-
 This module provides functions to manage logging for a library. It includes
 functions to get and set the verbosity level, add and remove handlers, and
 control propagation. It also includes a function to set formatting for all
 handlers bound to the root logger.
-
 Source code inspired by: https://gist.github.com/DiTo97/9a0377f24236b66134eb96da1ec1693f
 """
 

scrapegraphai/utils/model_costs.py

@@ -2,6 +2,10 @@
 This file contains the cost of models per 1k tokens for input and output.
 The file is on a best effort basis and may not be up to date. Any contributions are welcome.
 """
+
+"""
+Cost for 1k tokens in input
+"""
 MODEL_COST_PER_1K_TOKENS_INPUT = {
     ### MistralAI
     # General Purpose
@@ -53,8 +57,10 @@
     "amazon.titan-text-premier-v1:0": 0.0005,
 }
 
+"""
+Cost for 1k tokens in output
+"""
 MODEL_COST_PER_1K_TOKENS_OUTPUT = {
-    ### MistralAI
     # General Purpose
     "open-mistral-nemo": 0.00015,
     "open-mistral-nemo-2407": 0.00015,
@@ -102,4 +108,4 @@
     "amazon.titan-text-express-v1": 0.0006,
     "amazon.titan-text-lite-v1": 0.0002,
     "amazon.titan-text-premier-v1:0": 0.0015,
-}
+}

scrapegraphai/utils/output_parser.py

@@ -1,12 +1,13 @@
 """
 Functions to retrieve the correct output parser and format instructions for the LLM model.
 """
+from typing import Union, Dict, Any, Type, Callable
 from pydantic import BaseModel as BaseModelV2
 from pydantic.v1 import BaseModel as BaseModelV1
-from typing import Union, Dict, Any, Type, Callable
 from langchain_core.output_parsers import JsonOutputParser
 
-def get_structured_output_parser(schema: Union[Dict[str, Any], Type[BaseModelV1 | BaseModelV2], Type]) -> Callable:
+def get_structured_output_parser(schema: Union[Dict[str, Any],
+                                               Type[BaseModelV1 | BaseModelV2], Type]) -> Callable:
     """
     Get the correct output parser for the LLM model.
 
@@ -15,7 +16,7 @@ def get_structured_output_parser(schema: Union[Dict[str, Any], Type[BaseModelV1
     """
     if issubclass(schema, BaseModelV1):
         return _base_model_v1_output_parser
-    
+
     if issubclass(schema, BaseModelV2):
         return _base_model_v2_output_parser
 
@@ -29,12 +30,14 @@ def get_pydantic_output_parser(schema: Union[Dict[str, Any], Type[BaseModelV1 |
         JsonOutputParser: The output parser object.
     """
     if issubclass(schema, BaseModelV1):
-        raise ValueError("pydantic.v1 and langchain_core.pydantic_v1 are not supported with this LLM model. Please use pydantic v2 instead.")
-    
+        raise ValueError("""pydantic.v1 and langchain_core.pydantic_v1
+                         are not supported with this LLM model. Please use pydantic v2 instead.""")
+
     if issubclass(schema, BaseModelV2):
         return JsonOutputParser(pydantic_object=schema)
 
-    raise ValueError("The schema is not a pydantic subclass. With this LLM model you must use a pydantic schemas.")
+    raise ValueError("""The schema is not a pydantic subclass.
+                     With this LLM model you must use a pydantic schemas.""")
 
 def _base_model_v1_output_parser(x: BaseModelV1) -> dict:
     """
@@ -47,16 +50,15 @@ def _base_model_v1_output_parser(x: BaseModelV1) -> dict:
         dict: The parsed output.
     """
     work_dict = x.dict()
-    
-    # recursive dict parser
+
     def recursive_dict_parser(work_dict: dict) -> dict:
         dict_keys = work_dict.keys()
         for key in dict_keys:
             if isinstance(work_dict[key], BaseModelV1):
                 work_dict[key] = work_dict[key].dict()
                 recursive_dict_parser(work_dict[key])
         return work_dict
-    
+
     return recursive_dict_parser(work_dict)