Merge pull request #387 from shubihu/main

fix: Incorrect API Key Error with OpenAI Proxy

Author: VinciGit00

CHANGELOG.md

@@ -1,8 +1,12 @@
-## [1.7.0-beta.9](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.7.0-beta.8...v1.7.0-beta.9) (2024-06-16)
+## [1.6.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.6.0...v1.6.1) (2024-06-15)
+=======
 
 
 ### Bug Fixes
 
+* removed duplicate from ollama dictionary ([dcd216e](https://github.com/VinciGit00/Scrapegraph-ai/commit/dcd216e3457bdbbbc7b8dc27783866b748e322fa))
+
+## [1.6.0](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.5.7...v1.6.0) (2024-06-09)
 * fix robot node ([2419003](https://github.com/VinciGit00/Scrapegraph-ai/commit/24190039996b9cbe04952f6734d996e0cdb15296))
 
 ## [1.7.0-beta.8](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.7.0-beta.7...v1.7.0-beta.8) (2024-06-16)

pyproject.toml

@@ -2,7 +2,7 @@
 name = "scrapegraphai"
 
 
-version = "1.7.0b9"
+version = "1.6.1"
 
 
 

scrapegraphai/graphs/abstract_graph.py

@@ -275,7 +275,10 @@ def _create_default_embedder(self, llm_config=None) -> object:
                 google_api_key=llm_config["api_key"], model="models/embedding-001"
             )
         if isinstance(self.llm_model, OpenAI):
-            return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key)
+            return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key, base_url=self.llm_model.openai_api_base)
+        elif isinstance(self.llm_model, DeepSeek):
+            return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key)   
+
         elif isinstance(self.llm_model, AzureOpenAIEmbeddings):
             return self.llm_model
         elif isinstance(self.llm_model, AzureOpenAI):
@@ -393,4 +396,4 @@ def run(self) -> str:
         """
         Abstract method to execute the graph and return the result.
         """
-        pass
+        pass

scrapegraphai/helpers/models_tokens.py

@@ -35,50 +35,33 @@
         "gemini-1.5-pro-latest":128000,
         "models/embedding-001": 2048
     },
-
-    "ollama": {
-        "command-r": 12800,
-        "command-r-plus": 12800,
-        "codellama": 16000,
-        "dbrx": 32768,
-        "dbrx:instruct": 32768,
-        "deepseek-coder:33b": 16000,
-        "dolphin-mixtral": 32000,
-        "llama2": 4096,
-        "llama3": 8192,
-        "llama3:70b-instruct": 8192,
-        "llava": 4096,
-        "llava:34b": 4096,
-        "llava_next": 4096,
-        "mistral": 8192,
-        "falcon": 2048,
-        "codellama": 16000,
-        "dolphin-mixtral": 32000,
-        "mistral-openorca": 32000,
-        "stablelm-zephyr": 8192,
-        "command-r-plus": 12800,
-        "command-r": 12800,
-        "mistral:7b-instruct": 32768,
-        "mistral-openorca": 32000,
-        "mixtral:8x22b-instruct": 65536,
-        "nous-hermes2:34b": 4096,
-        "orca-mini": 2048,
-        "phi3:3.8b": 12800,
-        "phi3:14b": 12800,
-        "qwen:0.5b": 32000,
-        "qwen:1.8b": 32000,
-        "qwen:4b": 32000,
-        "qwen:14b": 32000,
-        "qwen:32b": 32000,
-        "qwen:72b": 32000,
-        "qwen:110b": 32000,
-        "stablelm-zephyr": 8192,
-        "wizardlm2:8x22b": 65536,
-        # embedding models
-        "nomic-embed-text": 8192,
-        "snowflake-arctic-embed:335m": 8192,
-        "snowflake-arctic-embed:l": 8192,
-        "mxbai-embed-large": 512,
+    "ollama": { "command-r": 12800, 
+               "codellama": 16000, 
+               "dbrx": 32768, 
+               "deepseek-coder:33b": 16000, 
+               "falcon": 2048, 
+               "llama2": 4096, 
+               "llama3": 8192, 
+               "scrapegraph": 8192, 
+               "llava": 4096, 
+               "mixtral:8x22b-instruct": 65536, 
+               "mistral-openorca": 32000, 
+               "nomic-embed-text": 8192, 
+               "nous-hermes2:34b": 4096, 
+               "orca-mini": 2048, 
+               "phi3:3.8b": 12800, 
+               "qwen:0.5b": 32000, 
+               "qwen:1.8b": 32000, 
+               "qwen:4b": 32000, 
+               "qwen:14b": 32000, 
+               "qwen:32b": 32000, 
+               "qwen:72b": 32000, 
+               "qwen:110b": 32000, 
+               "stablelm-zephyr": 8192, 
+               "wizardlm2:8x22b": 65536, 
+               # embedding models 
+               "snowflake-arctic-embed": 8192, 
+               "mxbai-embed-large": 512 
     },
     "oneapi": {
         "qwen-turbo": 6000