ADD benchmarks m2 max llama3

lurenss · lurenss · commit 12dcc5065a32 · 2024-04-23T12:28:51.000+02:00
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -21,7 +21,7 @@ The following sections will guide you through the installation process and the u
    :caption: Getting Started
 
    getting_started/installation
-   getting_started/examples 
+   getting_started/examples
    modules/modules
 
 Indices and tables
diff --git a/examples/benchmarks/GenerateScraper/.env.example b/examples/benchmarks/GenerateScraper/.env.example
@@ -1 +1 @@
-OPENAI_APIKEY="your openai api key"
+OPENAI_APIKEY="your openai key here"
diff --git a/examples/benchmarks/GenerateScraper/Readme.md b/examples/benchmarks/GenerateScraper/Readme.md
@@ -9,12 +9,14 @@ The time is measured in seconds
 
 The model runned for this benchmark is Mistral on Ollama with nomic-embed-text
 
+In particular, is tested with ScriptCreatorGraph
+
 | Hardware               | Model                                   | Example 1 | Example 2 |
 | ---------------------- | --------------------------------------- | --------- | --------- |
 | Macbook 14' m1 pro     | Mistral on Ollama with nomic-embed-text | 30.54s    | 35.76s    |
-| Macbook m2 max         | Mistral on Ollama with nomic-embed-text |           |           |
-| Macbook 14' m1 pro<br> | Llama3 on Ollama with nomic-embed-text  | 27.82s    | 29.986s   |
-| Macbook m2 max<br>     | Llama3 on Ollama with nomic-embed-text  |           |           |
+| Macbook m2 max         | Mistral on Ollama with nomic-embed-text | 18,46s    | 19.59     |
+| Macbook 14' m1 pro<br> | Llama3 on Ollama with nomic-embed-text  | 27.82s    | 29.98s    |
+| Macbook m2 max<br>     | Llama3 on Ollama with nomic-embed-text  | 20.83s    | 12.29s    |
 
 
 **Note**: the examples on Docker are not runned on other devices than the Macbook because the performance are to slow (10 times slower than Ollama). 
@@ -23,17 +25,17 @@ The model runned for this benchmark is Mistral on Ollama with nomic-embed-text
 **URL**: https://perinim.github.io/projects
 **Task**: List me all the projects with their description.
 
-| Name                | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
-| ------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- |
-| gpt-3.5-turbo       | 24.215268                | 1892         | 1802          | 90                | 1                   | 0.002883       |
-| gpt-4-turbo-preview | 6.614                    | 1936         | 1802          | 134               | 1                   | 0.02204        |
+| Name                | Execution time | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
+| ------------------- | ---------------| ------------ | ------------- | ----------------- | ------------------- | -------------- |
+| gpt-3.5-turbo       | 4.50s          | 1897         | 1802          | 95                | 1                   | 0.002893       |
+| gpt-4-turbo         | 7.88s          | 1920         | 1802          | 118               | 1                   | 0.02156        |
 
 ### Example 2: Wired
 **URL**: https://www.wired.com
 **Task**: List me all the articles with their description.
 
 | Name                | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
 | ------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- |
-| gpt-3.5-turbo       |                          |              |               |                   |                     |                |
-| gpt-4-turbo-preview |                          |              |               |                   |                     |                |
+| gpt-3.5-turbo       |   Error (text too long)  |      -       |      -        |         -         |           -         |        -       |
+| gpt-4-turbo         |   Error (TPM limit reach)|      -       |      -        |         -         |           -         |        -       |
 
diff --git a/examples/benchmarks/GenerateScraper/benchmark_openai_gpt35.py b/examples/benchmarks/GenerateScraper/benchmark_openai_gpt35.py
@@ -19,7 +19,7 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("GPT35_KEY")
+openai_key = os.getenv("OPENAI_APIKEY")
 
 graph_config = {
     "llm": {
diff --git a/examples/benchmarks/GenerateScraper/benchmark_openai_gpt4.py b/examples/benchmarks/GenerateScraper/benchmark_openai_gpt4.py
@@ -19,12 +19,12 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("GPT4_KEY")
+openai_key = os.getenv("OPENAI_APIKEY")
 
 graph_config = {
     "llm": {
         "api_key": openai_key,
-        "model": "gpt-4-turbo-preview",
+        "model": "gpt-4-turbo-2024-04-09",
     },
     "library": "beautifoulsoup"
 }
diff --git a/examples/benchmarks/SmartScraper/.env.example b/examples/benchmarks/SmartScraper/.env.example
@@ -1 +1 @@
-OPENAI_APIKEY="your openai api key"
+OPENAI_APIKEY="your openai key here"
diff --git a/examples/benchmarks/SmartScraper/Readme.md b/examples/benchmarks/SmartScraper/Readme.md
@@ -5,12 +5,14 @@ The two websites benchmark are:
 
 Both are strored locally as txt file in .txt format  because in this way we do not have to think about the internet connection
 
+In particular, is tested with SmartScraper
+
 | Hardware           | Moodel                                  | Example 1 | Example 2 |
 | ------------------ | --------------------------------------- | --------- | --------- |
 | Macbook 14' m1 pro | Mistral on Ollama with nomic-embed-text | 11.60s    | 26.61s    |
 | Macbook m2 max     | Mistral on Ollama with nomic-embed-text | 8.05s     | 12.17s    |
 | Macbook 14' m1 pro | Llama3 on Ollama with nomic-embed-text  | 29.871    | 35.32     |
-| Macbook m2 max     | Llama3 on Ollama with nomic-embed-text  |           |           |
+| Macbook m2 max     | Llama3 on Ollama with nomic-embed-text  | 18.36s    | 78.32s    |
 
 
 **Note**: the examples on Docker are not runned on other devices than the Macbook because the performance are to slow (10 times slower than Ollama). Indeed the results are the following:
@@ -23,17 +25,17 @@ Both are strored locally as txt file in .txt format  because in this way we do n
 **URL**: https://perinim.github.io/projects
 **Task**: List me all the projects with their description.
 
-| Name                | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
-| ------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- |
-| gpt-3.5-turbo       | 25.22                    | 445          | 272           | 173               | 1                   | 0.000754       |
-| gpt-4-turbo-preview | 9.53                     | 449          | 272           | 177               | 1                   | 0.00803        |
+| Name                | Execution time | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
+| ------------------- | ---------------| ------------ | ------------- | ----------------- | ------------------- | -------------- |
+| gpt-3.5-turbo       | 5.58s          | 445          | 272           | 173               | 1                   | 0.000754       |
+| gpt-4-turbo         | 9.76s          | 445          | 272           | 173               | 1                   | 0.00791        |
 
 ### Example 2: Wired
 **URL**: https://www.wired.com
 **Task**: List me all the articles with their description.
 
 | Name                | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
 | ------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- |
-| gpt-3.5-turbo       | 25.89                    | 445          | 272           | 173               | 1                   | 0.000754       |
-| gpt-4-turbo-preview | 64.70                    | 3573         | 2199          | 1374              | 1                   | 0.06321        |
+| gpt-3.5-turbo       | 6.50                     | 2442         | 2199          | 243               | 1                   | 0.003784       |
+| gpt-4-turbo         | 76.07                    | 3521         | 2199          | 1322              | 1                   | 0.06165        |
 
diff --git a/examples/benchmarks/SmartScraper/benchmark_openai_gpt35.py b/examples/benchmarks/SmartScraper/benchmark_openai_gpt35.py
@@ -19,7 +19,7 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("GPT35_KEY")
+openai_key = os.getenv("OPENAI_APIKEY")
 
 graph_config = {
     "llm": {
diff --git a/examples/benchmarks/SmartScraper/benchmark_openai_gpt4.py b/examples/benchmarks/SmartScraper/benchmark_openai_gpt4.py
@@ -20,12 +20,12 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("GPT4_KEY")
+openai_key = os.getenv("OPENAI_APIKEY")
 
 graph_config = {
     "llm": {
         "api_key": openai_key,
-        "model": "gpt-4-turbo-preview",
+        "model": "gpt-4-turbo",
     },
 }
 
diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py
@@ -9,6 +9,8 @@
         "gpt-3.5-turbo-instruct": 4096,
         "gpt-4-0125-preview": 128000,
         "gpt-4-turbo-preview": 128000,
+        "gpt-4-turbo": 128000,
+        "gpt-4-turbo-2024-04-09": 128000,
         "gpt-4-1106-preview": 128000,
         "gpt-4-vision-preview": 128000,
         "gpt-4": 8192,

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-OPENAI_APIKEY="your openai api key"`
	`1`	`+OPENAI_APIKEY="your openai key here"`