feat: refactoring of abstract graph

VinciGit00 · VinciGit00 · commit fff89f431f60 · 2024-06-04T19:41:11.000+02:00
diff --git a/examples/deepseek/.env.example b/examples/deepseek/.env.example
@@ -1 +1 @@
-OPENAI_APIKEY="your openai api key"
+DEEPSEEK_APIKEY="your api key"
diff --git a/examples/deepseek/csv_scraper_deepseek.py b/examples/deepseek/csv_scraper_deepseek.py
@@ -30,6 +30,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/csv_scraper_graph_multi_deepseek.py b/examples/deepseek/csv_scraper_graph_multi_deepseek.py
@@ -30,6 +30,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/custom_graph_deepseek.py b/examples/deepseek/custom_graph_deepseek.py
@@ -20,6 +20,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/json_scraper_deepseek.py b/examples/deepseek/json_scraper_deepseek.py
@@ -29,6 +29,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/json_scraper_multi_deepseek.py b/examples/deepseek/json_scraper_multi_deepseek.py
@@ -15,6 +15,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/pdf_scraper_graph_deepseek.py b/examples/deepseek/pdf_scraper_graph_deepseek.py
@@ -20,6 +20,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/pdf_scraper_multi_deepseek.py b/examples/deepseek/pdf_scraper_multi_deepseek.py
@@ -15,6 +15,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/scrape_plain_text_deepseek.py b/examples/deepseek/scrape_plain_text_deepseek.py
@@ -31,6 +31,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/script_generator_deepseek.py b/examples/deepseek/script_generator_deepseek.py
@@ -20,6 +20,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }
diff --git a/examples/deepseek/search_graph_deepseek.py b/examples/deepseek/search_graph_deepseek.py
@@ -19,6 +19,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "max_results": 2,
     "verbose": True,
diff --git a/examples/deepseek/smart_scraper_deepseek.py b/examples/deepseek/smart_scraper_deepseek.py
@@ -21,6 +21,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/smart_scraper_schema_deepseek.py b/examples/deepseek/smart_scraper_schema_deepseek.py
@@ -41,6 +41,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/xml_scraper_deepseek.py b/examples/deepseek/xml_scraper_deepseek.py
@@ -31,6 +31,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/deepseek/xml_scraper_graph_multi_deepseek.py b/examples/deepseek/xml_scraper_graph_multi_deepseek.py
@@ -30,6 +30,11 @@
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/groq/csv_scraper_graph_multi_groq.py b/examples/groq/csv_scraper_graph_multi_groq.py
@@ -30,6 +30,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }
diff --git a/examples/groq/csv_scraper_groq.py b/examples/groq/csv_scraper_groq.py
@@ -31,6 +31,11 @@
         "api_key": groq_key,
         "temperature": 0
     },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
+    },
 }
 # ************************************************
 # Create the CSVScraperGraph instance and run it
diff --git a/examples/groq/custom_graph_groq.py b/examples/groq/custom_graph_groq.py
@@ -19,6 +19,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False
diff --git a/examples/groq/json_scraper_groq.py b/examples/groq/json_scraper_groq.py
@@ -30,6 +30,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False
diff --git a/examples/groq/json_scraper_multi_groq.py b/examples/groq/json_scraper_multi_groq.py
@@ -15,6 +15,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }
diff --git a/examples/groq/pdf_scraper_graph_groq.py b/examples/groq/pdf_scraper_graph_groq.py
@@ -18,6 +18,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }
diff --git a/examples/groq/pdf_scraper_multi_groq.py b/examples/groq/pdf_scraper_multi_groq.py
@@ -14,6 +14,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }
diff --git a/examples/groq/scrape_plain_text_groq.py b/examples/groq/scrape_plain_text_groq.py
@@ -32,6 +32,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False
diff --git a/examples/groq/script_generator_groq.py b/examples/groq/script_generator_groq.py
@@ -19,6 +19,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }
diff --git a/examples/groq/search_graph_groq.py b/examples/groq/search_graph_groq.py
@@ -21,6 +21,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }
diff --git a/examples/groq/smart_scraper_groq.py b/examples/groq/smart_scraper_groq.py
@@ -22,10 +22,10 @@
         "api_key": groq_key,
         "temperature": 0
     },
-    "embeddings": {
+     "embeddings": {
         "model": "ollama/nomic-embed-text",
         "temperature": 0,
-        "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }
diff --git a/examples/groq/smart_scraper_multi_groq.py b/examples/groq/smart_scraper_multi_groq.py
@@ -19,6 +19,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False
diff --git a/examples/groq/smart_scraper_schema_groq.py b/examples/groq/smart_scraper_schema_groq.py
@@ -41,6 +41,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }
diff --git a/examples/groq/xml_scraper_graph_multi_groq.py b/examples/groq/xml_scraper_graph_multi_groq.py
@@ -30,6 +30,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }
diff --git a/examples/groq/xml_scraper_groq.py b/examples/groq/xml_scraper_groq.py
@@ -30,6 +30,11 @@
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False
diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-OPENAI_APIKEY="your openai api key"`
	`1`	`+DEEPSEEK_APIKEY="your api key"`