Merge remote-tracking branch 'origin' into pre/beta

Author: aziz-ullah-khan

examples/together/code_generator_graph_togehter.py

@@ -0,0 +1,60 @@
+""" 
+Basic example of scraping pipeline using Code Generator with schema
+"""
+
+import os, json
+from typing import List
+from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+from scrapegraphai.graphs import CodeGeneratorGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the output schema for the graph
+# ************************************************
+
+class Project(BaseModel):
+    title: str = Field(description="The title of the project")
+    description: str = Field(description="The description of the project")
+
+class Projects(BaseModel):
+    projects: List[Project]
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+together_key = os.getenv("TOGETHER_KEY")
+
+graph_config = {
+    "llm": {
+        "model": "togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
+        "api_key": together_key,
+    },
+    "verbose": True,
+    "headless": False,
+    "reduction": 2,
+    "max_iterations": {
+        "overall": 10,
+        "syntax": 3,
+        "execution": 3,
+        "validation": 3,
+        "semantic": 3
+    },
+    "output_file_name": "extracted_data.py"
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+code_generator_graph = CodeGeneratorGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io/projects/",
+    schema=Projects,
+    config=graph_config
+)
+
+result = code_generator_graph.run()
+print(result)

pyproject.toml

@@ -29,11 +29,10 @@ dependencies = [
     "playwright>=1.43.0",
     "undetected-playwright>=0.3.0",
     "langchain-ollama>=0.1.3",
-    "simpleeval>=1.0.0",
+    "qdrant-client>=1.11.3",
+    "fastembed>=0.3.6"
     "semchunk>=2.2.0",
     "transformers>=4.44.2",
-    "qdrant-client>=1.11.3",
-    "fastembed>=0.3.6",
     "googlesearch-python>=1.2.5"
 ]