fix: update to pydantic documentation

VinciGit00 · VinciGit00 · commit 76ce257efb9d · 2024-09-25T15:12:29.000+02:00
diff --git a/examples/anthropic/code_generator_graph_anthropic.py b/examples/anthropic/code_generator_graph_anthropic.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/azure/code_generator_graph_azure.py b/examples/azure/code_generator_graph_azure.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/bedrock/code_generator_graph_bedrock.py b/examples/bedrock/code_generator_graph_bedrock.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/deepseek/code_generator_graph_deepseek.py b/examples/deepseek/code_generator_graph_deepseek.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/ernie/code_generator_graph_ernie.py b/examples/ernie/code_generator_graph_ernie.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/fireworks/code_generator_graph_fireworks.py b/examples/fireworks/code_generator_graph_fireworks.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/google_genai/code_generator_graph_gemini.py b/examples/google_genai/code_generator_graph_gemini.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/google_vertexai/code_generator_graph_vertex.py b/examples/google_vertexai/code_generator_graph_vertex.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/groq/code_generator_graph_groq.py b/examples/groq/code_generator_graph_groq.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/huggingfacehub/code_generator_graph_huggingfacehub.py b/examples/huggingfacehub/code_generator_graph_huggingfacehub.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 from langchain_community.llms import HuggingFaceEndpoint
 from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
diff --git a/examples/local_models/code_generator_graph_ollama.py b/examples/local_models/code_generator_graph_ollama.py
@@ -5,7 +5,7 @@
 import json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/mistral/code_generator_graph_mistral.py b/examples/mistral/code_generator_graph_mistral.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/moonshot/code_generator_graph_moonshot.py b/examples/moonshot/code_generator_graph_moonshot.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from langchain_community.chat_models.moonshot import MoonshotChat
 from scrapegraphai.graphs import CodeGeneratorGraph
 
diff --git a/examples/nemotron/code_generator_graph_nemotron.py b/examples/nemotron/code_generator_graph_nemotron.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/oneapi/code_generator_graph_oneapi.py b/examples/oneapi/code_generator_graph_oneapi.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/examples/openai/code_generator_graph_openai.py b/examples/openai/code_generator_graph_openai.py
@@ -5,7 +5,7 @@
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()
diff --git a/extracted_data.py b/extracted_data.py
@@ -0,0 +1,28 @@
+def extract_data(html: str) -> dict:
+    from bs4 import BeautifulSoup
+
+    # Parse the HTML content using BeautifulSoup
+    soup = BeautifulSoup(html, 'html.parser')
+
+    # Initialize an empty list to hold project data
+    projects = []
+
+    # Find all project entries in the HTML
+    project_entries = soup.find_all('div', class_='grid-item')
+
+    # Iterate over each project entry to extract title and description
+    for entry in project_entries:
+        # Extract the title from the card-title class
+        title = entry.find('h4', class_='card-title').get_text(strip=True)
+        
+        # Extract the description from the card-text class
+        description = entry.find('p', class_='card-text').get_text(strip=True)
+        
+        # Append the extracted data as a dictionary to the projects list
+        projects.append({
+            'title': title,
+            'description': description
+        })
+
+    # Return the structured data as a dictionary matching the desired JSON schema
+    return {'projects': projects}
diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py
@@ -319,4 +319,3 @@ def handle_web_source(self, state, source):
         state["original_html"] = document
         state.update({self.output[0]: compressed_document,})
         return state
-    
diff --git a/scrapegraphai/utils/schema_trasform.py b/scrapegraphai/utils/schema_trasform.py
@@ -12,15 +12,15 @@ def transform_schema(pydantic_schema):
     Returns:
         dict: The transformed JSON schema.
     """
-    
+
     def process_properties(properties):
         result = {}
         for key, value in properties.items():
             if 'type' in value:
                 if value['type'] == 'array':
                     if '$ref' in value['items']:
                         ref_key = value['items']['$ref'].split('/')[-1]
-                        result[key] = [process_properties(pydantic_schema['definitions'][ref_key]['properties'])]
+                        result[key] = [process_properties(pydantic_schema['$defs'][ref_key]['properties'])]
                     else:
                         result[key] = [value['items']['type']]
                 else:
@@ -30,7 +30,7 @@ def process_properties(properties):
                     }
             elif '$ref' in value:
                 ref_key = value['$ref'].split('/')[-1]
-                result[key] = process_properties(pydantic_schema['definitions'][ref_key]['properties'])
+                result[key] = process_properties(pydantic_schema['$defs'][ref_key]['properties'])
         return result
 
-    return process_properties(pydantic_schema['properties'])
+    return process_properties(pydantic_schema['properties'])
