fix: update to pydantic documentation

examples/anthropic/code_generator_graph_anthropic.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/azure/code_generator_graph_azure.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/bedrock/code_generator_graph_bedrock.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/deepseek/code_generator_graph_deepseek.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/ernie/code_generator_graph_ernie.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/fireworks/code_generator_graph_fireworks.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/google_genai/code_generator_graph_gemini.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/google_vertexai/code_generator_graph_vertex.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/groq/code_generator_graph_groq.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/huggingfacehub/code_generator_graph_huggingfacehub.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 from langchain_community.llms import HuggingFaceEndpoint
 from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings

examples/local_models/code_generator_graph_ollama.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/mistral/code_generator_graph_mistral.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/moonshot/code_generator_graph_moonshot.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from langchain_community.chat_models.moonshot import MoonshotChat
 from scrapegraphai.graphs import CodeGeneratorGraph
 

examples/nemotron/code_generator_graph_nemotron.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/oneapi/code_generator_graph_oneapi.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

examples/openai/code_generator_graph_openai.py

@@ -5,7 +5,7 @@ Basic example of scraping pipeline using Code Generator with schema
 import os, json
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import CodeGeneratorGraph
 
 load_dotenv()

extracted_data.py

@@ -0,0 +1,28 @@
+def extract_data(html: str) -> dict:
+    from bs4 import BeautifulSoup
+
+    # Parse the HTML content using BeautifulSoup
+    soup = BeautifulSoup(html, 'html.parser')
+
+    # Initialize an empty list to hold project data
+    projects = []
+
+    # Find all project entries in the HTML
+    project_entries = soup.find_all('div', class_='grid-item')
+
+    # Iterate over each project entry to extract title and description
+    for entry in project_entries:
+        # Extract the title from the card-title class
+        title = entry.find('h4', class_='card-title').get_text(strip=True)
+        
+        # Extract the description from the card-text class
+        description = entry.find('p', class_='card-text').get_text(strip=True)
+        
+        # Append the extracted data as a dictionary to the projects list
+        projects.append({
+            'title': title,
+            'description': description
+        })
+
+    # Return the structured data as a dictionary matching the desired JSON schema
+    return {'projects': projects}

scrapegraphai/nodes/fetch_node.py

@@ -319,4 +319,3 @@ class FetchNode(BaseNode):
         state["original_html"] = document
         state.update({self.output[0]: compressed_document,})
         return state
-    

scrapegraphai/utils/schema_trasform.py

@@ -12,7 +12,7 @@ def transform_schema(pydantic_schema):
     Returns:
         dict: The transformed JSON schema.
     """
-    
+
     def process_properties(properties):
         result = {}
         for key, value in properties.items():
@@ -20,7 +20,7 @@ def transform_schema(pydantic_schema):
                 if value['type'] == 'array':
                     if '$ref' in value['items']:
                         ref_key = value['items']['$ref'].split('/')[-1]
-                        result[key] = [process_properties(pydantic_schema['definitions'][ref_key]['properties'])]
+                        result[key] = [process_properties(pydantic_schema['$defs'][ref_key]['properties'])]
                     else:
                         result[key] = [value['items']['type']]
                 else:
@@ -30,7 +30,7 @@ def transform_schema(pydantic_schema):
                     }
             elif '$ref' in value:
                 ref_key = value['$ref'].split('/')[-1]
-                result[key] = process_properties(pydantic_schema['definitions'][ref_key]['properties'])
+                result[key] = process_properties(pydantic_schema['$defs'][ref_key]['properties'])
         return result
 
-    return process_properties(pydantic_schema['properties'])
+    return process_properties(pydantic_schema['properties'])