From ebdb74967dfd35f32d62b2539c7efc804c029bfb Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sat, 17 Aug 2024 15:00:44 +0200 Subject: [PATCH] add new model for gpt --- examples/openai/smart_scraper_schema_openai.py | 4 ++-- pyproject.toml | 4 ++-- requirements-dev.lock | 7 +++---- requirements.lock | 10 +++++----- scrapegraphai/nodes/generate_answer_node.py | 6 ++++-- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/examples/openai/smart_scraper_schema_openai.py b/examples/openai/smart_scraper_schema_openai.py index 4decac7e..5a13e9ae 100644 --- a/examples/openai/smart_scraper_schema_openai.py +++ b/examples/openai/smart_scraper_schema_openai.py @@ -5,7 +5,7 @@ Basic example of scraping pipeline using SmartScraper with schema import os, json from typing import List from dotenv import load_dotenv -from pydantic import BaseModel, Field +from langchain_core.pydantic_v1 import BaseModel, Field from scrapegraphai.graphs import SmartScraperGraph load_dotenv() @@ -30,7 +30,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key":openai_key, - "model": "chatgpt-4o-latest", + "model": "gpt-4o-mini", }, "verbose": True, "headless": False, diff --git a/pyproject.toml b/pyproject.toml index f6843503..cb3e6be2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,12 @@ authors = [ ] dependencies = [ - "langchain>=0.2.10", + "langchain==0.2.14", "langchain-fireworks>=0.1.3", "langchain_community>=0.2.9", "langchain-google-genai>=1.0.7", "langchain-google-vertexai>=1.0.7", - "langchain-openai>=0.1.17", + "langchain-openai==0.1.22", "langchain-groq>=0.1.3", "langchain-aws>=0.1.3", "langchain-anthropic>=0.1.11", diff --git a/requirements-dev.lock b/requirements-dev.lock index ffcd2d40..44cc64de 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -179,7 +179,6 @@ graphviz==0.20.3 # via scrapegraphai greenlet==3.0.3 # via playwright - # via sqlalchemy groq==0.9.0 # via langchain-groq grpc-google-iam-v1==0.13.1 @@ -255,7 +254,7 @@ jsonschema-specifications==2023.12.1 # via jsonschema kiwisolver==1.4.5 # via matplotlib -langchain==0.2.12 +langchain==0.2.14 # via langchain-community # via scrapegraphai langchain-anthropic==0.1.22 @@ -264,7 +263,7 @@ langchain-aws==0.1.16 # via scrapegraphai langchain-community==0.2.11 # via scrapegraphai -langchain-core==0.2.29 +langchain-core==0.2.33 # via langchain # via langchain-anthropic # via langchain-aws @@ -292,7 +291,7 @@ langchain-mistralai==0.1.12 # via scrapegraphai langchain-nvidia-ai-endpoints==0.2.1 # via scrapegraphai -langchain-openai==0.1.21 +langchain-openai==0.1.22 # via scrapegraphai langchain-text-splitters==0.2.2 # via langchain diff --git a/requirements.lock b/requirements.lock index f449a7b7..1812ab21 100644 --- a/requirements.lock +++ b/requirements.lock @@ -133,7 +133,6 @@ graphviz==0.20.3 # via scrapegraphai greenlet==3.0.3 # via playwright - # via sqlalchemy groq==0.9.0 # via langchain-groq grpc-google-iam-v1==0.13.1 @@ -178,6 +177,7 @@ jinja2==3.1.4 # via torch jiter==0.5.0 # via anthropic + # via openai jmespath==1.0.1 # via boto3 # via botocore @@ -187,7 +187,7 @@ jsonpatch==1.33 # via langchain-core jsonpointer==3.0.0 # via jsonpatch -langchain==0.2.11 +langchain==0.2.14 # via langchain-community # via scrapegraphai langchain-anthropic==0.1.20 @@ -196,7 +196,7 @@ langchain-aws==0.1.12 # via scrapegraphai langchain-community==0.2.10 # via scrapegraphai -langchain-core==0.2.28 +langchain-core==0.2.33 # via langchain # via langchain-anthropic # via langchain-aws @@ -224,7 +224,7 @@ langchain-mistralai==0.1.12 # via scrapegraphai langchain-nvidia-ai-endpoints==0.1.7 # via scrapegraphai -langchain-openai==0.1.17 +langchain-openai==0.1.22 # via scrapegraphai langchain-text-splitters==0.2.2 # via langchain @@ -264,7 +264,7 @@ numpy==1.26.4 # via sentence-transformers # via shapely # via transformers -openai==1.37.0 +openai==1.41.0 # via langchain-fireworks # via langchain-openai orjson==3.10.6 diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py index 8aa01e0a..f307425e 100644 --- a/scrapegraphai/nodes/generate_answer_node.py +++ b/scrapegraphai/nodes/generate_answer_node.py @@ -88,8 +88,10 @@ class GenerateAnswerNode(BaseNode): # Initialize the output parser if self.node_config.get("schema", None) is not None: output_parser = JsonOutputParser(pydantic_object=self.node_config["schema"]) - if isinstance(self.llm_model, ChatOpenAI): - self.llm_model = self.llm_model.with_structured_output(self.node_config["schema"], method="json_mode") + if isinstance(self.llm_model, ChatOpenAI) and (self.llm_model.model_name=="gpt-4o-mini" or self.llm_model.model_name=="gpt-4o-2024-08-06"): + self.llm_model = self.llm_model.with_structured_output( + schema = self.node_config["schema"], + method="json_schema") else: output_parser = JsonOutputParser()