From 66a29bc5cc2e8fc43409ca6fa6de928ccad78802 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Mon, 5 Aug 2024 20:54:53 +0200 Subject: [PATCH] refactoring of the openai examples Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com> --- examples/openai/csv_scraper_graph_multi_openai.py | 2 +- examples/openai/csv_scraper_openai.py | 2 +- examples/openai/custom_graph_openai.py | 2 +- examples/openai/deep_scraper_openai.py | 2 +- examples/openai/json_scraper_multi_openai.py | 2 +- examples/openai/json_scraper_openai.py | 2 +- examples/openai/md_scraper_openai.py | 2 +- examples/openai/pdf_scraper_multi_openai.py | 9 ++++----- examples/openai/pdf_scraper_openai.py | 2 +- examples/openai/scrape_plain_text_openai.py | 2 +- examples/openai/script_generator_openai.py | 2 +- examples/openai/search_graph_schema_openai.py | 7 +++---- examples/openai/search_link_graph_openai.py | 11 +++++++++-- examples/openai/smart_scraper_schema_openai.py | 2 +- examples/openai/speech_graph_openai.py | 2 +- examples/openai/xml_scraper_graph_multi_openai.py | 2 +- examples/openai/xml_scraper_openai.py | 2 +- scrapegraphai/graphs/search_graph.py | 1 - 18 files changed, 30 insertions(+), 26 deletions(-) diff --git a/examples/openai/csv_scraper_graph_multi_openai.py b/examples/openai/csv_scraper_graph_multi_openai.py index 771ad679..7b91c896 100644 --- a/examples/openai/csv_scraper_graph_multi_openai.py +++ b/examples/openai/csv_scraper_graph_multi_openai.py @@ -27,7 +27,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, } diff --git a/examples/openai/csv_scraper_openai.py b/examples/openai/csv_scraper_openai.py index 211f14f9..744fc7a4 100644 --- a/examples/openai/csv_scraper_openai.py +++ b/examples/openai/csv_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, } diff --git a/examples/openai/custom_graph_openai.py b/examples/openai/custom_graph_openai.py index 905473e0..6687e0ef 100644 --- a/examples/openai/custom_graph_openai.py +++ b/examples/openai/custom_graph_openai.py @@ -19,7 +19,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, } diff --git a/examples/openai/deep_scraper_openai.py b/examples/openai/deep_scraper_openai.py index 4860a31f..5b7202d4 100644 --- a/examples/openai/deep_scraper_openai.py +++ b/examples/openai/deep_scraper_openai.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4", + "model": "gpt-4o", }, "verbose": True, "max_depth": 1 diff --git a/examples/openai/json_scraper_multi_openai.py b/examples/openai/json_scraper_multi_openai.py index 021cd6e1..b27e5050 100644 --- a/examples/openai/json_scraper_multi_openai.py +++ b/examples/openai/json_scraper_multi_openai.py @@ -13,7 +13,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", } } diff --git a/examples/openai/json_scraper_openai.py b/examples/openai/json_scraper_openai.py index 25fc85af..eb5d1e7e 100644 --- a/examples/openai/json_scraper_openai.py +++ b/examples/openai/json_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, } diff --git a/examples/openai/md_scraper_openai.py b/examples/openai/md_scraper_openai.py index 7a163137..c3e2a3df 100644 --- a/examples/openai/md_scraper_openai.py +++ b/examples/openai/md_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, } diff --git a/examples/openai/pdf_scraper_multi_openai.py b/examples/openai/pdf_scraper_multi_openai.py index 9e699e58..49a9c7fa 100644 --- a/examples/openai/pdf_scraper_multi_openai.py +++ b/examples/openai/pdf_scraper_multi_openai.py @@ -3,11 +3,10 @@ Module for showing how PDFScraper multi works """ import os import json -from dotenv import load_dotenv -from scrapegraphai.graphs import PdfScraperMultiGraph - -from pydantic import BaseModel, Field from typing import List +from dotenv import load_dotenv +from pydantic import BaseModel, Field +from scrapegraphai.graphs import PdfScraperMultiGraph load_dotenv() @@ -20,7 +19,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, "verbose": True, } diff --git a/examples/openai/pdf_scraper_openai.py b/examples/openai/pdf_scraper_openai.py index e07a7ab5..2b0e19f3 100644 --- a/examples/openai/pdf_scraper_openai.py +++ b/examples/openai/pdf_scraper_openai.py @@ -14,7 +14,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, "verbose": True, } diff --git a/examples/openai/scrape_plain_text_openai.py b/examples/openai/scrape_plain_text_openai.py index ffe0054a..7f390cff 100644 --- a/examples/openai/scrape_plain_text_openai.py +++ b/examples/openai/scrape_plain_text_openai.py @@ -30,7 +30,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, } diff --git a/examples/openai/script_generator_openai.py b/examples/openai/script_generator_openai.py index 14c00ab4..046a25ec 100644 --- a/examples/openai/script_generator_openai.py +++ b/examples/openai/script_generator_openai.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, "library": "beautifulsoup" } diff --git a/examples/openai/search_graph_schema_openai.py b/examples/openai/search_graph_schema_openai.py index e5131461..ecbcc644 100644 --- a/examples/openai/search_graph_schema_openai.py +++ b/examples/openai/search_graph_schema_openai.py @@ -3,14 +3,13 @@ Example of Search Graph """ import os +from typing import List from dotenv import load_dotenv -load_dotenv() - +from pydantic import BaseModel, Field from scrapegraphai.graphs import SearchGraph from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info -from pydantic import BaseModel, Field -from typing import List +load_dotenv() # ************************************************ # Define the output schema for the graph diff --git a/examples/openai/search_link_graph_openai.py b/examples/openai/search_link_graph_openai.py index 10d10d4c..818f9434 100644 --- a/examples/openai/search_link_graph_openai.py +++ b/examples/openai/search_link_graph_openai.py @@ -1,16 +1,23 @@ """ Basic example of scraping pipeline using SmartScraper """ + +import os +from dotenv import load_dotenv from scrapegraphai.graphs import SearchLinkGraph from scrapegraphai.utils import prettify_exec_info + +load_dotenv() # ************************************************ # Define the configuration for the graph # ************************************************ +openai_key = os.getenv("OPENAI_APIKEY") + graph_config = { "llm": { - "api_key": "s", - "model": "gpt-3.5-turbo", + "api_key": openai_key, + "model": "gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/smart_scraper_schema_openai.py b/examples/openai/smart_scraper_schema_openai.py index 076f1327..d9e1bd1c 100644 --- a/examples/openai/smart_scraper_schema_openai.py +++ b/examples/openai/smart_scraper_schema_openai.py @@ -30,7 +30,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key":openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/speech_graph_openai.py b/examples/openai/speech_graph_openai.py index 15cc2cfb..603ce51c 100644 --- a/examples/openai/speech_graph_openai.py +++ b/examples/openai/speech_graph_openai.py @@ -25,7 +25,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", "temperature": 0.7, }, "tts_model": { diff --git a/examples/openai/xml_scraper_graph_multi_openai.py b/examples/openai/xml_scraper_graph_multi_openai.py index 46633bba..ef46b877 100644 --- a/examples/openai/xml_scraper_graph_multi_openai.py +++ b/examples/openai/xml_scraper_graph_multi_openai.py @@ -29,7 +29,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key":openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/xml_scraper_openai.py b/examples/openai/xml_scraper_openai.py index 5be5716e..b2b5075e 100644 --- a/examples/openai/xml_scraper_openai.py +++ b/examples/openai/xml_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "gpt-4o", }, "verbose":False, } diff --git a/scrapegraphai/graphs/search_graph.py b/scrapegraphai/graphs/search_graph.py index 787e537a..0c0f1104 100644 --- a/scrapegraphai/graphs/search_graph.py +++ b/scrapegraphai/graphs/search_graph.py @@ -53,7 +53,6 @@ class SearchGraph(AbstractGraph): self.copy_config = copy(config) else: self.copy_config = deepcopy(config) - self.copy_schema = deepcopy(schema) self.considered_urls = [] # New attribute to store URLs