diff --git a/examples/anthropic/csv_scraper_graph_multi_haiku.py b/examples/anthropic/csv_scraper_graph_multi_haiku.py index b833af01..d574da5c 100644 --- a/examples/anthropic/csv_scraper_graph_multi_haiku.py +++ b/examples/anthropic/csv_scraper_graph_multi_haiku.py @@ -26,8 +26,8 @@ text = pd.read_csv(file_path) graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000}, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/csv_scraper_haiku.py b/examples/anthropic/csv_scraper_haiku.py index 2e0ebe81..745926a3 100644 --- a/examples/anthropic/csv_scraper_haiku.py +++ b/examples/anthropic/csv_scraper_haiku.py @@ -32,9 +32,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/custom_graph_haiku.py b/examples/anthropic/custom_graph_haiku.py index cea14361..d8b4dc19 100644 --- a/examples/anthropic/custom_graph_haiku.py +++ b/examples/anthropic/custom_graph_haiku.py @@ -5,10 +5,9 @@ Example of custom graph using existing nodes import os from dotenv import load_dotenv -from langchain_openai import OpenAIEmbeddings -from langchain_openai import ChatOpenAI +from langchain_anthropic import ChatAnthropic from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode +from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode load_dotenv() # ************************************************ @@ -19,16 +18,14 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + }, } # ************************************************ # Define the graph nodes # ************************************************ -llm_model = OpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) +llm_model = ChatAnthropic(graph_config["llm"]) # define the nodes for the graph robot_node = RobotsNode( @@ -57,15 +54,6 @@ parse_node = ParseNode( "verbose": True, } ) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) generate_answer_node = GenerateAnswerNode( input="user_prompt & (relevant_chunks | parsed_doc | doc)", output=["answer"], @@ -84,14 +72,12 @@ graph = BaseGraph( robot_node, fetch_node, parse_node, - rag_node, generate_answer_node, ], edges=[ (robot_node, fetch_node), (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) + (parse_node, generate_answer_node) ], entry_point=robot_node ) diff --git a/examples/anthropic/json_scraper_haiku.py b/examples/anthropic/json_scraper_haiku.py index 2610b658..9d5fc8db 100644 --- a/examples/anthropic/json_scraper_haiku.py +++ b/examples/anthropic/json_scraper_haiku.py @@ -26,9 +26,8 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/json_scraper_multi_haiku.py b/examples/anthropic/json_scraper_multi_haiku.py index 0327673b..d016439d 100644 --- a/examples/anthropic/json_scraper_multi_haiku.py +++ b/examples/anthropic/json_scraper_multi_haiku.py @@ -11,9 +11,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } FILE_NAME = "inputs/example.json" diff --git a/examples/anthropic/pdf_scraper_graph_haiku.py b/examples/anthropic/pdf_scraper_graph_haiku.py index 61be06b4..ee221ac6 100644 --- a/examples/anthropic/pdf_scraper_graph_haiku.py +++ b/examples/anthropic/pdf_scraper_graph_haiku.py @@ -14,9 +14,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } source = """ diff --git a/examples/anthropic/pdf_scraper_multi_haiku.py b/examples/anthropic/pdf_scraper_multi_haiku.py index 974dd2f8..2d117c35 100644 --- a/examples/anthropic/pdf_scraper_multi_haiku.py +++ b/examples/anthropic/pdf_scraper_multi_haiku.py @@ -11,9 +11,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # *************** diff --git a/examples/anthropic/scrape_plain_text_haiku.py b/examples/anthropic/scrape_plain_text_haiku.py index d3f36638..d3099026 100644 --- a/examples/anthropic/scrape_plain_text_haiku.py +++ b/examples/anthropic/scrape_plain_text_haiku.py @@ -28,9 +28,8 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/script_generator_haiku.py b/examples/anthropic/script_generator_haiku.py index 889ce0b5..bdd0c23b 100644 --- a/examples/anthropic/script_generator_haiku.py +++ b/examples/anthropic/script_generator_haiku.py @@ -16,9 +16,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/script_multi_generator_haiku.py b/examples/anthropic/script_multi_generator_haiku.py index f7c69010..bacf0bfc 100644 --- a/examples/anthropic/script_multi_generator_haiku.py +++ b/examples/anthropic/script_multi_generator_haiku.py @@ -16,10 +16,9 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, - "library": "beautifulsoup" + "model": "anthropic/claude-3-haiku-20240307", + }, + "library": "beautifulsoup" } # ************************************************ diff --git a/examples/anthropic/search_graph_haiku.py b/examples/anthropic/search_graph_haiku.py index f90d7598..97a5213d 100644 --- a/examples/anthropic/search_graph_haiku.py +++ b/examples/anthropic/search_graph_haiku.py @@ -15,9 +15,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/search_graph_schema_haiku.py b/examples/anthropic/search_graph_schema_haiku.py index c9e7a875..1158d58a 100644 --- a/examples/anthropic/search_graph_schema_haiku.py +++ b/examples/anthropic/search_graph_schema_haiku.py @@ -27,8 +27,8 @@ class Dishes(BaseModel): graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000}, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/search_link_graph_haiku.py b/examples/anthropic/search_link_graph_haiku.py index ccfbc1d2..70798cf3 100644 --- a/examples/anthropic/search_link_graph_haiku.py +++ b/examples/anthropic/search_link_graph_haiku.py @@ -14,23 +14,11 @@ from langchain_openai import AzureOpenAIEmbeddings load_dotenv() -llm_model_instance = AzureChatOpenAI( - openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], - azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"] -) - -embedder_model_instance = AzureOpenAIEmbeddings( - azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"], - openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"], -) - -# ************************************************ -# Create the SmartScraperGraph instance and run it -# ************************************************ - graph_config = { - "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} + "llm": { + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/smart_scraper_haiku.py b/examples/anthropic/smart_scraper_haiku.py index f0bb2a57..51ca1bf5 100644 --- a/examples/anthropic/smart_scraper_haiku.py +++ b/examples/anthropic/smart_scraper_haiku.py @@ -19,9 +19,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } smart_scraper_graph = SmartScraperGraph( diff --git a/examples/anthropic/smart_scraper_multi_haiku.py b/examples/anthropic/smart_scraper_multi_haiku.py index eb2001d4..f96de0ab 100644 --- a/examples/anthropic/smart_scraper_multi_haiku.py +++ b/examples/anthropic/smart_scraper_multi_haiku.py @@ -17,9 +17,8 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ******************************************************* diff --git a/examples/anthropic/smart_scraper_schema_haiku.py b/examples/anthropic/smart_scraper_schema_haiku.py index 83cedd2a..bd447a06 100644 --- a/examples/anthropic/smart_scraper_schema_haiku.py +++ b/examples/anthropic/smart_scraper_schema_haiku.py @@ -33,8 +33,8 @@ class Projects(BaseModel): graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000}, + "model": "anthropic/claude-3-haiku-20240307", + }, } smart_scraper_graph = SmartScraperGraph( diff --git a/examples/anthropic/xml_scraper_graph_multi_haiku.py b/examples/anthropic/xml_scraper_graph_multi_haiku.py index 6b79f709..6e9bc5f8 100644 --- a/examples/anthropic/xml_scraper_graph_multi_haiku.py +++ b/examples/anthropic/xml_scraper_graph_multi_haiku.py @@ -26,8 +26,8 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000}, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/anthropic/xml_scraper_haiku.py b/examples/anthropic/xml_scraper_haiku.py index dd64f571..2dc4b8d2 100644 --- a/examples/anthropic/xml_scraper_haiku.py +++ b/examples/anthropic/xml_scraper_haiku.py @@ -26,9 +26,8 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "anthropic/claude-3-haiku-20240307", + }, } # ************************************************ diff --git a/examples/deepseek/csv_scraper_deepseek.py b/examples/deepseek/csv_scraper_deepseek.py index b734b543..60b1c394 100644 --- a/examples/deepseek/csv_scraper_deepseek.py +++ b/examples/deepseek/csv_scraper_deepseek.py @@ -27,7 +27,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/csv_scraper_graph_multi_deepseek.py b/examples/deepseek/csv_scraper_graph_multi_deepseek.py index ea5e9154..0a08f83f 100644 --- a/examples/deepseek/csv_scraper_graph_multi_deepseek.py +++ b/examples/deepseek/csv_scraper_graph_multi_deepseek.py @@ -27,7 +27,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/json_scraper_deepseek.py b/examples/deepseek/json_scraper_deepseek.py index dfe6f489..02991c0d 100644 --- a/examples/deepseek/json_scraper_deepseek.py +++ b/examples/deepseek/json_scraper_deepseek.py @@ -26,7 +26,7 @@ with open(file_path, 'r', encoding="utf-8") as file: graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/json_scraper_multi_deepseek.py b/examples/deepseek/json_scraper_multi_deepseek.py index b957dde0..4f9ca32d 100644 --- a/examples/deepseek/json_scraper_multi_deepseek.py +++ b/examples/deepseek/json_scraper_multi_deepseek.py @@ -12,7 +12,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/pdf_scraper_graph_deepseek.py b/examples/deepseek/pdf_scraper_graph_deepseek.py index d66bbef5..c9c5e0b2 100644 --- a/examples/deepseek/pdf_scraper_graph_deepseek.py +++ b/examples/deepseek/pdf_scraper_graph_deepseek.py @@ -17,7 +17,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/pdf_scraper_multi_deepseek.py b/examples/deepseek/pdf_scraper_multi_deepseek.py index 211e4635..e43dd10a 100644 --- a/examples/deepseek/pdf_scraper_multi_deepseek.py +++ b/examples/deepseek/pdf_scraper_multi_deepseek.py @@ -12,7 +12,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/scrape_plain_text_deepseek.py b/examples/deepseek/scrape_plain_text_deepseek.py index d7a070d7..a7834a8f 100644 --- a/examples/deepseek/scrape_plain_text_deepseek.py +++ b/examples/deepseek/scrape_plain_text_deepseek.py @@ -28,7 +28,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/script_generator_deepseek.py b/examples/deepseek/script_generator_deepseek.py index fd5fd4dd..3de06f25 100644 --- a/examples/deepseek/script_generator_deepseek.py +++ b/examples/deepseek/script_generator_deepseek.py @@ -17,7 +17,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/script_multi_generator_deepseek.py b/examples/deepseek/script_multi_generator_deepseek.py index 2ebfd90a..cc577ecd 100644 --- a/examples/deepseek/script_multi_generator_deepseek.py +++ b/examples/deepseek/script_multi_generator_deepseek.py @@ -17,7 +17,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/search_graph_deepseek.py b/examples/deepseek/search_graph_deepseek.py index 176d6107..54d2e9fa 100644 --- a/examples/deepseek/search_graph_deepseek.py +++ b/examples/deepseek/search_graph_deepseek.py @@ -15,7 +15,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/search_graph_schema_deepseek.py b/examples/deepseek/search_graph_schema_deepseek.py index f5db278e..bcebe76d 100644 --- a/examples/deepseek/search_graph_schema_deepseek.py +++ b/examples/deepseek/search_graph_schema_deepseek.py @@ -31,7 +31,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/search_link_graph_deepseek.py b/examples/deepseek/search_link_graph_deepseek.py index 6a35f177..96f886a9 100644 --- a/examples/deepseek/search_link_graph_deepseek.py +++ b/examples/deepseek/search_link_graph_deepseek.py @@ -16,7 +16,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/smart_scraper_deepseek.py b/examples/deepseek/smart_scraper_deepseek.py index ed291b02..50314819 100644 --- a/examples/deepseek/smart_scraper_deepseek.py +++ b/examples/deepseek/smart_scraper_deepseek.py @@ -18,7 +18,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/smart_scraper_multi_deepseek.py b/examples/deepseek/smart_scraper_multi_deepseek.py index fafe7261..374cc6e2 100644 --- a/examples/deepseek/smart_scraper_multi_deepseek.py +++ b/examples/deepseek/smart_scraper_multi_deepseek.py @@ -16,7 +16,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/smart_scraper_schema_deepseek.py b/examples/deepseek/smart_scraper_schema_deepseek.py index 5cbbb702..6d164eb1 100644 --- a/examples/deepseek/smart_scraper_schema_deepseek.py +++ b/examples/deepseek/smart_scraper_schema_deepseek.py @@ -30,7 +30,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/xml_scraper_deepseek.py b/examples/deepseek/xml_scraper_deepseek.py index ba401b91..d69665f4 100644 --- a/examples/deepseek/xml_scraper_deepseek.py +++ b/examples/deepseek/xml_scraper_deepseek.py @@ -28,7 +28,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/deepseek/xml_scraper_graph_multi_deepseek.py b/examples/deepseek/xml_scraper_graph_multi_deepseek.py index 0f53a6b2..5098c9fd 100644 --- a/examples/deepseek/xml_scraper_graph_multi_deepseek.py +++ b/examples/deepseek/xml_scraper_graph_multi_deepseek.py @@ -27,7 +27,7 @@ deepseek_key = os.getenv("DEEPSEEK_APIKEY") graph_config = { "llm": { - "model": "deepseek-chat", + "model": "deepseek/deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', }, diff --git a/examples/ernie/csv_scraper_ernie.py b/examples/ernie/csv_scraper_ernie.py index 1594d17c..410e300e 100644 --- a/examples/ernie/csv_scraper_ernie.py +++ b/examples/ernie/csv_scraper_ernie.py @@ -25,16 +25,12 @@ text = pd.read_csv(file_path) graph_config = { "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434",} + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 } +} # ************************************************ # Create the CSVScraperGraph instance and run it diff --git a/examples/ernie/custom_graph_ernie.py b/examples/ernie/custom_graph_ernie.py index f750276a..5dad8bac 100644 --- a/examples/ernie/custom_graph_ernie.py +++ b/examples/ernie/custom_graph_ernie.py @@ -14,24 +14,20 @@ from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNod # Define the configuration for the graph # ************************************************ -graph_config = { - "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434",} +graph_config = { + "llm": { + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 + } } # ************************************************ # Define the graph nodes # ************************************************ -llm_model = OpenAI(graph_config["llm"]) +llm_model = ChatOpenAI(graph_config["llm"]) embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) # define the nodes for the graph diff --git a/examples/ernie/deep_scraper_ernie.py b/examples/ernie/deep_scraper_ernie.py index 059f7a74..b8c6501a 100644 --- a/examples/ernie/deep_scraper_ernie.py +++ b/examples/ernie/deep_scraper_ernie.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { - "model": "ernie-bot-turbo", + "model": "ernie/ernie-bot-turbo", "ernie_client_id": "", "ernie_client_secret": "", "temperature": 0.1 diff --git a/examples/ernie/json_scraper_ernie.py b/examples/ernie/json_scraper_ernie.py index ddd67050..e73ebc10 100644 --- a/examples/ernie/json_scraper_ernie.py +++ b/examples/ernie/json_scraper_ernie.py @@ -21,17 +21,13 @@ with open(file_path, 'r', encoding="utf-8") as file: # Define the configuration for the graph # ************************************************ -graph_config = { - "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434"} +graph_config = { + "llm": { + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 + } } # ************************************************ diff --git a/examples/ernie/pdf_scraper_graph_ernie.py b/examples/ernie/pdf_scraper_graph_ernie.py index 3de975a0..6016da7a 100644 --- a/examples/ernie/pdf_scraper_graph_ernie.py +++ b/examples/ernie/pdf_scraper_graph_ernie.py @@ -7,16 +7,12 @@ from scrapegraphai.graphs import PDFScraperGraph graph_config = { "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434",} + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 } +} source = """ The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian diff --git a/examples/ernie/scrape_plain_text_ernie.py b/examples/ernie/scrape_plain_text_ernie.py index 27b4f08b..c6bb715a 100644 --- a/examples/ernie/scrape_plain_text_ernie.py +++ b/examples/ernie/scrape_plain_text_ernie.py @@ -25,17 +25,13 @@ with open(file_path, 'r', encoding="utf-8") as file: # Define the configuration for the graph # ************************************************ -graph_config = { - "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434",} +graph_config = { + "llm": { + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 + } } # ************************************************ diff --git a/examples/ernie/script_generator_ernie.py b/examples/ernie/script_generator_ernie.py index 14c00ab4..42e136ff 100644 --- a/examples/ernie/script_generator_ernie.py +++ b/examples/ernie/script_generator_ernie.py @@ -13,12 +13,12 @@ load_dotenv() # Define the configuration for the graph # ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { +graph_config = { "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 }, "library": "beautifulsoup" } diff --git a/examples/ernie/script_multi_generator_ernie.py b/examples/ernie/script_multi_generator_ernie.py index 73e9f5ab..285d491a 100644 --- a/examples/ernie/script_multi_generator_ernie.py +++ b/examples/ernie/script_multi_generator_ernie.py @@ -9,17 +9,13 @@ from scrapegraphai.utils import prettify_exec_info # Define the configuration for the graph # ************************************************ -graph_config = { +graph_config = { "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434"}, + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 + }, "library": "beautifulsoup" } diff --git a/examples/ernie/search_graph_ernie.py b/examples/ernie/search_graph_ernie.py index c04d9f9b..0e811683 100644 --- a/examples/ernie/search_graph_ernie.py +++ b/examples/ernie/search_graph_ernie.py @@ -12,17 +12,13 @@ load_dotenv() # Define the configuration for the graph # ************************************************ -graph_config = { +graph_config = { "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434"}, + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 + }, "library": "beautifulsoup" } diff --git a/examples/ernie/search_link_graph_ernie.py b/examples/ernie/search_link_graph_ernie.py index 466b230c..f38b2772 100644 --- a/examples/ernie/search_link_graph_ernie.py +++ b/examples/ernie/search_link_graph_ernie.py @@ -8,17 +8,13 @@ from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_i # Define the configuration for the graph # ************************************************ -graph_config = { +graph_config = { "llm": { - "model": "ernie-bot-turbo", - "ernie_client_id": "", - "ernie_client_secret": "", - "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434"}, + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 + }, "library": "beautifulsoup" } diff --git a/examples/ernie/smart_scraper_ernie.py b/examples/ernie/smart_scraper_ernie.py index dcee0972..56084dad 100644 --- a/examples/ernie/smart_scraper_ernie.py +++ b/examples/ernie/smart_scraper_ernie.py @@ -14,15 +14,14 @@ load_dotenv() # Define the configuration for the graph # ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { +graph_config = { "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 }, - "verbose": False, - "headless": False, + "library": "beautifulsoup" } # ************************************************ diff --git a/examples/ernie/smart_scraper_multi_ernie.py b/examples/ernie/smart_scraper_multi_ernie.py index ddfc6239..6b62b685 100644 --- a/examples/ernie/smart_scraper_multi_ernie.py +++ b/examples/ernie/smart_scraper_multi_ernie.py @@ -12,12 +12,12 @@ load_dotenv() # Define the configuration for the graph # ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") - -graph_config = { +graph_config = { "llm": { - "api_key": openai_key, - "model": "gpt-4o", + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 }, "verbose": True, "headless": False, diff --git a/examples/ernie/smart_scraper_schema_ernie.py b/examples/ernie/smart_scraper_schema_ernie.py index 64a74937..b0fe3d7e 100644 --- a/examples/ernie/smart_scraper_schema_ernie.py +++ b/examples/ernie/smart_scraper_schema_ernie.py @@ -34,10 +34,12 @@ class Projects(BaseModel): openai_key = os.getenv("OPENAI_APIKEY") -graph_config = { +graph_config = { "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 }, "verbose": True, "headless": False, diff --git a/examples/ernie/speech_graph_ernie.py b/examples/ernie/speech_graph_ernie.py index 15cc2cfb..cece3149 100644 --- a/examples/ernie/speech_graph_ernie.py +++ b/examples/ernie/speech_graph_ernie.py @@ -24,9 +24,10 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", - "temperature": 0.7, + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 }, "tts_model": { "api_key": openai_key, diff --git a/examples/ernie/xml_scraper_ernie.py b/examples/ernie/xml_scraper_ernie.py index 5be5716e..a5bf03e0 100644 --- a/examples/ernie/xml_scraper_ernie.py +++ b/examples/ernie/xml_scraper_ernie.py @@ -27,8 +27,10 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { - "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "ernie/ernie-bot-turbo", + "ernie_client_id": "", + "ernie_client_secret": "", + "temperature": 0.1 }, "verbose":False, } diff --git a/examples/mistral/csv_scraper_graph_multi_mistral.py b/examples/mistral/csv_scraper_graph_multi_mistral.py index c3a25e2a..615e59e4 100644 --- a/examples/mistral/csv_scraper_graph_multi_mistral.py +++ b/examples/mistral/csv_scraper_graph_multi_mistral.py @@ -27,7 +27,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, } diff --git a/examples/mistral/csv_scraper_mistral.py b/examples/mistral/csv_scraper_mistral.py index 63ecfbca..195fb16a 100644 --- a/examples/mistral/csv_scraper_mistral.py +++ b/examples/mistral/csv_scraper_mistral.py @@ -28,7 +28,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, } diff --git a/examples/mistral/custom_graph_mistral.py b/examples/mistral/custom_graph_mistral.py index c839f7b6..f02ead0c 100644 --- a/examples/mistral/custom_graph_mistral.py +++ b/examples/mistral/custom_graph_mistral.py @@ -18,7 +18,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, } diff --git a/examples/mistral/deep_scraper_mistral.py b/examples/mistral/deep_scraper_mistral.py index 5cf576e7..bf0f6ba4 100644 --- a/examples/mistral/deep_scraper_mistral.py +++ b/examples/mistral/deep_scraper_mistral.py @@ -18,7 +18,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, "max_depth": 1 diff --git a/examples/mistral/json_scraper_mistral.py b/examples/mistral/json_scraper_mistral.py index 2a29c5a7..12f55127 100644 --- a/examples/mistral/json_scraper_mistral.py +++ b/examples/mistral/json_scraper_mistral.py @@ -28,7 +28,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, } diff --git a/examples/mistral/json_scraper_multi_mistral.py b/examples/mistral/json_scraper_multi_mistral.py index 07e65c95..1369eda7 100644 --- a/examples/mistral/json_scraper_multi_mistral.py +++ b/examples/mistral/json_scraper_multi_mistral.py @@ -13,7 +13,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", } } diff --git a/examples/mistral/md_scraper_mistral.py b/examples/mistral/md_scraper_mistral.py index 45995cb7..c4e3f2c7 100644 --- a/examples/mistral/md_scraper_mistral.py +++ b/examples/mistral/md_scraper_mistral.py @@ -28,7 +28,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, } diff --git a/examples/mistral/pdf_scraper_mistral.py b/examples/mistral/pdf_scraper_mistral.py index 9636f7f7..b006fdb8 100644 --- a/examples/mistral/pdf_scraper_mistral.py +++ b/examples/mistral/pdf_scraper_mistral.py @@ -14,7 +14,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, } diff --git a/examples/mistral/pdf_scraper_multi_mistral.py b/examples/mistral/pdf_scraper_multi_mistral.py index 97ad3222..e9f1613f 100644 --- a/examples/mistral/pdf_scraper_multi_mistral.py +++ b/examples/mistral/pdf_scraper_multi_mistral.py @@ -19,7 +19,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, } diff --git a/examples/mistral/scrape_plain_text_mistral.py b/examples/mistral/scrape_plain_text_mistral.py index 3bf199ad..f2b38172 100644 --- a/examples/mistral/scrape_plain_text_mistral.py +++ b/examples/mistral/scrape_plain_text_mistral.py @@ -30,7 +30,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, } diff --git a/examples/mistral/script_generator_mistral.py b/examples/mistral/script_generator_mistral.py index 464a522c..4fe45773 100644 --- a/examples/mistral/script_generator_mistral.py +++ b/examples/mistral/script_generator_mistral.py @@ -18,7 +18,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "library": "beautifulsoup" } diff --git a/examples/mistral/script_generator_schema_mistral.py b/examples/mistral/script_generator_schema_mistral.py index 8172f9a1..b9c77285 100644 --- a/examples/mistral/script_generator_schema_mistral.py +++ b/examples/mistral/script_generator_schema_mistral.py @@ -32,7 +32,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "library": "beautifulsoup", "verbose": True, diff --git a/examples/mistral/script_multi_generator_mistral.py b/examples/mistral/script_multi_generator_mistral.py index 4efa6914..f4d5d5b5 100644 --- a/examples/mistral/script_multi_generator_mistral.py +++ b/examples/mistral/script_multi_generator_mistral.py @@ -18,7 +18,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "library": "beautifulsoup", "verbose": True, diff --git a/examples/mistral/search_graph_mistral.py b/examples/mistral/search_graph_mistral.py index 68a480d3..f8573f5e 100644 --- a/examples/mistral/search_graph_mistral.py +++ b/examples/mistral/search_graph_mistral.py @@ -16,7 +16,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "max_results": 2, "verbose": True, diff --git a/examples/mistral/search_graph_schema_mistral.py b/examples/mistral/search_graph_schema_mistral.py index d4588289..7c71c0b1 100644 --- a/examples/mistral/search_graph_schema_mistral.py +++ b/examples/mistral/search_graph_schema_mistral.py @@ -31,7 +31,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "max_results": 2, "verbose": True, diff --git a/examples/mistral/search_link_graph_mistral.py b/examples/mistral/search_link_graph_mistral.py index 7191b27e..3216ff2c 100644 --- a/examples/mistral/search_link_graph_mistral.py +++ b/examples/mistral/search_link_graph_mistral.py @@ -17,7 +17,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, "headless": False, diff --git a/examples/mistral/smart_scraper_mistral.py b/examples/mistral/smart_scraper_mistral.py index 80d09e6d..7291a40a 100644 --- a/examples/mistral/smart_scraper_mistral.py +++ b/examples/mistral/smart_scraper_mistral.py @@ -16,7 +16,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("MISTRAL_API_KEY"), - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, "headless": False, diff --git a/examples/mistral/smart_scraper_multi_mistral.py b/examples/mistral/smart_scraper_multi_mistral.py index c86bb787..2654fbcb 100644 --- a/examples/mistral/smart_scraper_multi_mistral.py +++ b/examples/mistral/smart_scraper_multi_mistral.py @@ -18,7 +18,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, "headless": False, diff --git a/examples/mistral/smart_scraper_schema_mistral.py b/examples/mistral/smart_scraper_schema_mistral.py index 6d6b9ad3..3e1e505a 100644 --- a/examples/mistral/smart_scraper_schema_mistral.py +++ b/examples/mistral/smart_scraper_schema_mistral.py @@ -30,7 +30,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key":mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, "headless": False, diff --git a/examples/mistral/xml_scraper_graph_multi_mistral.py b/examples/mistral/xml_scraper_graph_multi_mistral.py index b9d46b0e..0ea9d30c 100644 --- a/examples/mistral/xml_scraper_graph_multi_mistral.py +++ b/examples/mistral/xml_scraper_graph_multi_mistral.py @@ -29,7 +29,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key":mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose": True, "headless": False, diff --git a/examples/mistral/xml_scraper_mistral.py b/examples/mistral/xml_scraper_mistral.py index c2675c6d..eb6036bf 100644 --- a/examples/mistral/xml_scraper_mistral.py +++ b/examples/mistral/xml_scraper_mistral.py @@ -28,7 +28,7 @@ mistral_key = os.getenv("MISTRAL_API_KEY") graph_config = { "llm": { "api_key": mistral_key, - "model": "mistral/open-mistral-nemo", + "model": "mistralai/open-mistral-nemo", }, "verbose":False, } diff --git a/examples/openai/csv_scraper_graph_multi_openai.py b/examples/openai/csv_scraper_graph_multi_openai.py index 7b91c896..5e876dcb 100644 --- a/examples/openai/csv_scraper_graph_multi_openai.py +++ b/examples/openai/csv_scraper_graph_multi_openai.py @@ -27,7 +27,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, } diff --git a/examples/openai/csv_scraper_openai.py b/examples/openai/csv_scraper_openai.py index 744fc7a4..f4410fcd 100644 --- a/examples/openai/csv_scraper_openai.py +++ b/examples/openai/csv_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, } diff --git a/examples/openai/custom_graph_openai.py b/examples/openai/custom_graph_openai.py index cc7e715d..b1471a21 100644 --- a/examples/openai/custom_graph_openai.py +++ b/examples/openai/custom_graph_openai.py @@ -27,7 +27,7 @@ graph_config = { # Define the graph nodes # ************************************************ -llm_model = OpenAI(graph_config["llm"]) +llm_model = ChatOpenAI(graph_config["llm"]) embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) # define the nodes for the graph diff --git a/examples/openai/deep_scraper_openai.py b/examples/openai/deep_scraper_openai.py index 5b7202d4..b20e164d 100644 --- a/examples/openai/deep_scraper_openai.py +++ b/examples/openai/deep_scraper_openai.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "max_depth": 1 diff --git a/examples/openai/json_scraper_multi_openai.py b/examples/openai/json_scraper_multi_openai.py index b27e5050..f7cb528a 100644 --- a/examples/openai/json_scraper_multi_openai.py +++ b/examples/openai/json_scraper_multi_openai.py @@ -13,7 +13,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", } } diff --git a/examples/openai/json_scraper_openai.py b/examples/openai/json_scraper_openai.py index eb5d1e7e..e20a5870 100644 --- a/examples/openai/json_scraper_openai.py +++ b/examples/openai/json_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, } diff --git a/examples/openai/md_scraper_openai.py b/examples/openai/md_scraper_openai.py index 2c264ab9..3456c89a 100644 --- a/examples/openai/md_scraper_openai.py +++ b/examples/openai/md_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, } diff --git a/examples/openai/omni_scraper_openai.py b/examples/openai/omni_scraper_openai.py index 1d1d86ba..3e6e62ee 100644 --- a/examples/openai/omni_scraper_openai.py +++ b/examples/openai/omni_scraper_openai.py @@ -19,7 +19,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "headless": True, diff --git a/examples/openai/omni_search_openai.py b/examples/openai/omni_search_openai.py index ed0f8f3c..fb967def 100644 --- a/examples/openai/omni_search_openai.py +++ b/examples/openai/omni_search_openai.py @@ -17,7 +17,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "max_results": 2, "max_images": 1, diff --git a/examples/openai/pdf_scraper_multi_openai.py b/examples/openai/pdf_scraper_multi_openai.py index 49a9c7fa..91e219e3 100644 --- a/examples/openai/pdf_scraper_multi_openai.py +++ b/examples/openai/pdf_scraper_multi_openai.py @@ -19,7 +19,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, } diff --git a/examples/openai/pdf_scraper_openai.py b/examples/openai/pdf_scraper_openai.py index 2b0e19f3..e076defe 100644 --- a/examples/openai/pdf_scraper_openai.py +++ b/examples/openai/pdf_scraper_openai.py @@ -14,7 +14,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, } diff --git a/examples/openai/scrape_plain_text_openai.py b/examples/openai/scrape_plain_text_openai.py index 7f390cff..eb8c76e5 100644 --- a/examples/openai/scrape_plain_text_openai.py +++ b/examples/openai/scrape_plain_text_openai.py @@ -30,7 +30,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, } diff --git a/examples/openai/screenshot_scraper.py b/examples/openai/screenshot_scraper.py index 826dcc50..c72c44d1 100644 --- a/examples/openai/screenshot_scraper.py +++ b/examples/openai/screenshot_scraper.py @@ -18,7 +18,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("OPENAI_API_KEY"), - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/script_generator_openai.py b/examples/openai/script_generator_openai.py index 046a25ec..e67ad52b 100644 --- a/examples/openai/script_generator_openai.py +++ b/examples/openai/script_generator_openai.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "library": "beautifulsoup" } diff --git a/examples/openai/script_generator_schema_openai.py b/examples/openai/script_generator_schema_openai.py index a728c8a1..5e542c53 100644 --- a/examples/openai/script_generator_schema_openai.py +++ b/examples/openai/script_generator_schema_openai.py @@ -32,7 +32,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "openai/gpt-3.5-turbo", }, "library": "beautifulsoup", "verbose": True, diff --git a/examples/openai/script_multi_generator_openai.py b/examples/openai/script_multi_generator_openai.py index d46d2294..3fdd029f 100644 --- a/examples/openai/script_multi_generator_openai.py +++ b/examples/openai/script_multi_generator_openai.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "library": "beautifulsoup", "verbose": True, diff --git a/examples/openai/search_graph_openai.py b/examples/openai/search_graph_openai.py index c12caa4f..8d869c19 100644 --- a/examples/openai/search_graph_openai.py +++ b/examples/openai/search_graph_openai.py @@ -16,7 +16,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "max_results": 2, "verbose": True, diff --git a/examples/openai/search_graph_schema_openai.py b/examples/openai/search_graph_schema_openai.py index ecbcc644..571f08b0 100644 --- a/examples/openai/search_graph_schema_openai.py +++ b/examples/openai/search_graph_schema_openai.py @@ -31,7 +31,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-3.5-turbo", + "model": "openai/gpt-3.5-turbo", }, "max_results": 2, "verbose": True, diff --git a/examples/openai/search_link_graph_openai.py b/examples/openai/search_link_graph_openai.py index 818f9434..a988731b 100644 --- a/examples/openai/search_link_graph_openai.py +++ b/examples/openai/search_link_graph_openai.py @@ -17,7 +17,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/smart_scraper_multi_openai.py b/examples/openai/smart_scraper_multi_openai.py index 504e00a8..8f5e648b 100644 --- a/examples/openai/smart_scraper_multi_openai.py +++ b/examples/openai/smart_scraper_multi_openai.py @@ -18,7 +18,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/smart_scraper_openai.py b/examples/openai/smart_scraper_openai.py index 119f67e5..2962f51b 100644 --- a/examples/openai/smart_scraper_openai.py +++ b/examples/openai/smart_scraper_openai.py @@ -18,7 +18,7 @@ load_dotenv() graph_config = { "llm": { "api_key": os.getenv("OPENAI_API_KEY"), - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/smart_scraper_schema_openai.py b/examples/openai/smart_scraper_schema_openai.py index 828a9b0a..0c1618d6 100644 --- a/examples/openai/smart_scraper_schema_openai.py +++ b/examples/openai/smart_scraper_schema_openai.py @@ -30,7 +30,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key":openai_key, - "model": "gpt-4o-mini", + "model": "openai/gpt-4o-mini", }, "verbose": True, "headless": False, diff --git a/examples/openai/speech_graph_openai.py b/examples/openai/speech_graph_openai.py index 603ce51c..7c368df7 100644 --- a/examples/openai/speech_graph_openai.py +++ b/examples/openai/speech_graph_openai.py @@ -25,7 +25,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", "temperature": 0.7, }, "tts_model": { diff --git a/examples/openai/xml_scraper_graph_multi_openai.py b/examples/openai/xml_scraper_graph_multi_openai.py index ef46b877..6610a49f 100644 --- a/examples/openai/xml_scraper_graph_multi_openai.py +++ b/examples/openai/xml_scraper_graph_multi_openai.py @@ -29,7 +29,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key":openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose": True, "headless": False, diff --git a/examples/openai/xml_scraper_openai.py b/examples/openai/xml_scraper_openai.py index b2b5075e..04b3ec9d 100644 --- a/examples/openai/xml_scraper_openai.py +++ b/examples/openai/xml_scraper_openai.py @@ -28,7 +28,7 @@ openai_key = os.getenv("OPENAI_APIKEY") graph_config = { "llm": { "api_key": openai_key, - "model": "gpt-4o", + "model": "openai/gpt-4o", }, "verbose":False, } diff --git a/examples/single_node/kg_node.py b/examples/single_node/kg_node.py index dd5a6d04..37d1d9a4 100644 --- a/examples/single_node/kg_node.py +++ b/examples/single_node/kg_node.py @@ -57,7 +57,7 @@ graph_config = { # Define the node # ************************************************ -llm_model = OpenAI(graph_config["llm"]) +llm_model = ChatOpenAI(graph_config["llm"]) robots_node = KnowledgeGraphNode( input="user_prompt & answer_dict", diff --git a/examples/single_node/robot_node.py b/examples/single_node/robot_node.py index c2bcbbd1..dcb70e3d 100644 --- a/examples/single_node/robot_node.py +++ b/examples/single_node/robot_node.py @@ -11,12 +11,12 @@ from scrapegraphai.nodes import RobotsNode graph_config = { "llm": { - "model": "ollama/llama3", + "model": "llama3", "temperature": 0, "streaming": True }, "embeddings": { - "model": "ollama/nomic-embed-text", + "model": "nomic-embed-text", "temperature": 0, # "base_url": "http://localhost:11434", # set ollama URL arbitrarily } diff --git a/examples/single_node/search_internet_node.py b/examples/single_node/search_internet_node.py index 8a8149fa..c998cdd1 100644 --- a/examples/single_node/search_internet_node.py +++ b/examples/single_node/search_internet_node.py @@ -2,7 +2,7 @@ Example of custom graph using existing nodes """ -from scrapegraphai.models import Ollama +from langchain_community.chat_models import ChatOllama from scrapegraphai.nodes import SearchInternetNode # ************************************************ @@ -24,7 +24,7 @@ graph_config = { # Define the node # ************************************************ -llm_model = Ollama(graph_config["llm"]) +llm_model = ChatOllama(graph_config["llm"]) search_node = SearchInternetNode( input="user_input", diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index d4c9f39e..556d6ad6 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -51,7 +51,7 @@ models_tokens = { "gemini-1.5-pro-latest": 128000, "models/embedding-001": 2048 }, - "google_vertexai": { + "google_vertexai": { "gemini-1.5-flash": 128000, "gemini-1.5-pro": 128000, "gemini-1.0-pro": 128000, @@ -137,6 +137,7 @@ models_tokens = { "claude-3-sonnet-20240229": 200000, "claude-3-haiku-20240307": 200000, "claude-3-5-sonnet-20240620": 200000, + "claude-3-haiku-20240307": 4000, }, "bedrock": { "anthropic.claude-3-5-sonnet-20240620": 200000,