From 0b4cfd6522dcad0eb418f0badd0f7824a1efd534 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 1 Aug 2024 14:38:50 +0200 Subject: [PATCH] fix: abstract_graph and removed unused embeddings Co-Authored-By: Federico Aguzzi <62149513+f-aguzzi@users.noreply.github.com> --- examples/bedrock/csv_scraper_bedrock.py | 3 - .../csv_scraper_graph_multi_bedrock.py | 3 - examples/bedrock/custom_graph_bedrock.py | 3 - examples/bedrock/json_scraper_bedrock.py | 3 - .../bedrock/json_scraper_multi_bedrock.py | 3 - examples/bedrock/pdf_scraper_graph_bedrock.py | 3 - .../pdf_scraper_graph_multi_bedrock.py | 3 - examples/bedrock/scrape_plain_text_bedrock.py | 3 - examples/bedrock/script_generator_bedrock.py | 5 +- .../bedrock/script_multi_generator_bedrock.py | 5 +- examples/bedrock/search_graph_bedrock.py | 3 - .../bedrock/search_graph_schema_bedrock.py | 3 - examples/bedrock/search_link_graph_bedrock.py | 3 - examples/bedrock/smart_scraper_bedrock.py | 3 - .../bedrock/smart_scraper_multi_bedrock.py | 3 - .../bedrock/smart_scraper_schema_bedrock.py | 3 - examples/bedrock/xml_scraper_bedrock.py | 3 - .../xml_scraper_graph_multi_bedrock.py | 3 - examples/deepseek/csv_scraper_deepseek.py | 5 -- .../csv_scraper_graph_multi_deepseek.py | 5 -- examples/deepseek/custom_graph_deepseek.py | 89 ------------------- examples/deepseek/json_scraper_deepseek.py | 5 -- .../deepseek/json_scraper_multi_deepseek.py | 5 -- .../deepseek/pdf_scraper_graph_deepseek.py | 5 -- .../deepseek/pdf_scraper_multi_deepseek.py | 5 -- .../deepseek/scrape_plain_text_deepseek.py | 5 -- .../deepseek/script_generator_deepseek.py | 5 -- .../script_multi_generator_deepseek.py | 5 -- examples/deepseek/search_graph_deepseek.py | 5 -- .../deepseek/search_graph_schema_deepseek.py | 5 -- .../deepseek/search_link_graph_deepseek.py | 5 -- examples/deepseek/smart_scraper_deepseek.py | 5 -- .../deepseek/smart_scraper_multi_deepseek.py | 5 -- .../deepseek/smart_scraper_schema_deepseek.py | 5 -- examples/deepseek/xml_scraper_deepseek.py | 5 -- .../xml_scraper_graph_multi_deepseek.py | 5 -- examples/fireworks/csv_scraper_fireworks.py | 6 -- .../csv_scraper_graph_multi_fireworks.py | 5 -- examples/fireworks/custom_graph_fireworks.py | 27 +----- examples/fireworks/deep_scraper_fireworks.py | 7 +- .../fireworks/json_scraper_fireworkspy.py | 5 -- .../fireworks/json_scraper_multi_fireworks.py | 5 -- examples/fireworks/pdf_scraper_fireworks.py | 5 -- .../fireworks/pdf_scraper_multi_fireworks.py | 5 -- .../fireworks/scrape_plain_text_fireworks.py | 5 -- .../fireworks/script_generator_fireworks.py | 5 -- .../script_generator_schema_fireworks.py | 5 -- .../script_multi_generator_fireworks.py | 5 -- examples/fireworks/search_graph_fireworks.py | 5 -- .../search_graph_schema_fireworks.py | 5 -- .../fireworks/search_link_graph_fireworks.py | 5 -- examples/fireworks/smart_scraper_fireworks.py | 5 -- .../smart_scraper_multi_fireworks.py | 6 +- .../smart_scraper_schema_fireworks.py | 5 -- examples/fireworks/xml_scraper_fireworks.py | 5 -- .../xml_scraper_graph_multi_fireworks.py | 5 -- examples/groq/csv_scraper_graph_multi_groq.py | 5 -- examples/groq/csv_scraper_groq.py | 5 -- examples/groq/custom_graph_groq.py | 22 +---- examples/groq/json_scraper_groq.py | 5 -- examples/groq/json_scraper_multi_groq.py | 5 -- examples/groq/pdf_scraper_graph_groq.py | 5 -- examples/groq/pdf_scraper_multi_groq.py | 5 -- examples/groq/scrape_plain_text_groq.py | 5 -- examples/groq/script_generator_groq.py | 5 -- examples/groq/script_multi_generator_groq.py | 5 -- examples/groq/search_graph_groq.py | 5 -- examples/groq/search_graph_schema_groq.py | 5 -- examples/groq/search_link_graph_groq.py | 5 -- examples/groq/smart_scraper_groq.py | 5 -- examples/groq/smart_scraper_multi_groq.py | 5 -- examples/groq/smart_scraper_schema_groq.py | 5 -- examples/groq/xml_scraper_graph_multi_groq.py | 5 -- examples/groq/xml_scraper_groq.py | 5 -- .../csv_scraper_graph_multi_huggingfacehub.py | 1 - .../csv_scraper_huggingfacehub.py | 1 - .../custom_graph_huggingfacehub.py | 1 - .../json_scraper_huggingfacehub.py | 1 - .../json_scraper_multi_huggingfacehub.py | 1 - .../pdf_scraper_graph_huggingfacehub.py | 1 - .../pdf_scraper_multi_huggingfacehub.py | 1 - .../scrape_plain_text_huggingfacehub.py | 1 - .../script_generator_huggingfacehub.py | 1 - .../script_multi_generator_huggingfacehub.py | 1 - .../search_graph_huggingfacehub.py | 1 - .../search_link_graph_huggingfacehub.py | 1 - .../smart_scraper_huggingfacehub.py | 1 - .../smart_scraper_multi_huggingfacehub.py | 1 - .../smart_scraper_schema_huggingfacehub.py | 1 - .../xml_scraper_graph_multi_huggingfacehub.py | 1 - .../xml_scraper_huggingfacehub.py | 1 - examples/local_models/custom_graph_ollama.py | 24 ++--- .../local_models/json_scraper_multi_ollama.py | 4 - examples/local_models/json_scraper_ollama.py | 5 -- .../local_models/pdf_scraper_multi_ollama.py | 4 - examples/local_models/pdf_scraper_ollama.py | 4 - .../local_models/scrape_plain_text_ollama.py | 5 -- .../script_multi_generator_ollama.py | 5 -- examples/local_models/search_graph_ollama.py | 5 -- .../search_graph_schema_ollama.py | 5 -- .../local_models/search_link_graph_ollama.py | 6 +- examples/local_models/smart_scraper_ollama.py | 6 +- .../smart_scraper_schema_ollama.py | 6 +- .../xml_scraper_graph_multi_ollama.py | 6 +- examples/local_models/xml_scraper_ollama.py | 5 -- scrapegraphai/graphs/abstract_graph.py | 3 - 106 files changed, 21 insertions(+), 554 deletions(-) delete mode 100644 examples/deepseek/custom_graph_deepseek.py diff --git a/examples/bedrock/csv_scraper_bedrock.py b/examples/bedrock/csv_scraper_bedrock.py index f015f77b..a69417c0 100644 --- a/examples/bedrock/csv_scraper_bedrock.py +++ b/examples/bedrock/csv_scraper_bedrock.py @@ -33,9 +33,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } # ************************************************ diff --git a/examples/bedrock/csv_scraper_graph_multi_bedrock.py b/examples/bedrock/csv_scraper_graph_multi_bedrock.py index c776c508..b9dd7f6f 100644 --- a/examples/bedrock/csv_scraper_graph_multi_bedrock.py +++ b/examples/bedrock/csv_scraper_graph_multi_bedrock.py @@ -28,9 +28,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/custom_graph_bedrock.py b/examples/bedrock/custom_graph_bedrock.py index 45358555..9002a598 100644 --- a/examples/bedrock/custom_graph_bedrock.py +++ b/examples/bedrock/custom_graph_bedrock.py @@ -28,9 +28,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/json_scraper_bedrock.py b/examples/bedrock/json_scraper_bedrock.py index 0729adfe..dc1bf769 100644 --- a/examples/bedrock/json_scraper_bedrock.py +++ b/examples/bedrock/json_scraper_bedrock.py @@ -32,9 +32,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/json_scraper_multi_bedrock.py b/examples/bedrock/json_scraper_multi_bedrock.py index 5dc666b8..5848ef17 100644 --- a/examples/bedrock/json_scraper_multi_bedrock.py +++ b/examples/bedrock/json_scraper_multi_bedrock.py @@ -10,9 +10,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } FILE_NAME = "inputs/example.json" diff --git a/examples/bedrock/pdf_scraper_graph_bedrock.py b/examples/bedrock/pdf_scraper_graph_bedrock.py index 6ee4b753..dcef848e 100644 --- a/examples/bedrock/pdf_scraper_graph_bedrock.py +++ b/examples/bedrock/pdf_scraper_graph_bedrock.py @@ -18,9 +18,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/pdf_scraper_graph_multi_bedrock.py b/examples/bedrock/pdf_scraper_graph_multi_bedrock.py index 7102c406..37e61c42 100644 --- a/examples/bedrock/pdf_scraper_graph_multi_bedrock.py +++ b/examples/bedrock/pdf_scraper_graph_multi_bedrock.py @@ -11,9 +11,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } # *************** diff --git a/examples/bedrock/scrape_plain_text_bedrock.py b/examples/bedrock/scrape_plain_text_bedrock.py index 01bec609..0214a1e3 100644 --- a/examples/bedrock/scrape_plain_text_bedrock.py +++ b/examples/bedrock/scrape_plain_text_bedrock.py @@ -33,9 +33,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/script_generator_bedrock.py b/examples/bedrock/script_generator_bedrock.py index 0d3f7d07..26863193 100644 --- a/examples/bedrock/script_generator_bedrock.py +++ b/examples/bedrock/script_generator_bedrock.py @@ -19,10 +19,7 @@ graph_config = { "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" - }, - "library": "beautifulsoup" + "library": "beautifulsoup" } # ************************************************ diff --git a/examples/bedrock/script_multi_generator_bedrock.py b/examples/bedrock/script_multi_generator_bedrock.py index 2f892546..ecef966d 100644 --- a/examples/bedrock/script_multi_generator_bedrock.py +++ b/examples/bedrock/script_multi_generator_bedrock.py @@ -15,10 +15,7 @@ graph_config = { "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" - }, - "library": "beautifulsoup" + "library": "beautifulsoup" } # ************************************************ diff --git a/examples/bedrock/search_graph_bedrock.py b/examples/bedrock/search_graph_bedrock.py index 9b32d3db..b27f6e5d 100644 --- a/examples/bedrock/search_graph_bedrock.py +++ b/examples/bedrock/search_graph_bedrock.py @@ -16,9 +16,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } # ************************************************ diff --git a/examples/bedrock/search_graph_schema_bedrock.py b/examples/bedrock/search_graph_schema_bedrock.py index 90539155..a49ba730 100644 --- a/examples/bedrock/search_graph_schema_bedrock.py +++ b/examples/bedrock/search_graph_schema_bedrock.py @@ -27,9 +27,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/search_link_graph_bedrock.py b/examples/bedrock/search_link_graph_bedrock.py index 116dea01..fc1e6233 100644 --- a/examples/bedrock/search_link_graph_bedrock.py +++ b/examples/bedrock/search_link_graph_bedrock.py @@ -15,9 +15,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/smart_scraper_bedrock.py b/examples/bedrock/smart_scraper_bedrock.py index 03394434..9c747c00 100644 --- a/examples/bedrock/smart_scraper_bedrock.py +++ b/examples/bedrock/smart_scraper_bedrock.py @@ -19,9 +19,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/smart_scraper_multi_bedrock.py b/examples/bedrock/smart_scraper_multi_bedrock.py index 7aeb71cd..b363d6ab 100644 --- a/examples/bedrock/smart_scraper_multi_bedrock.py +++ b/examples/bedrock/smart_scraper_multi_bedrock.py @@ -17,9 +17,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/smart_scraper_schema_bedrock.py b/examples/bedrock/smart_scraper_schema_bedrock.py index 6213ea1f..2829efec 100644 --- a/examples/bedrock/smart_scraper_schema_bedrock.py +++ b/examples/bedrock/smart_scraper_schema_bedrock.py @@ -26,9 +26,6 @@ graph_config = { "client": "client_name", "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 - }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" } } diff --git a/examples/bedrock/xml_scraper_bedrock.py b/examples/bedrock/xml_scraper_bedrock.py index 018a8387..5f81fbf6 100644 --- a/examples/bedrock/xml_scraper_bedrock.py +++ b/examples/bedrock/xml_scraper_bedrock.py @@ -32,9 +32,6 @@ graph_config = { "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" - } } # ************************************************ diff --git a/examples/bedrock/xml_scraper_graph_multi_bedrock.py b/examples/bedrock/xml_scraper_graph_multi_bedrock.py index a0ed3560..638ce280 100644 --- a/examples/bedrock/xml_scraper_graph_multi_bedrock.py +++ b/examples/bedrock/xml_scraper_graph_multi_bedrock.py @@ -29,9 +29,6 @@ graph_config = { "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", "temperature": 0.0 }, - "embeddings": { - "model": "bedrock/cohere.embed-multilingual-v3" - } } # ************************************************ diff --git a/examples/deepseek/csv_scraper_deepseek.py b/examples/deepseek/csv_scraper_deepseek.py index fd55469d..b734b543 100644 --- a/examples/deepseek/csv_scraper_deepseek.py +++ b/examples/deepseek/csv_scraper_deepseek.py @@ -30,11 +30,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/csv_scraper_graph_multi_deepseek.py b/examples/deepseek/csv_scraper_graph_multi_deepseek.py index d665bc31..ea5e9154 100644 --- a/examples/deepseek/csv_scraper_graph_multi_deepseek.py +++ b/examples/deepseek/csv_scraper_graph_multi_deepseek.py @@ -30,11 +30,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/custom_graph_deepseek.py b/examples/deepseek/custom_graph_deepseek.py deleted file mode 100644 index a265db95..00000000 --- a/examples/deepseek/custom_graph_deepseek.py +++ /dev/null @@ -1,89 +0,0 @@ -""" -Example of custom graph using Gemini Google model -""" - -import os -from dotenv import load_dotenv -from scrapegraphai.models import Gemini -from scrapegraphai.graphs import BaseGraph -from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode -load_dotenv() - -# ************************************************ -# Define the configuration for the graph -# ************************************************ - -deepseek_key = os.getenv("DEEPSEEK_APIKEY") - -graph_config = { - "llm": { - "model": "deepseek-chat", - "openai_api_key": deepseek_key, - "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, -} - -# ************************************************ -# Define the graph nodes -# ************************************************ - -llm_model = Gemini(graph_config["llm"]) - -# define the nodes for the graph -fetch_node = FetchNode( - input="url | local_dir", - output=["doc"], -) -parse_node = ParseNode( - input="doc", - output=["parsed_doc"], - node_config={"chunk_size": 4096} -) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={"llm": llm_model}, -) -generate_answer_node = GenerateAnswerNode( - input="user_prompt & (relevant_chunks | parsed_doc | doc)", - output=["answer"], - node_config={"llm": llm_model}, -) - -# ************************************************ -# Create the graph by defining the connections -# ************************************************ - -graph = BaseGraph( - nodes={ - fetch_node, - parse_node, - rag_node, - generate_answer_node, - }, - edges={ - (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) - }, - entry_point=fetch_node -) - -# ************************************************ -# Execute the graph -# ************************************************ - -result, execution_info = graph.execute({ - "user_prompt": "List me the projects with their description", - "url": "https://perinim.github.io/projects/" -}) - -# get the answer from the result -result = result.get("answer", "No answer found.") -print(result) diff --git a/examples/deepseek/json_scraper_deepseek.py b/examples/deepseek/json_scraper_deepseek.py index 696a08d9..dfe6f489 100644 --- a/examples/deepseek/json_scraper_deepseek.py +++ b/examples/deepseek/json_scraper_deepseek.py @@ -29,11 +29,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/json_scraper_multi_deepseek.py b/examples/deepseek/json_scraper_multi_deepseek.py index 17660ddb..b957dde0 100644 --- a/examples/deepseek/json_scraper_multi_deepseek.py +++ b/examples/deepseek/json_scraper_multi_deepseek.py @@ -15,11 +15,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/pdf_scraper_graph_deepseek.py b/examples/deepseek/pdf_scraper_graph_deepseek.py index fe6f2658..d66bbef5 100644 --- a/examples/deepseek/pdf_scraper_graph_deepseek.py +++ b/examples/deepseek/pdf_scraper_graph_deepseek.py @@ -20,11 +20,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/pdf_scraper_multi_deepseek.py b/examples/deepseek/pdf_scraper_multi_deepseek.py index c884b798..211e4635 100644 --- a/examples/deepseek/pdf_scraper_multi_deepseek.py +++ b/examples/deepseek/pdf_scraper_multi_deepseek.py @@ -15,11 +15,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/scrape_plain_text_deepseek.py b/examples/deepseek/scrape_plain_text_deepseek.py index 7076dd39..d7a070d7 100644 --- a/examples/deepseek/scrape_plain_text_deepseek.py +++ b/examples/deepseek/scrape_plain_text_deepseek.py @@ -31,11 +31,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/script_generator_deepseek.py b/examples/deepseek/script_generator_deepseek.py index 09db0876..fd5fd4dd 100644 --- a/examples/deepseek/script_generator_deepseek.py +++ b/examples/deepseek/script_generator_deepseek.py @@ -20,11 +20,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "library": "beautifulsoup" } diff --git a/examples/deepseek/script_multi_generator_deepseek.py b/examples/deepseek/script_multi_generator_deepseek.py index 41e363b5..2ebfd90a 100644 --- a/examples/deepseek/script_multi_generator_deepseek.py +++ b/examples/deepseek/script_multi_generator_deepseek.py @@ -20,11 +20,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "library": "beautifulsoup" } diff --git a/examples/deepseek/search_graph_deepseek.py b/examples/deepseek/search_graph_deepseek.py index d607e1b1..176d6107 100644 --- a/examples/deepseek/search_graph_deepseek.py +++ b/examples/deepseek/search_graph_deepseek.py @@ -18,11 +18,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "max_results": 2, "verbose": True, diff --git a/examples/deepseek/search_graph_schema_deepseek.py b/examples/deepseek/search_graph_schema_deepseek.py index 8debee2f..f5db278e 100644 --- a/examples/deepseek/search_graph_schema_deepseek.py +++ b/examples/deepseek/search_graph_schema_deepseek.py @@ -34,11 +34,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/search_link_graph_deepseek.py b/examples/deepseek/search_link_graph_deepseek.py index 30e4a9b3..6a35f177 100644 --- a/examples/deepseek/search_link_graph_deepseek.py +++ b/examples/deepseek/search_link_graph_deepseek.py @@ -19,11 +19,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/smart_scraper_deepseek.py b/examples/deepseek/smart_scraper_deepseek.py index 9fe00a2a..ed291b02 100644 --- a/examples/deepseek/smart_scraper_deepseek.py +++ b/examples/deepseek/smart_scraper_deepseek.py @@ -21,11 +21,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/smart_scraper_multi_deepseek.py b/examples/deepseek/smart_scraper_multi_deepseek.py index c88ab525..fafe7261 100644 --- a/examples/deepseek/smart_scraper_multi_deepseek.py +++ b/examples/deepseek/smart_scraper_multi_deepseek.py @@ -19,11 +19,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/smart_scraper_schema_deepseek.py b/examples/deepseek/smart_scraper_schema_deepseek.py index a16ae575..5cbbb702 100644 --- a/examples/deepseek/smart_scraper_schema_deepseek.py +++ b/examples/deepseek/smart_scraper_schema_deepseek.py @@ -33,11 +33,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/xml_scraper_deepseek.py b/examples/deepseek/xml_scraper_deepseek.py index 3b2af61b..ba401b91 100644 --- a/examples/deepseek/xml_scraper_deepseek.py +++ b/examples/deepseek/xml_scraper_deepseek.py @@ -31,11 +31,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/deepseek/xml_scraper_graph_multi_deepseek.py b/examples/deepseek/xml_scraper_graph_multi_deepseek.py index 5d3c29d5..0f53a6b2 100644 --- a/examples/deepseek/xml_scraper_graph_multi_deepseek.py +++ b/examples/deepseek/xml_scraper_graph_multi_deepseek.py @@ -30,11 +30,6 @@ graph_config = { "model": "deepseek-chat", "openai_api_key": deepseek_key, "openai_api_base": 'https://api.deepseek.com/v1', - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/fireworks/csv_scraper_fireworks.py b/examples/fireworks/csv_scraper_fireworks.py index b1d7526d..f588c4c5 100644 --- a/examples/fireworks/csv_scraper_fireworks.py +++ b/examples/fireworks/csv_scraper_fireworks.py @@ -29,12 +29,6 @@ graph_config = { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, - "verbose": True, "headless": False, } diff --git a/examples/fireworks/csv_scraper_graph_multi_fireworks.py b/examples/fireworks/csv_scraper_graph_multi_fireworks.py index 81393d60..ebc46e61 100644 --- a/examples/fireworks/csv_scraper_graph_multi_fireworks.py +++ b/examples/fireworks/csv_scraper_graph_multi_fireworks.py @@ -28,11 +28,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/custom_graph_fireworks.py b/examples/fireworks/custom_graph_fireworks.py index a02b774e..d0dcd994 100644 --- a/examples/fireworks/custom_graph_fireworks.py +++ b/examples/fireworks/custom_graph_fireworks.py @@ -4,9 +4,7 @@ Example of custom graph using existing nodes import os from dotenv import load_dotenv - -from langchain_openai import OpenAIEmbeddings -from scrapegraphai.models import OpenAI +from langchain_openai import ChatOpenAI from scrapegraphai.graphs import BaseGraph from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode load_dotenv() @@ -21,11 +19,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, @@ -35,8 +28,7 @@ graph_config = { # Define the graph nodes # ************************************************ -llm_model = OpenAI(graph_config["llm"]) -embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) +llm_model = ChatOpenAI(graph_config["llm"]) # define the nodes for the graph robot_node = RobotsNode( @@ -65,15 +57,7 @@ parse_node = ParseNode( "verbose": True, } ) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) + generate_answer_node = GenerateAnswerNode( input="user_prompt & (relevant_chunks | parsed_doc | doc)", output=["answer"], @@ -92,14 +76,11 @@ graph = BaseGraph( robot_node, fetch_node, parse_node, - rag_node, - generate_answer_node, ], edges=[ (robot_node, fetch_node), (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) + (parse_node, generate_answer_node) ], entry_point=robot_node ) diff --git a/examples/fireworks/deep_scraper_fireworks.py b/examples/fireworks/deep_scraper_fireworks.py index 67a80868..86fb1717 100644 --- a/examples/fireworks/deep_scraper_fireworks.py +++ b/examples/fireworks/deep_scraper_fireworks.py @@ -19,11 +19,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "max_depth": 1 @@ -49,4 +44,4 @@ print(result) graph_exec_info = deep_scraper_graph.get_execution_info() print(deep_scraper_graph.get_state("relevant_links")) -print(prettify_exec_info(graph_exec_info)) \ No newline at end of file +print(prettify_exec_info(graph_exec_info)) diff --git a/examples/fireworks/json_scraper_fireworkspy.py b/examples/fireworks/json_scraper_fireworkspy.py index 0dd188fb..a76a89c5 100644 --- a/examples/fireworks/json_scraper_fireworkspy.py +++ b/examples/fireworks/json_scraper_fireworkspy.py @@ -29,11 +29,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/json_scraper_multi_fireworks.py b/examples/fireworks/json_scraper_multi_fireworks.py index b4cf4fc7..cd16c525 100644 --- a/examples/fireworks/json_scraper_multi_fireworks.py +++ b/examples/fireworks/json_scraper_multi_fireworks.py @@ -14,11 +14,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/pdf_scraper_fireworks.py b/examples/fireworks/pdf_scraper_fireworks.py index 20db556b..3bb3f3d4 100644 --- a/examples/fireworks/pdf_scraper_fireworks.py +++ b/examples/fireworks/pdf_scraper_fireworks.py @@ -15,11 +15,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/fireworks/pdf_scraper_multi_fireworks.py b/examples/fireworks/pdf_scraper_multi_fireworks.py index 891a4454..c1077061 100644 --- a/examples/fireworks/pdf_scraper_multi_fireworks.py +++ b/examples/fireworks/pdf_scraper_multi_fireworks.py @@ -20,11 +20,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/fireworks/scrape_plain_text_fireworks.py b/examples/fireworks/scrape_plain_text_fireworks.py index a45b2691..331f05e2 100644 --- a/examples/fireworks/scrape_plain_text_fireworks.py +++ b/examples/fireworks/scrape_plain_text_fireworks.py @@ -32,11 +32,6 @@ graph_config = { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, } diff --git a/examples/fireworks/script_generator_fireworks.py b/examples/fireworks/script_generator_fireworks.py index dea59e12..2ee3294c 100644 --- a/examples/fireworks/script_generator_fireworks.py +++ b/examples/fireworks/script_generator_fireworks.py @@ -19,11 +19,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/script_generator_schema_fireworks.py b/examples/fireworks/script_generator_schema_fireworks.py index f7aa4c83..6355a4e8 100644 --- a/examples/fireworks/script_generator_schema_fireworks.py +++ b/examples/fireworks/script_generator_schema_fireworks.py @@ -32,11 +32,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "library": "beautifulsoup", diff --git a/examples/fireworks/script_multi_generator_fireworks.py b/examples/fireworks/script_multi_generator_fireworks.py index 42aff923..98671768 100644 --- a/examples/fireworks/script_multi_generator_fireworks.py +++ b/examples/fireworks/script_multi_generator_fireworks.py @@ -19,11 +19,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "library": "beautifulsoup", diff --git a/examples/fireworks/search_graph_fireworks.py b/examples/fireworks/search_graph_fireworks.py index 4d4d33cb..a091190c 100644 --- a/examples/fireworks/search_graph_fireworks.py +++ b/examples/fireworks/search_graph_fireworks.py @@ -18,11 +18,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "max_results": 2, "verbose": True, diff --git a/examples/fireworks/search_graph_schema_fireworks.py b/examples/fireworks/search_graph_schema_fireworks.py index 9180522b..d88d991e 100644 --- a/examples/fireworks/search_graph_schema_fireworks.py +++ b/examples/fireworks/search_graph_schema_fireworks.py @@ -33,11 +33,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "max_results": 2, "verbose": True, diff --git a/examples/fireworks/search_link_graph_fireworks.py b/examples/fireworks/search_link_graph_fireworks.py index a1d3a979..e71e2a4f 100644 --- a/examples/fireworks/search_link_graph_fireworks.py +++ b/examples/fireworks/search_link_graph_fireworks.py @@ -18,11 +18,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "max_results": 2, "verbose": True, diff --git a/examples/fireworks/smart_scraper_fireworks.py b/examples/fireworks/smart_scraper_fireworks.py index 40071d8f..cff9aedb 100644 --- a/examples/fireworks/smart_scraper_fireworks.py +++ b/examples/fireworks/smart_scraper_fireworks.py @@ -20,11 +20,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/smart_scraper_multi_fireworks.py b/examples/fireworks/smart_scraper_multi_fireworks.py index 68e28055..09e2c811 100644 --- a/examples/fireworks/smart_scraper_multi_fireworks.py +++ b/examples/fireworks/smart_scraper_multi_fireworks.py @@ -19,11 +19,7 @@ graph_config = { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, + "verbose": True, "headless": False, } diff --git a/examples/fireworks/smart_scraper_schema_fireworks.py b/examples/fireworks/smart_scraper_schema_fireworks.py index b8685c3e..d71593f3 100644 --- a/examples/fireworks/smart_scraper_schema_fireworks.py +++ b/examples/fireworks/smart_scraper_schema_fireworks.py @@ -31,11 +31,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/xml_scraper_fireworks.py b/examples/fireworks/xml_scraper_fireworks.py index efc98bd8..59d9e6a3 100644 --- a/examples/fireworks/xml_scraper_fireworks.py +++ b/examples/fireworks/xml_scraper_fireworks.py @@ -28,11 +28,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/fireworks/xml_scraper_graph_multi_fireworks.py b/examples/fireworks/xml_scraper_graph_multi_fireworks.py index d14b8db0..690836a4 100644 --- a/examples/fireworks/xml_scraper_graph_multi_fireworks.py +++ b/examples/fireworks/xml_scraper_graph_multi_fireworks.py @@ -29,11 +29,6 @@ graph_config = { "llm": { "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False, diff --git a/examples/groq/csv_scraper_graph_multi_groq.py b/examples/groq/csv_scraper_graph_multi_groq.py index 87e3279c..475b8cac 100644 --- a/examples/groq/csv_scraper_graph_multi_groq.py +++ b/examples/groq/csv_scraper_graph_multi_groq.py @@ -30,11 +30,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/csv_scraper_groq.py b/examples/groq/csv_scraper_groq.py index 20839a75..805ce5fc 100644 --- a/examples/groq/csv_scraper_groq.py +++ b/examples/groq/csv_scraper_groq.py @@ -31,11 +31,6 @@ graph_config = { "api_key": groq_key, "temperature": 0 }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, } # ************************************************ # Create the CSVScraperGraph instance and run it diff --git a/examples/groq/custom_graph_groq.py b/examples/groq/custom_graph_groq.py index d0384ffd..79d2f0c6 100644 --- a/examples/groq/custom_graph_groq.py +++ b/examples/groq/custom_graph_groq.py @@ -4,7 +4,7 @@ Example of custom graph using existing nodes import os from dotenv import load_dotenv -from scrapegraphai.models import OpenAI +from langchain_openai import ChatOpenAI from scrapegraphai.graphs import BaseGraph from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode load_dotenv() @@ -19,11 +19,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False @@ -33,7 +28,7 @@ graph_config = { # Define the graph nodes # ************************************************ -llm_model = OpenAI(graph_config["llm"]) +llm_model = ChatOpenAI(graph_config["llm"]) # define the nodes for the graph robot_node = RobotsNode( @@ -62,14 +57,7 @@ parse_node = ParseNode( "verbose": True, } ) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "verbose": True, - } -) + generate_answer_node = GenerateAnswerNode( input="user_prompt & (relevant_chunks | parsed_doc | doc)", output=["answer"], @@ -88,14 +76,12 @@ graph = BaseGraph( robot_node, fetch_node, parse_node, - rag_node, generate_answer_node, ], edges=[ (robot_node, fetch_node), (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) + (parse_node, generate_answer_node) ], entry_point=robot_node ) diff --git a/examples/groq/json_scraper_groq.py b/examples/groq/json_scraper_groq.py index 3faddae8..a9099069 100644 --- a/examples/groq/json_scraper_groq.py +++ b/examples/groq/json_scraper_groq.py @@ -30,11 +30,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False diff --git a/examples/groq/json_scraper_multi_groq.py b/examples/groq/json_scraper_multi_groq.py index 13b49be6..df3b9276 100644 --- a/examples/groq/json_scraper_multi_groq.py +++ b/examples/groq/json_scraper_multi_groq.py @@ -15,11 +15,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "library": "beautifulsoup" } diff --git a/examples/groq/pdf_scraper_graph_groq.py b/examples/groq/pdf_scraper_graph_groq.py index a9ca57ee..2560c11e 100644 --- a/examples/groq/pdf_scraper_graph_groq.py +++ b/examples/groq/pdf_scraper_graph_groq.py @@ -18,11 +18,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, } diff --git a/examples/groq/pdf_scraper_multi_groq.py b/examples/groq/pdf_scraper_multi_groq.py index f1afc058..c43a7087 100644 --- a/examples/groq/pdf_scraper_multi_groq.py +++ b/examples/groq/pdf_scraper_multi_groq.py @@ -14,11 +14,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "library": "beautifulsoup" } diff --git a/examples/groq/scrape_plain_text_groq.py b/examples/groq/scrape_plain_text_groq.py index 73cda250..329df51f 100644 --- a/examples/groq/scrape_plain_text_groq.py +++ b/examples/groq/scrape_plain_text_groq.py @@ -32,11 +32,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False diff --git a/examples/groq/script_generator_groq.py b/examples/groq/script_generator_groq.py index a370eb3c..9e280e2b 100644 --- a/examples/groq/script_generator_groq.py +++ b/examples/groq/script_generator_groq.py @@ -19,11 +19,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "library": "beautifulsoup" } diff --git a/examples/groq/script_multi_generator_groq.py b/examples/groq/script_multi_generator_groq.py index 1757a3de..31f4041e 100644 --- a/examples/groq/script_multi_generator_groq.py +++ b/examples/groq/script_multi_generator_groq.py @@ -20,11 +20,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "library": "beautifulsoup" } diff --git a/examples/groq/search_graph_groq.py b/examples/groq/search_graph_groq.py index e82ffb7c..e3044c0e 100644 --- a/examples/groq/search_graph_groq.py +++ b/examples/groq/search_graph_groq.py @@ -21,11 +21,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/search_graph_schema_groq.py b/examples/groq/search_graph_schema_groq.py index 41f03dc4..4cc2209d 100644 --- a/examples/groq/search_graph_schema_groq.py +++ b/examples/groq/search_graph_schema_groq.py @@ -34,11 +34,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/search_link_graph_groq.py b/examples/groq/search_link_graph_groq.py index f940c2a4..5d82f37f 100644 --- a/examples/groq/search_link_graph_groq.py +++ b/examples/groq/search_link_graph_groq.py @@ -19,11 +19,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/smart_scraper_groq.py b/examples/groq/smart_scraper_groq.py index f828cdec..ab38edc0 100644 --- a/examples/groq/smart_scraper_groq.py +++ b/examples/groq/smart_scraper_groq.py @@ -20,11 +20,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/smart_scraper_multi_groq.py b/examples/groq/smart_scraper_multi_groq.py index 18ba3992..6ead098c 100644 --- a/examples/groq/smart_scraper_multi_groq.py +++ b/examples/groq/smart_scraper_multi_groq.py @@ -19,11 +19,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False diff --git a/examples/groq/smart_scraper_schema_groq.py b/examples/groq/smart_scraper_schema_groq.py index e0c51c98..f9c1a40b 100644 --- a/examples/groq/smart_scraper_schema_groq.py +++ b/examples/groq/smart_scraper_schema_groq.py @@ -33,11 +33,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/xml_scraper_graph_multi_groq.py b/examples/groq/xml_scraper_graph_multi_groq.py index 7b102c0f..62540671 100644 --- a/examples/groq/xml_scraper_graph_multi_groq.py +++ b/examples/groq/xml_scraper_graph_multi_groq.py @@ -30,11 +30,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "headless": False } diff --git a/examples/groq/xml_scraper_groq.py b/examples/groq/xml_scraper_groq.py index 1c086175..2172ea77 100644 --- a/examples/groq/xml_scraper_groq.py +++ b/examples/groq/xml_scraper_groq.py @@ -30,11 +30,6 @@ graph_config = { "model": "groq/gemma-7b-it", "api_key": groq_key, "temperature": 0 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, "verbose": True, "headless": False diff --git a/examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.py b/examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.py index 4517bbe9..48b04dab 100644 --- a/examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.py +++ b/examples/huggingfacehub/csv_scraper_graph_multi_huggingfacehub.py @@ -40,7 +40,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } diff --git a/examples/huggingfacehub/csv_scraper_huggingfacehub.py b/examples/huggingfacehub/csv_scraper_huggingfacehub.py index 9d1dbe0b..18ce1194 100644 --- a/examples/huggingfacehub/csv_scraper_huggingfacehub.py +++ b/examples/huggingfacehub/csv_scraper_huggingfacehub.py @@ -43,7 +43,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/custom_graph_huggingfacehub.py b/examples/huggingfacehub/custom_graph_huggingfacehub.py index ad903b5d..0c392cc1 100644 --- a/examples/huggingfacehub/custom_graph_huggingfacehub.py +++ b/examples/huggingfacehub/custom_graph_huggingfacehub.py @@ -33,7 +33,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/json_scraper_huggingfacehub.py b/examples/huggingfacehub/json_scraper_huggingfacehub.py index 3a9a163d..d709cc0d 100644 --- a/examples/huggingfacehub/json_scraper_huggingfacehub.py +++ b/examples/huggingfacehub/json_scraper_huggingfacehub.py @@ -43,7 +43,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/json_scraper_multi_huggingfacehub.py b/examples/huggingfacehub/json_scraper_multi_huggingfacehub.py index 8ca3ba51..c029431e 100644 --- a/examples/huggingfacehub/json_scraper_multi_huggingfacehub.py +++ b/examples/huggingfacehub/json_scraper_multi_huggingfacehub.py @@ -24,7 +24,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } FILE_NAME = "inputs/example.json" curr_dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/examples/huggingfacehub/pdf_scraper_graph_huggingfacehub.py b/examples/huggingfacehub/pdf_scraper_graph_huggingfacehub.py index bb2724fe..eb0b1895 100644 --- a/examples/huggingfacehub/pdf_scraper_graph_huggingfacehub.py +++ b/examples/huggingfacehub/pdf_scraper_graph_huggingfacehub.py @@ -25,7 +25,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } source = """ diff --git a/examples/huggingfacehub/pdf_scraper_multi_huggingfacehub.py b/examples/huggingfacehub/pdf_scraper_multi_huggingfacehub.py index d24d522c..4db809b2 100644 --- a/examples/huggingfacehub/pdf_scraper_multi_huggingfacehub.py +++ b/examples/huggingfacehub/pdf_scraper_multi_huggingfacehub.py @@ -23,7 +23,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # Covert to list diff --git a/examples/huggingfacehub/scrape_plain_text_huggingfacehub.py b/examples/huggingfacehub/scrape_plain_text_huggingfacehub.py index f07e5666..76d32cda 100644 --- a/examples/huggingfacehub/scrape_plain_text_huggingfacehub.py +++ b/examples/huggingfacehub/scrape_plain_text_huggingfacehub.py @@ -45,7 +45,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/script_generator_huggingfacehub.py b/examples/huggingfacehub/script_generator_huggingfacehub.py index 4804db93..a3fcaaf4 100644 --- a/examples/huggingfacehub/script_generator_huggingfacehub.py +++ b/examples/huggingfacehub/script_generator_huggingfacehub.py @@ -36,7 +36,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ # Create the ScriptCreatorGraph instance and run it diff --git a/examples/huggingfacehub/script_multi_generator_huggingfacehub.py b/examples/huggingfacehub/script_multi_generator_huggingfacehub.py index 5afeff0d..0ee89189 100644 --- a/examples/huggingfacehub/script_multi_generator_huggingfacehub.py +++ b/examples/huggingfacehub/script_multi_generator_huggingfacehub.py @@ -33,7 +33,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/search_graph_huggingfacehub.py b/examples/huggingfacehub/search_graph_huggingfacehub.py index b3c58ce5..7c4a0c43 100644 --- a/examples/huggingfacehub/search_graph_huggingfacehub.py +++ b/examples/huggingfacehub/search_graph_huggingfacehub.py @@ -29,7 +29,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/search_link_graph_huggingfacehub.py b/examples/huggingfacehub/search_link_graph_huggingfacehub.py index a49fb3b9..75b41282 100644 --- a/examples/huggingfacehub/search_link_graph_huggingfacehub.py +++ b/examples/huggingfacehub/search_link_graph_huggingfacehub.py @@ -26,7 +26,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } diff --git a/examples/huggingfacehub/smart_scraper_huggingfacehub.py b/examples/huggingfacehub/smart_scraper_huggingfacehub.py index bd415d41..6f9a863f 100644 --- a/examples/huggingfacehub/smart_scraper_huggingfacehub.py +++ b/examples/huggingfacehub/smart_scraper_huggingfacehub.py @@ -38,7 +38,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } smart_scraper_graph = SmartScraperGraph( diff --git a/examples/huggingfacehub/smart_scraper_multi_huggingfacehub.py b/examples/huggingfacehub/smart_scraper_multi_huggingfacehub.py index e1a332f9..046883a2 100644 --- a/examples/huggingfacehub/smart_scraper_multi_huggingfacehub.py +++ b/examples/huggingfacehub/smart_scraper_multi_huggingfacehub.py @@ -28,7 +28,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ******************************************************* diff --git a/examples/huggingfacehub/smart_scraper_schema_huggingfacehub.py b/examples/huggingfacehub/smart_scraper_schema_huggingfacehub.py index 784079e4..31719697 100644 --- a/examples/huggingfacehub/smart_scraper_schema_huggingfacehub.py +++ b/examples/huggingfacehub/smart_scraper_schema_huggingfacehub.py @@ -48,7 +48,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } smart_scraper_graph = SmartScraperGraph( diff --git a/examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.py b/examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.py index 24d6babd..1a244b86 100644 --- a/examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.py +++ b/examples/huggingfacehub/xml_scraper_graph_multi_huggingfacehub.py @@ -40,7 +40,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/huggingfacehub/xml_scraper_huggingfacehub.py b/examples/huggingfacehub/xml_scraper_huggingfacehub.py index cc8a4425..ddd73b5f 100644 --- a/examples/huggingfacehub/xml_scraper_huggingfacehub.py +++ b/examples/huggingfacehub/xml_scraper_huggingfacehub.py @@ -40,7 +40,6 @@ embedder_model_instance = HuggingFaceInferenceAPIEmbeddings( graph_config = { "llm": {"model_instance": llm_model_instance}, - "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/local_models/custom_graph_ollama.py b/examples/local_models/custom_graph_ollama.py index b9a42949..66dd59b6 100644 --- a/examples/local_models/custom_graph_ollama.py +++ b/examples/local_models/custom_graph_ollama.py @@ -4,7 +4,7 @@ Example of custom graph using existing nodes import os from langchain_openai import OpenAIEmbeddings -from scrapegraphai.models import OpenAI +from langchain_openai import ChatOpenAI from scrapegraphai.graphs import BaseGraph from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode @@ -20,11 +20,7 @@ graph_config = { # "model_tokens": 2000, # set context length arbitrarily "base_url": "http://localhost:11434", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, + "verbose": True, } @@ -32,7 +28,7 @@ graph_config = { # Define the graph nodes # ************************************************ -llm_model = OpenAI(graph_config["llm"]) +llm_model = ChatOpenAI(graph_config["llm"]) embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key) # define the nodes for the graph @@ -62,15 +58,7 @@ parse_node = ParseNode( "verbose": True, } ) -rag_node = RAGNode( - input="user_prompt & (parsed_doc | doc)", - output=["relevant_chunks"], - node_config={ - "llm_model": llm_model, - "embedder_model": embedder, - "verbose": True, - } -) + generate_answer_node = GenerateAnswerNode( input="user_prompt & (relevant_chunks | parsed_doc | doc)", output=["answer"], @@ -89,14 +77,12 @@ graph = BaseGraph( robot_node, fetch_node, parse_node, - rag_node, generate_answer_node, ], edges=[ (robot_node, fetch_node), (fetch_node, parse_node), - (parse_node, rag_node), - (rag_node, generate_answer_node) + (parse_node, generate_answer_node) ], entry_point=robot_node ) diff --git a/examples/local_models/json_scraper_multi_ollama.py b/examples/local_models/json_scraper_multi_ollama.py index 91f4fab4..6e9c3da3 100644 --- a/examples/local_models/json_scraper_multi_ollama.py +++ b/examples/local_models/json_scraper_multi_ollama.py @@ -12,10 +12,6 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly "model_tokens": 4000, }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - }, "verbose": True, "headless": False, } diff --git a/examples/local_models/json_scraper_ollama.py b/examples/local_models/json_scraper_ollama.py index 2dd072ac..ca4eb32e 100644 --- a/examples/local_models/json_scraper_ollama.py +++ b/examples/local_models/json_scraper_ollama.py @@ -31,11 +31,6 @@ graph_config = { # "model_tokens": 2000, # set context length arbitrarily "base_url": "http://localhost:11434", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, "verbose": True, } diff --git a/examples/local_models/pdf_scraper_multi_ollama.py b/examples/local_models/pdf_scraper_multi_ollama.py index c0b65a63..ce258bf6 100644 --- a/examples/local_models/pdf_scraper_multi_ollama.py +++ b/examples/local_models/pdf_scraper_multi_ollama.py @@ -11,10 +11,6 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly "model_tokens": 4000, }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - }, "verbose": True, } diff --git a/examples/local_models/pdf_scraper_ollama.py b/examples/local_models/pdf_scraper_ollama.py index d79afb3a..84eb40f9 100644 --- a/examples/local_models/pdf_scraper_ollama.py +++ b/examples/local_models/pdf_scraper_ollama.py @@ -10,10 +10,6 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly "model_tokens": 4000, }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - }, "verbose": True, "headless": False, } diff --git a/examples/local_models/scrape_plain_text_ollama.py b/examples/local_models/scrape_plain_text_ollama.py index 9700d713..fe24c2a9 100644 --- a/examples/local_models/scrape_plain_text_ollama.py +++ b/examples/local_models/scrape_plain_text_ollama.py @@ -30,11 +30,6 @@ graph_config = { # "model_tokens": 2000, # set context length arbitrarily "base_url": "http://localhost:11434", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, "verbose": True, } diff --git a/examples/local_models/script_multi_generator_ollama.py b/examples/local_models/script_multi_generator_ollama.py index dc34c910..d94faba6 100644 --- a/examples/local_models/script_multi_generator_ollama.py +++ b/examples/local_models/script_multi_generator_ollama.py @@ -20,11 +20,6 @@ graph_config = { # "model_tokens": 2000, # set context length arbitrarily, "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, "library": "beautifoulsoup", "verbose": True, } diff --git a/examples/local_models/search_graph_ollama.py b/examples/local_models/search_graph_ollama.py index 8ecb60c1..039ca00e 100644 --- a/examples/local_models/search_graph_ollama.py +++ b/examples/local_models/search_graph_ollama.py @@ -16,11 +16,6 @@ graph_config = { # "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, "max_results": 5, "verbose": True, } diff --git a/examples/local_models/search_graph_schema_ollama.py b/examples/local_models/search_graph_schema_ollama.py index ae7c0632..fb87954f 100644 --- a/examples/local_models/search_graph_schema_ollama.py +++ b/examples/local_models/search_graph_schema_ollama.py @@ -29,11 +29,6 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, "verbose": True, "headless": False } diff --git a/examples/local_models/search_link_graph_ollama.py b/examples/local_models/search_link_graph_ollama.py index 5c594270..a05067dd 100644 --- a/examples/local_models/search_link_graph_ollama.py +++ b/examples/local_models/search_link_graph_ollama.py @@ -14,11 +14,7 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, + "verbose": True, "headless": False } diff --git a/examples/local_models/smart_scraper_ollama.py b/examples/local_models/smart_scraper_ollama.py index c3d60559..01c9f964 100644 --- a/examples/local_models/smart_scraper_ollama.py +++ b/examples/local_models/smart_scraper_ollama.py @@ -14,11 +14,7 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, + "verbose": True, "headless": False } diff --git a/examples/local_models/smart_scraper_schema_ollama.py b/examples/local_models/smart_scraper_schema_ollama.py index 7168d513..5fcff433 100644 --- a/examples/local_models/smart_scraper_schema_ollama.py +++ b/examples/local_models/smart_scraper_schema_ollama.py @@ -24,11 +24,7 @@ graph_config = { "format": "json", # Ollama needs the format to be specified explicitly # "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, + "verbose": True, "headless": False } diff --git a/examples/local_models/xml_scraper_graph_multi_ollama.py b/examples/local_models/xml_scraper_graph_multi_ollama.py index d84c6c9f..0494ff2c 100644 --- a/examples/local_models/xml_scraper_graph_multi_ollama.py +++ b/examples/local_models/xml_scraper_graph_multi_ollama.py @@ -29,11 +29,7 @@ graph_config = { # "model_tokens": 2000, # set context length arbitrarily "base_url": "http://localhost:11434", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, + "verbose": True, } diff --git a/examples/local_models/xml_scraper_ollama.py b/examples/local_models/xml_scraper_ollama.py index cc8c3ad9..50c4f8e7 100644 --- a/examples/local_models/xml_scraper_ollama.py +++ b/examples/local_models/xml_scraper_ollama.py @@ -30,11 +30,6 @@ graph_config = { # "model_tokens": 2000, # set context length arbitrarily "base_url": "http://localhost:11434", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, "verbose": True, } diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index 4ed08057..0348b3cc 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -29,8 +29,6 @@ class AbstractGraph(ABC): config (dict): Configuration parameters for the graph. schema (BaseModel): The schema for the graph output. llm_model: An instance of a language model client, configured for generating answers. - embedder_model: An instance of an embedding model client, - configured for generating embeddings. verbose (bool): A flag indicating whether to show print statements during execution. headless (bool): A flag indicating whether to run the graph in headless mode. @@ -85,7 +83,6 @@ class AbstractGraph(ABC): "verbose": self.verbose, "loader_kwargs": self.loader_kwargs, "llm_model": self.llm_model, - "embedder_model": self.embedder_model, "cache_path": self.cache_path, }