diff --git a/pyproject.toml b/pyproject.toml index f0b86b3f..3df0e6bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,6 @@ name = "scrapegraphai" version = "1.14.0b1" - description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py index aeb52ee7..856438cd 100644 --- a/scrapegraphai/nodes/__init__.py +++ b/scrapegraphai/nodes/__init__.py @@ -4,7 +4,6 @@ __init__.py file for node folder from .base_node import BaseNode from .fetch_node import FetchNode -from .conditional_node import ConditionalNode from .get_probable_tags_node import GetProbableTagsNode from .generate_answer_node import GenerateAnswerNode from .parse_node import ParseNode diff --git a/tests/graphs/scrape_plain_text_llama3_test.py b/tests/graphs/scrape_plain_text_llama3.1_test.py similarity index 86% rename from tests/graphs/scrape_plain_text_llama3_test.py rename to tests/graphs/scrape_plain_text_llama3.1_test.py index 93045163..6659c692 100644 --- a/tests/graphs/scrape_plain_text_llama3_test.py +++ b/tests/graphs/scrape_plain_text_llama3.1_test.py @@ -26,15 +26,10 @@ def graph_config(): """ return { "llm": { - "model": "ollama/llama3", + "model": "ollama/llama3.1", "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/scrape_plain_text_mistral_test.py b/tests/graphs/scrape_plain_text_mistral_test.py index b887161c..888999ab 100644 --- a/tests/graphs/scrape_plain_text_mistral_test.py +++ b/tests/graphs/scrape_plain_text_mistral_test.py @@ -30,11 +30,6 @@ def graph_config(): "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/scrape_xml_ollama_test.py b/tests/graphs/scrape_xml_ollama_test.py index 04494543..2bc38a59 100644 --- a/tests/graphs/scrape_xml_ollama_test.py +++ b/tests/graphs/scrape_xml_ollama_test.py @@ -32,11 +32,6 @@ def graph_config(): "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/script_generator_test.py b/tests/graphs/script_generator_test.py index bf5ada42..7bcfeea7 100644 --- a/tests/graphs/script_generator_test.py +++ b/tests/graphs/script_generator_test.py @@ -18,11 +18,6 @@ def graph_config(): "base_url": "http://localhost:11434", "library": "beautifulsoup", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, "library": "beautifulsoup" } diff --git a/tests/graphs/search_link_ollama.py b/tests/graphs/search_link_ollama.py index 3b41f699..530ad2a6 100644 --- a/tests/graphs/search_link_ollama.py +++ b/tests/graphs/search_link_ollama.py @@ -4,14 +4,10 @@ from scrapegraphai.utils import prettify_exec_info def test_smart_scraper_pipeline(): graph_config = { "llm": { - "model": "ollama/llama3", + "model": "ollama/llama3.1", "temperature": 0, "format": "json", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - }, "verbose": True, "headless": False } diff --git a/tests/graphs/smart_scraper_ernie_test.py b/tests/graphs/smart_scraper_ernie_test.py index 5efd8d0b..1da35790 100644 --- a/tests/graphs/smart_scraper_ernie_test.py +++ b/tests/graphs/smart_scraper_ernie_test.py @@ -16,11 +16,6 @@ def graph_config(): "ernie_client_id": "", "ernie_client_secret": "", "temperature": 0.1 - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/smart_scraper_fireworks_test.py b/tests/graphs/smart_scraper_fireworks_test.py index 0cb91dcc..818f15b9 100644 --- a/tests/graphs/smart_scraper_fireworks_test.py +++ b/tests/graphs/smart_scraper_fireworks_test.py @@ -20,11 +20,6 @@ def graph_config(): "api_key": fireworks_api_key, "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct" }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - # "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, "verbose": True, "headless": False, } diff --git a/tests/graphs/smart_scraper_ollama_test.py b/tests/graphs/smart_scraper_ollama_test.py index b35907c0..a358feb6 100644 --- a/tests/graphs/smart_scraper_ollama_test.py +++ b/tests/graphs/smart_scraper_ollama_test.py @@ -16,11 +16,6 @@ def graph_config(): "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } }