From de1ec250ef05ffe8b09fd68ade80410e55831ae7 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sun, 11 Aug 2024 18:04:31 +0200 Subject: [PATCH] refactoring pyproject.toml Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com> --- examples/local_models/script_generator_ollama.py | 9 ++------- pyproject.toml | 6 ++---- scrapegraphai/graphs/abstract_graph.py | 4 ++-- scrapegraphai/nodes/parse_node.py | 2 +- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/examples/local_models/script_generator_ollama.py b/examples/local_models/script_generator_ollama.py index 3ad0b55f..caa0455c 100644 --- a/examples/local_models/script_generator_ollama.py +++ b/examples/local_models/script_generator_ollama.py @@ -9,16 +9,11 @@ from scrapegraphai.utils import prettify_exec_info graph_config = { "llm": { - "model": "ollama/mistral", - "temperature": 0, + "model": "ollama/llama3.1", + "temperature": 0.5, # "model_tokens": 2000, # set context length arbitrarily, "base_url": "http://localhost:11434", # set ollama URL arbitrarily }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", # set ollama URL arbitrarily - }, "library": "beautifoulsoup", "verbose": True, } diff --git a/pyproject.toml b/pyproject.toml index 3df0e6bd..a1fdf6a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,8 @@ dependencies = [ "langchain-groq>=0.1.3", "langchain-aws>=0.1.3", "langchain-anthropic>=0.1.11", + "langchain-mistralai>=0.1.12", + "langchain-huggingface>=0.0.3", "langchain-nvidia-ai-endpoints>=0.1.6", "html2text>=2024.2.26", "faiss-cpu>=1.8.0", @@ -38,11 +40,7 @@ dependencies = [ "google>=3.0.0", "undetected-playwright>=0.3.0", "semchunk>=1.0.1", - "langchain-fireworks>=0.1.3", - "langchain-community>=0.2.9", - "langchain-huggingface>=0.0.3", "browserbase>=0.3.0", - "langchain-mistralai>=0.1.12", ] license = "MIT" diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index f22f764c..ab53862d 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -170,12 +170,12 @@ class AbstractGraph(ABC): if llm_params["model"].startswith("vertexai"): return handle_model(llm_params["model"], "google_vertexai", llm_params["model"]) - + if "ollama" in llm_params["model"]: model_name = llm_params["model"].split("ollama/")[-1] token_key = model_name if "model_tokens" not in llm_params else llm_params["model_tokens"] return handle_model(model_name, "ollama", token_key) - + if "hugging_face" in llm_params["model"]: model_name = llm_params["model"].split("/")[-1] return handle_model(model_name, "hugging_face", model_name) diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py index 48741085..1a5c1119 100644 --- a/scrapegraphai/nodes/parse_node.py +++ b/scrapegraphai/nodes/parse_node.py @@ -90,7 +90,7 @@ class ParseNode(BaseNode): chunk_size=self.node_config.get("chunk_size", 4096)-250, token_counter=lambda text: len(text.split()), memoize=False) - + state.update({self.output[0]: chunks}) return state