refactoring pyproject.toml

Co-Authored-By: Matteo Vedovati <68272450+vedovati-matteo@users.noreply.github.com>

examples/local_models/script_generator_ollama.py

@@ -9,16 +9,11 @@ from scrapegraphai.utils import prettify_exec_info
 
 graph_config = {
     "llm": {
-        "model": "ollama/mistral",
-        "temperature": 0,
+        "model": "ollama/llama3.1",
+        "temperature": 0.5,
         # "model_tokens": 2000, # set context length arbitrarily,
         "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
-    "embeddings": {
-        "model": "ollama/nomic-embed-text",
-        "temperature": 0,
-        "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
-    },
     "library": "beautifoulsoup",
     "verbose": True,
 }

pyproject.toml

@@ -23,6 +23,8 @@ dependencies = [
     "langchain-groq>=0.1.3",
     "langchain-aws>=0.1.3",
     "langchain-anthropic>=0.1.11",
+    "langchain-mistralai>=0.1.12",
+    "langchain-huggingface>=0.0.3",
     "langchain-nvidia-ai-endpoints>=0.1.6",
     "html2text>=2024.2.26",
     "faiss-cpu>=1.8.0",
@@ -38,11 +40,7 @@ dependencies = [
     "google>=3.0.0",
     "undetected-playwright>=0.3.0",
     "semchunk>=1.0.1",
-    "langchain-fireworks>=0.1.3",
-    "langchain-community>=0.2.9",
-    "langchain-huggingface>=0.0.3",
     "browserbase>=0.3.0",
-    "langchain-mistralai>=0.1.12",
 ]
 
 license = "MIT"

scrapegraphai/graphs/abstract_graph.py

@@ -170,12 +170,12 @@ class AbstractGraph(ABC):
 
         if llm_params["model"].startswith("vertexai"):
             return handle_model(llm_params["model"], "google_vertexai", llm_params["model"])
-
+        
         if "ollama" in llm_params["model"]:
             model_name = llm_params["model"].split("ollama/")[-1]
             token_key = model_name if "model_tokens" not in llm_params else llm_params["model_tokens"]
             return handle_model(model_name, "ollama", token_key)
-
+    
         if "hugging_face" in llm_params["model"]:
             model_name = llm_params["model"].split("/")[-1]
             return handle_model(model_name, "hugging_face", model_name)

scrapegraphai/nodes/parse_node.py

@@ -90,7 +90,7 @@ class ParseNode(BaseNode):
                                 chunk_size=self.node_config.get("chunk_size", 4096)-250,
                                 token_counter=lambda text: len(text.split()),
                                 memoize=False)
- 
+
         state.update({self.output[0]: chunks})
 
         return state