feat: refactoring of abstract graph

examples/deepseek/.env.example

@@ -1 +1 @@
-OPENAI_APIKEY="your openai api key"
+DEEPSEEK_APIKEY="your api key"

examples/deepseek/csv_scraper_deepseek.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/csv_scraper_graph_multi_deepseek.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/custom_graph_deepseek.py

@@ -20,6 +20,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/json_scraper_deepseek.py

@@ -29,6 +29,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/json_scraper_multi_deepseek.py

@@ -15,6 +15,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/pdf_scraper_graph_deepseek.py

@@ -20,6 +20,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/pdf_scraper_multi_deepseek.py

@@ -15,6 +15,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/scrape_plain_text_deepseek.py

@@ -31,6 +31,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/script_generator_deepseek.py

@@ -20,6 +20,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }

examples/deepseek/search_graph_deepseek.py

@@ -19,6 +19,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "max_results": 2,
     "verbose": True,

examples/deepseek/smart_scraper_deepseek.py

@@ -21,6 +21,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/smart_scraper_schema_deepseek.py

@@ -41,6 +41,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/xml_scraper_deepseek.py

@@ -31,6 +31,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/deepseek/xml_scraper_graph_multi_deepseek.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/groq/csv_scraper_graph_multi_groq.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }

examples/groq/csv_scraper_groq.py

@@ -31,6 +31,11 @@ graph_config = {
         "api_key": groq_key,
         "temperature": 0
     },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
+    },
 }
 # ************************************************
 # Create the CSVScraperGraph instance and run it

examples/groq/custom_graph_groq.py

@@ -19,6 +19,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False

examples/groq/json_scraper_groq.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False

examples/groq/json_scraper_multi_groq.py

@@ -15,6 +15,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }

examples/groq/pdf_scraper_graph_groq.py

@@ -18,6 +18,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
 }

examples/groq/pdf_scraper_multi_groq.py

@@ -14,6 +14,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }

examples/groq/scrape_plain_text_groq.py

@@ -32,6 +32,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False

examples/groq/script_generator_groq.py

@@ -19,6 +19,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "library": "beautifulsoup"
 }

examples/groq/search_graph_groq.py

@@ -21,6 +21,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }

examples/groq/smart_scraper_groq.py

@@ -22,10 +22,10 @@ graph_config = {
         "api_key": groq_key,
         "temperature": 0
     },
-    "embeddings": {
+     "embeddings": {
         "model": "ollama/nomic-embed-text",
         "temperature": 0,
-        "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }

examples/groq/smart_scraper_multi_groq.py

@@ -19,6 +19,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False

examples/groq/smart_scraper_schema_groq.py

@@ -41,6 +41,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }

examples/groq/xml_scraper_graph_multi_groq.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "headless": False
 }

examples/groq/xml_scraper_groq.py

@@ -30,6 +30,11 @@ graph_config = {
         "model": "groq/gemma-7b-it",
         "api_key": groq_key,
         "temperature": 0
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
     "verbose": True,
     "headless": False

scrapegraphai/graphs/abstract_graph.py

@@ -69,8 +69,7 @@ class AbstractGraph(ABC):
         self.config = config
         self.schema = schema
         self.llm_model = self._create_llm(config["llm"], chat=True)
-        self.embedder_model = self._create_default_embedder(llm_config=config["llm"]                                                            
-                                                            ) if "embeddings" not in config else self._create_embedder(
+        self.embedder_model = self._create_default_embedder(llm_config=config["llm"]                                                            ) if "embeddings" not in config else self._create_embedder(
             config["embeddings"])
         self.verbose = False if config is None else config.get(
             "verbose", False)
@@ -102,6 +101,7 @@ class AbstractGraph(ABC):
             "llm_model": self.llm_model,
             "embedder_model": self.embedder_model
             }
+       
         self.set_common_params(common_params, overwrite=False)
 
         # set burr config
@@ -124,28 +124,7 @@ class AbstractGraph(ABC):
 
         for node in self.graph.nodes:
             node.update_config(params, overwrite)
-
-    def _set_model_token(self, llm):
-
-        if "Azure" in str(type(llm)):
-            try:
-                self.model_token = models_tokens["azure"][llm.model_name]
-            except KeyError:
-                raise KeyError("Model not supported")
-
-        elif "HuggingFaceEndpoint" in str(type(llm)):
-            if "mistral" in llm.repo_id:
-                try:
-                    self.model_token = models_tokens["mistral"][llm.repo_id]
-                except KeyError:
-                    raise KeyError("Model not supported")
-        elif "Google" in str(type(llm)):
-            try:
-                if "gemini" in llm.model:
-                    self.model_token = models_tokens["gemini"][llm.model]
-            except KeyError:
-                raise KeyError("Model not supported")
-
+    
     def _create_llm(self, llm_config: dict, chat=False) -> object:
         """
         Create a large language model instance based on the configuration provided.
@@ -165,8 +144,6 @@ class AbstractGraph(ABC):
 
         # If model instance is passed directly instead of the model details
         if "model_instance" in llm_params:
-            if chat:
-                self._set_model_token(llm_params["model_instance"])
             return llm_params["model_instance"]
 
         # Instantiate the language model based on the model name