chore: made some libs optional

README.md

@@ -32,36 +32,12 @@ The reference page for Scrapegraph-ai is available on the official page of PyPI:
 ```bash
 pip install scrapegraphai
 
-# IMPORTANT (to fetch webpage content)
+# IMPORTANT (to fetch websites content)
 playwright install
 ```
 
 **Note**: it is recommended to install the library in a virtual environment to avoid conflicts with other libraries 🐱
 
-<details>
-<summary><b>Optional Dependencies</b></summary>
-Additional dependecies can be added while installing the library:
-
-- <b>More Language Models</b>: additional language models are installed, such as Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
-
-  This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints.
-  ```bash
-  pip install scrapegraphai[other-language-models]
-  ```
-- <b>Semantic Options</b>: this group includes tools for advanced semantic processing, such as Graphviz.
-
-  ```bash
-  pip install scrapegraphai[more-semantic-options]
-  ```
-
-- <b>Browsers Options</b>: this group includes additional browser management tools/services, such as Browserbase.
-
-  ```bash
-  pip install scrapegraphai[more-browser-options]
-  ```
-
-</details>
-
 
 ## 💻 Usage
 There are multiple standard scraping pipelines that can be used to extract information from a website (or local file).

docs/turkish.md

@@ -31,31 +31,6 @@ playwright install
 
 **Not**: Diğer kütüphanelerle çakışmaları önlemek için kütüphaneyi sanal bir ortamda kurmanız önerilir 🐱
 
-<details>
-<summary><b>Opsiyonel Bağımlılıklar</b></summary>
-Kütüphaneyi kurarken ek bağımlılıklar ekleyebilirsiniz:
-
-- **Daha Fazla Dil Modeli**: Fireworks, Groq, Anthropic, Hugging Face ve Nvidia AI Endpoints gibi ek dil modelleri kurulur.
-
-  Bu grup, Fireworks, Groq, Anthropic, Together AI, Hugging Face ve Nvidia AI Endpoints gibi ek dil modellerini kullanmanızı sağlar.
-
-  ```bash
-  pip install scrapegraphai[other-language-models]
-  ```
-
-- **Semantik Seçenekler**: Graphviz gibi gelişmiş semantik işleme araçlarını içerir.
-
-  ```bash
-  pip install scrapegraphai[more-semantic-options]
-  ```
-
-- **Tarayıcı Seçenekleri**: Browserbase gibi ek tarayıcı yönetim araçları/hizmetlerini içerir.
-
-  ```bash
-  pip install scrapegraphai[more-browser-options]
-  ```
-
-</details>
 
 ## 💻 Kullanım
 

examples/openai/depth_search_graph_openai.py

@@ -7,7 +7,7 @@ from scrapegraphai.graphs import DepthSearchGraph
 
 load_dotenv()
 
-openai_key = os.getenv("OPENAI_APIKEY")
+openai_key = os.getenv("OPENAI_API_KEY")
 
 graph_config = {
     "llm": {

examples/openai/search_graph_openai.py

@@ -11,7 +11,7 @@ load_dotenv()
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("OPENAI_APIKEY")
+openai_key = os.getenv("OPENAI_API_KEY")
 
 graph_config = {
     "llm": {

examples/openai/speech_graph_openai.py

@@ -20,7 +20,7 @@ output_path = os.path.join(curr_dir, FILE_NAME)
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("OPENAI_APIKEY")
+openai_key = os.getenv("OPENAI_API_KEY")
 
 graph_config = {
     "llm": {

funding.json

@@ -1,83 +0,0 @@
-{
-  "id": 0,
-  "guid": "",
-  "version": "v1.0.0",
-  "url": "",
-  "meta": {},
-  "status": "",
-  "status_message": null,
-  "crawl_errors": 0,
-  "crawl_message": null,
-  "created_at": "2024-10-31T10:00:00Z",
-  "updated_at": "2024-10-31T10:00:00Z",
-  "entity": {
-    "type": "organisation",
-    "role": "owner",
-    "name": "ScrapeGraphAI, Inc.",
-    "email": "contact@scrapegraphai.com",
-    "phone": "",
-    "description": "An AI-powered web scraping framework that intelligently extracts structured data from websites with automatic pattern recognition, adaptive scraping strategies, and built-in rate limiting. Recognized as a top 200 open-source AI project globally.",
-    "webpageUrl": {
-      "url": "https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/funding.json"
-    }
-  },
-  "projects": [
-    {
-      "guid": "scrapegraph-core",
-      "name": "ScrapeGraphAI Core",
-      "description": "An AI-powered web scraping framework that intelligently extracts structured data from websites with automatic pattern recognition, adaptive scraping strategies, and built-in rate limiting. Recognized as a top 200 open-source AI project globally.",
-      "webpageUrl": {
-        "url": "https://github.com/ScrapeGraphAI/Scrapegraph-ai/blob/main/funding.json"
-      },
-      "repositoryUrl": {
-        "url": "https://github.com/ScrapeGraphAI/Scrapegraph-ai"
-      },
-      "licenses": [
-        "spdx:MIT"
-      ],
-      "tags": [
-        "web-scraping",
-        "ai",
-        "data-extraction",
-        "python",
-        "machine-learning",
-        "open-source",
-        "llm"
-      ]
-    }
-  ],
-  "funding": {
-    "channels": [
-      {
-        "guid": "stripe",
-        "type": "bank",
-        "address": "https://buy.stripe.com/5kAaGW2E5gHH4vK3ce",
-        "description": "Will accept direct bank transfers via Stripe in the address link, for more info contact us via email contact@scrapegraphai.com"
-      }
-    ],
-    "plans": [
-      {
-        "guid": "developer-compensation",
-        "status": "active",
-        "name": "Developer Compensation",
-        "description": "Provides financial support for developers working on maintenance, updates, and feature additions for the projects.",
-        "amount": 3000,
-        "currency": "USD",
-        "frequency": "monthly",
-        "channels": [
-          "stripe"
-        ]
-      }
-    ],
-    "history": [
-      {
-        "year": 2024,
-        "income": 15000,
-        "expenses": 15000,
-        "taxes": 0,
-        "currency": "USD",
-        "description": "From some companies that sponsor us in our Github repo page"
-      }
-    ]
-  }
-}

pyproject.toml

@@ -11,12 +11,11 @@ authors = [
 
 dependencies = [
     "langchain>=0.3.0",
-    "langchain-google-genai>=1.0.7",
     "langchain-openai>=0.1.22",
     "langchain-mistralai>=0.1.12",
     "langchain_community>=0.2.9",
     "langchain-aws>=0.1.3",
-    "mistral-common>=1.4.0",
+    "langchain-ollama>=0.1.3",
     "html2text>=2024.2.26",
     "beautifulsoup4>=4.12.3",
     "python-dotenv>=1.0.1",
@@ -26,16 +25,11 @@ dependencies = [
     "free-proxy>=1.1.1",
     "playwright>=1.43.0",
     "undetected-playwright>=0.3.0",
-    "langchain-ollama>=0.1.3",
     "semchunk>=2.2.0",
-    "qdrant-client>=1.11.3",
-    "fastembed>=0.3.6",
-    
-    "transformers>=4.44.2",
     "googlesearch-python>=1.2.5",
     "async-timeout>=4.0.3",
     "simpleeval>=1.0.0",
-    "scrapegraph-py>=1.7.0"
+    "jsonschema>=4.23.0",
 ]
 
 readme = "README.md"
@@ -73,30 +67,7 @@ requires-python = ">=3.10,<4.0"
 [project.optional-dependencies]
 burr = ["burr[start]==0.22.1"]
 docs = ["sphinx==6.0", "furo==2024.5.6"]
-
+ocr = [
-# Group 1: Other Language Models
-other-language-models = [
-    "langchain-google-vertexai>=1.0.7",
-    "langchain-fireworks>=0.1.3",
-    "langchain-groq>=0.1.3",
-    "langchain-anthropic>=0.1.11",
-    "langchain-huggingface>=0.0.3",
-    "langchain-nvidia-ai-endpoints>=0.1.6",
-    "langchain_together>=0.2.0"
-]
-
-# Group 2: More Semantic Options
-more-semantic-options = [
-    "graphviz>=0.20.3",
-]
-
-# Group 3: More Browser Options
-more-browser-options = [
-    "browserbase>=0.3.0",
-]
-
-# Group 4: Surya Library
-screenshot_scraper = [
     "surya-ocr>=0.5.0",
     "matplotlib>=3.7.2",
     "ipywidgets>=8.1.0",
@@ -105,21 +76,13 @@ screenshot_scraper = [
 
 [build-system]
 requires = ["hatchling==1.26.3"]
-
 build-backend = "hatchling.build"
 
-[dependency-groups]
-dev = [
-    "burr[start]==0.22.1",
-    "sphinx==6.0",
-    "furo==2024.5.6",
-]
-
 [tool.uv]
 dev-dependencies = [
-    "poethepoet>=0.31.1",
+    "pytest>=8.0.0",
-    "pytest==8.0.0",
+    "pytest-mock>=3.14.0",
-    "pytest-mock==3.14.0",
+    "pytest-asyncio>=0.25.0",
     "pylint>=3.2.5",
 ]
 

scrapegraphai/builders/graph_builder.py

@@ -4,7 +4,6 @@ GraphBuilder Module
 from langchain_core.prompts import ChatPromptTemplate
 from langchain.chains import create_extraction_chain
 from langchain_community.chat_models import ErnieBotChat
-from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain_openai import ChatOpenAI
 from ..helpers import nodes_metadata, graph_schema
 
@@ -70,6 +69,10 @@ class GraphBuilder:
         if "gpt-" in llm_params["model"]:
             return ChatOpenAI(llm_params)
         elif "gemini" in llm_params["model"]:
+            try:
+                from langchain_google_genai import ChatGoogleGenerativeAI
+            except ImportError:
+                raise ImportError("langchain_google_genai is not installed. Please install it using 'pip install langchain-google-genai'.")
             return ChatGoogleGenerativeAI(llm_params)
         elif "ernie" in llm_params["model"]:
             return ErnieBotChat(llm_params)

scrapegraphai/graphs/abstract_graph.py

@@ -234,7 +234,7 @@ class AbstractGraph(ABC):
                         from langchain_together import ChatTogether
                     except ImportError:
                         raise ImportError("""The langchain_together module is not installed. 
-                                          Please install it using `pip install scrapegraphai[other-language-models]`.""")
+                                          Please install it using `pip install langchain-together`.""")
                     return ChatTogether(**llm_params)
 
                 elif model_provider == "nvidia":
@@ -242,7 +242,7 @@ class AbstractGraph(ABC):
                         from langchain_nvidia_ai_endpoints import ChatNVIDIA
                     except ImportError:
                         raise ImportError("""The langchain_nvidia_ai_endpoints module is not installed. 
-                                          Please install it using `pip install scrapegraphai[other-language-models]`.""")
+                                          Please install it using `pip install langchain-nvidia-ai-endpoints`.""")
                     return ChatNVIDIA(**llm_params)
 
         except Exception as e:

scrapegraphai/graphs/smart_scraper_graph.py

@@ -3,8 +3,6 @@ SmartScraperGraph Module
 """
 from typing import Optional
 from pydantic import BaseModel
-from scrapegraph_py import Client
-from scrapegraph_py.logger import sgai_logger
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from ..nodes import (
@@ -67,6 +65,11 @@ class SmartScraperGraph(AbstractGraph):
             BaseGraph: A graph instance representing the web scraping workflow.
         """
         if self.llm_model == "scrapegraphai/smart-scraper":
+            try:
+                from scrapegraph_py import Client
+                from scrapegraph_py.logger import sgai_logger
+            except ImportError:
+                raise ImportError("scrapegraph_py is not installed. Please install it using 'pip install scrapegraph-py'.")
             
             sgai_logger.set_logging(level="INFO")
 

scrapegraphai/nodes/fetch_node.py

@@ -10,7 +10,6 @@ from langchain_core.documents import Document
 from ..utils.cleanup_html import cleanup_html
 from ..docloaders import ChromiumLoader
 from ..utils.convert_to_md import convert_to_md
-from ..utils.logging import get_logger
 from .base_node import BaseNode
 
 class FetchNode(BaseNode):
@@ -79,24 +78,6 @@ class FetchNode(BaseNode):
             None if node_config is None else node_config.get("storage_state", None)
         )
 
-    def is_valid_url(self, source: str) -> bool:
-        """
-        Validates if the source string is a valid URL using regex.
-
-        Parameters:
-        source (str): The URL string to validate
-
-        Raises:
-        ValueError: If the URL is invalid
-        """
-        import re
-
-        url_pattern = r"^https?://[^\s/$.?#].[^\s]*$"
-        if not bool(re.match(url_pattern, source)):
-            raise ValueError(
-                f"Invalid URL format: {source}. URL must start with http(s):// and contain a valid domain."
-            )
-        return True
 
     def execute(self, state):
         """
@@ -129,12 +110,9 @@ class FetchNode(BaseNode):
         elif self.input == "pdf_dir":
             return state
 
-        # For web sources, validate URL before proceeding
         try:
-            if self.is_valid_url(source):
+            return self.handle_web_source(state, source)
-                return self.handle_web_source(state, source)
         except ValueError as e:
-            # Re-raise the exception from is_valid_url
             raise
 
         return self.handle_local_source(state, source)

scrapegraphai/nodes/rag_node.py

@@ -3,8 +3,6 @@ RAGNode Module
 """
 from typing import List, Optional
 from .base_node import BaseNode
-from qdrant_client import QdrantClient
-from qdrant_client.models import PointStruct, VectorParams, Distance
 
 class RAGNode(BaseNode):
     """
@@ -42,6 +40,14 @@ class RAGNode(BaseNode):
     def execute(self, state: dict) -> dict:
         self.logger.info(f"--- Executing {self.node_name} Node ---")
         
+        try:
+            import qdrant_client
+        except ImportError:
+            raise ImportError("qdrant_client is not installed. Please install it using 'pip install qdrant-client'.")
+        
+        from qdrant_client import QdrantClient
+        from qdrant_client.models import PointStruct, VectorParams, Distance
+
         if self.node_config.get("client_type") in ["memory", None]:
             client = QdrantClient(":memory:")
         elif self.node_config.get("client_type") == "local_db":

scrapegraphai/utils/screenshot_scraping/text_detection.py

@@ -22,7 +22,7 @@ def detect_text(image, languages: list = ["en"]):
         from surya.model.recognition.model import load_model as load_rec_model
         from surya.model.recognition.processor import load_processor as load_rec_processor
     except:
-        raise ImportError("The dependencies for screenshot scraping are not installed. Please install them using `pip install scrapegraphai[screenshot_scraper]`.")
+        raise ImportError("The dependencies for OCR are not installed. Please install them using `pip install scrapegraphai[ocr]`.")
 
 
     langs = languages

scrapegraphai/utils/tokenizers/tokenizer_mistral.py

@@ -1,10 +1,6 @@
 """
 Tokenization utilities for Mistral models
 """
-from mistral_common.protocol.instruct.messages import UserMessage
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
-from mistral_common.protocol.instruct.tool_calls import Function, Tool
-from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
 from langchain_core.language_models.chat_models import BaseChatModel
 from ..logging import get_logger
 
@@ -31,6 +27,13 @@ def num_tokens_mistral(text: str, llm_model:BaseChatModel) -> int:
         raise NotImplementedError(f"The model provider you are using ('{llm_model}') "
             "does not give us a model name so we cannot identify which encoding to use")
 
+    try:
+        from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+        from mistral_common.protocol.instruct.messages import UserMessage
+        from mistral_common.protocol.instruct.request import ChatCompletionRequest
+    except ImportError:
+        raise ImportError("mistral_common is not installed. Please install it using 'pip install mistral-common'.")
+
     tokenizer = MistralTokenizer.from_model(model)
 
     tokenized = tokenizer.encode_chat_completion(

tests/graphs/abstract_graph_test.py

@@ -11,7 +11,6 @@ from scrapegraphai.nodes import (
 from scrapegraphai.models import OneApi, DeepSeek
 from langchain_openai import ChatOpenAI, AzureChatOpenAI
 from langchain_ollama import ChatOllama
-from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain_aws import ChatBedrock
 
 
@@ -68,7 +67,6 @@ class TestAbstractGraph:
             "api_version": "no version",
             "azure_endpoint": "https://www.example.com/"},
             AzureChatOpenAI),
-        ({"model": "google_genai/gemini-pro", "google_api_key": "google-key-test"}, ChatGoogleGenerativeAI),
         ({"model": "ollama/llama2"}, ChatOllama),
         ({"model": "oneapi/qwen-turbo", "api_key": "oneapi-api-key"}, OneApi),
         ({"model": "deepseek/deepseek-coder", "api_key": "deepseek-api-key"}, DeepSeek),
@@ -86,7 +84,6 @@ class TestAbstractGraph:
     @pytest.mark.parametrize("llm_config, expected_model", [
         ({"model": "openai/gpt-3.5-turbo", "openai_api_key": "sk-randomtest001", "rate_limit": {"requests_per_second": 1}}, ChatOpenAI),
         ({"model": "azure_openai/gpt-3.5-turbo", "api_key": "random-api-key", "api_version": "no version", "azure_endpoint": "https://www.example.com/", "rate_limit": {"requests_per_second": 1}}, AzureChatOpenAI),
-        ({"model": "google_genai/gemini-pro", "google_api_key": "google-key-test", "rate_limit": {"requests_per_second": 1}}, ChatGoogleGenerativeAI),
         ({"model": "ollama/llama2", "rate_limit": {"requests_per_second": 1}}, ChatOllama),
         ({"model": "oneapi/qwen-turbo", "api_key": "oneapi-api-key", "rate_limit": {"requests_per_second": 1}}, OneApi),
         ({"model": "deepseek/deepseek-coder", "api_key": "deepseek-api-key", "rate_limit": {"requests_per_second": 1}}, DeepSeek),

tests/graphs/smart_scraper_ollama_test.py

@@ -3,7 +3,6 @@ Module for testing th smart scraper class
 """
 import pytest
 from scrapegraphai.graphs import SmartScraperGraph
-from transformers import GPT2TokenizerFast
 
 
 @pytest.fixture
@@ -52,10 +51,3 @@ def test_get_execution_info(graph_config: dict):
 
     assert graph_exec_info is not None
 
-
-def test_gpt2_tokenizer_loading():
-    """
-    Test loading of GPT2TokenizerFast
-    """
-    tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
-    assert tokenizer is not None