CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: selenium import in ChromiumLoader

Browse Source
This commit is contained in:
PeriniM 2025-01-06 00:44:16 +01:00
parent 927c99b2e8
commit e374e055d6
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapegraphai/docloaders/chromium.py
View File

@ -4,8 +4,6 @@ from langchain_community.document_loaders.base import BaseLoader
from langchain_core.documents import Document from langchain_core.documents import Document
import aiohttp import aiohttp
import async_timeout import async_timeout
from selenium import webdriver
from selenium.webdriver.chrome.options import Options as ChromeOptions
from typing import Union from typing import Union
from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy
@ -90,7 +88,10 @@ class ChromiumLoader(BaseLoader):
Returns: Returns:
str: The scraped HTML content or an error message if an exception occurs. str: The scraped HTML content or an error message if an exception occurs.
""" """
import undetected_chromedriver as uc try:
import undetected_chromedriver as uc
except ImportError:
raise ImportError("undetected_chromedriver is required for ChromiumLoader. Please install it with `pip install undetected-chromedriver`.")
logger.info(f"Starting scraping with {self.backend}...") logger.info(f"Starting scraping with {self.backend}...")
results = "" results = ""
@ -102,6 +103,7 @@ class ChromiumLoader(BaseLoader):
# Handling browser selection # Handling browser selection
if self.backend == "selenium": if self.backend == "selenium":
if self.browser_name == "chromium": if self.browser_name == "chromium":
from selenium.webdriver.chrome.options import Options as ChromeOptions
options = ChromeOptions() options = ChromeOptions()
options.headless = self.headless options.headless = self.headless
# Initialize undetected chromedriver for Selenium # Initialize undetected chromedriver for Selenium
@ -112,6 +114,7 @@ class ChromiumLoader(BaseLoader):
break break
elif self.browser_name == "firefox": elif self.browser_name == "firefox":
from selenium.webdriver.firefox.options import Options as FirefoxOptions from selenium.webdriver.firefox.options import Options as FirefoxOptions
from selenium import webdriver
options = FirefoxOptions() options = FirefoxOptions()
options.headless = self.headless options.headless = self.headless
# Initialize undetected Firefox driver (if required) # Initialize undetected Firefox driver (if required)