CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: update chromium

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-11-04 09:21:29 +01:00
parent 950e859b1b
commit 38c6dd2aa1
1 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
scrapegraphai/docloaders/chromium.py
View File

@ -1,6 +1,3 @@
"""
chromiumloader module
"""
import asyncio import asyncio
from typing import Any, AsyncIterator, Iterator, List, Optional from typing import Any, AsyncIterator, Iterator, List, Optional
from langchain_community.document_loaders.base import BaseLoader from langchain_community.document_loaders.base import BaseLoader
@ -12,15 +9,16 @@ from ..utils import Proxy, dynamic_import, get_logger, parse_or_search_proxy
logger = get_logger("web-loader") logger = get_logger("web-loader")
class ChromiumLoader(BaseLoader): class ChromiumLoader(BaseLoader):
"""scrapes HTML pages from URLs using a (headless) instance of the """Scrapes HTML pages from URLs using a (headless) instance of the
Chromium web driver with proxy protection Chromium web driver with proxy protection.
Attributes: Attributes:
backend: The web driver backend library; defaults to 'playwright'. backend: The web driver backend library; defaults to 'playwright'.
browser_config: A dictionary containing additional browser kwargs. browser_config: A dictionary containing additional browser kwargs.
headless: whether to run browser in headless mode. headless: Whether to run browser in headless mode.
proxy: A dictionary containing proxy settings; None disables protection. proxy: A dictionary containing proxy settings; None disables protection.
urls: A list of URLs to scrape content from. urls: A list of URLs to scrape content from.
requires_js_support: Flag to determine if JS rendering is required.
""" """
RETRY_LIMIT = 3 RETRY_LIMIT = 3
@ -34,15 +32,17 @@ class ChromiumLoader(BaseLoader):
headless: bool = True, headless: bool = True,
proxy: Optional[Proxy] = None, proxy: Optional[Proxy] = None,
load_state: str = "domcontentloaded", load_state: str = "domcontentloaded",
requires_js_support: bool = False,
**kwargs: Any, **kwargs: Any,
): ):
"""Initialize the loader with a list of URL paths. """Initialize the loader with a list of URL paths.
Args: Args:
backend: The web driver backend library; defaults to 'playwright'. backend: The web driver backend library; defaults to 'playwright'.
headless: whether to run browser in headless mode. headless: Whether to run browser in headless mode.
proxy: A dictionary containing proxy information; None disables protection. proxy: A dictionary containing proxy information; None disables protection.
urls: A list of URLs to scrape content from. urls: A list of URLs to scrape content from.
requires_js_support: Whether to use JS rendering for scraping.
kwargs: A dictionary containing additional browser kwargs. kwargs: A dictionary containing additional browser kwargs.
Raises: Raises:
@ -61,6 +61,7 @@ class ChromiumLoader(BaseLoader):
self.proxy = parse_or_search_proxy(proxy) if proxy else None self.proxy = parse_or_search_proxy(proxy) if proxy else None
self.urls = urls self.urls = urls
self.load_state = load_state self.load_state = load_state
self.requires_js_support = requires_js_support
async def ascrape_undetected_chromedriver(self, url: str) -> str: async def ascrape_undetected_chromedriver(self, url: str) -> str:
""" """
@ -186,7 +187,9 @@ class ChromiumLoader(BaseLoader):
Yields: Yields:
Document: The scraped content encapsulated within a Document object. Document: The scraped content encapsulated within a Document object.
""" """
scraping_fn = getattr(self, f"ascrape_{self.backend}") scraping_fn = (
self.ascrape_with_js_support if self.requires_js_support else getattr(self, f"ascrape_{self.backend}")
)
for url in self.urls: for url in self.urls:
html_content = asyncio.run(scraping_fn(url)) html_content = asyncio.run(scraping_fn(url))
@ -206,7 +209,9 @@ class ChromiumLoader(BaseLoader):
Document: A Document object containing the scraped content, along with its Document: A Document object containing the scraped content, along with its
source URL as metadata. source URL as metadata.
""" """
scraping_fn = getattr(self, f"ascrape_{self.backend}") scraping_fn = (
self.ascrape_with_js_support if self.requires_js_support else getattr(self, f"ascrape_{self.backend}")
)
tasks = [scraping_fn(url) for url in self.urls] tasks = [scraping_fn(url) for url in self.urls]
results = await asyncio.gather(*tasks) results = await asyncio.gather(*tasks)