From 21147c46a53e943dd5f297e6c7c3433edadfbc27 Mon Sep 17 00:00:00 2001 From: PeriniM Date: Mon, 6 Jan 2025 12:17:30 +0100 Subject: [PATCH] feat: add timeout and retry_limit in loader_kwargs (#865 #831) --- scrapegraphai/docloaders/chromium.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapegraphai/docloaders/chromium.py b/scrapegraphai/docloaders/chromium.py index 5e9aa75e..a8f6a0aa 100644 --- a/scrapegraphai/docloaders/chromium.py +++ b/scrapegraphai/docloaders/chromium.py @@ -35,7 +35,7 @@ class ChromiumLoader(BaseLoader): storage_state: Optional[str] = None, browser_name: str = "chromium", #default chromium retry_limit: int = 1, - timeout: int = 10, + timeout: int = 60, **kwargs: Any, ): """Initialize the loader with a list of URL paths. @@ -69,8 +69,8 @@ class ChromiumLoader(BaseLoader): self.requires_js_support = requires_js_support self.storage_state = storage_state self.browser_name = browser_name - self.retry_limit = retry_limit - self.timeout = timeout + self.retry_limit = kwargs.get("retry_limit", retry_limit) + self.timeout = kwargs.get("timeout", timeout) async def scrape(self, url:str) -> str: if self.backend == "playwright":