CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: add timeout and retry_limit in loader_kwargs (#865 #831)

Browse Source
This commit is contained in:
PeriniM 2025-01-06 12:17:30 +01:00
parent 2a15581865
commit 21147c46a5
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapegraphai/docloaders/chromium.py
View File

@ -35,7 +35,7 @@ class ChromiumLoader(BaseLoader):
storage_state: Optional[str] = None, storage_state: Optional[str] = None,
browser_name: str = "chromium", #default chromium browser_name: str = "chromium", #default chromium
retry_limit: int = 1, retry_limit: int = 1,
timeout: int = 10, timeout: int = 60,
**kwargs: Any, **kwargs: Any,
): ):
"""Initialize the loader with a list of URL paths. """Initialize the loader with a list of URL paths.
@ -69,8 +69,8 @@ class ChromiumLoader(BaseLoader):
self.requires_js_support = requires_js_support self.requires_js_support = requires_js_support
self.storage_state = storage_state self.storage_state = storage_state
self.browser_name = browser_name self.browser_name = browser_name
self.retry_limit = retry_limit self.retry_limit = kwargs.get("retry_limit", retry_limit)
self.timeout = timeout self.timeout = kwargs.get("timeout", timeout)
async def scrape(self, url:str) -> str: async def scrape(self, url:str) -> str:
if self.backend == "playwright": if self.backend == "playwright":