CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

chore: chromium browser asnc handling

Browse Source
This commit is contained in:
PeriniM 2025-01-06 04:08:15 +01:00
parent a82af04afe
commit 5be7c497cd
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapegraphai/docloaders/chromium.py
View File

@ -325,14 +325,13 @@ class ChromiumLoader(BaseLoader):
await page.wait_for_load_state(self.load_state) await page.wait_for_load_state(self.load_state)
results = await page.content() results = await page.content()
logger.info("Content scraped") logger.info("Content scraped")
await browser.close()
return results return results
except (aiohttp.ClientError, asyncio.TimeoutError, Exception) as e: except (aiohttp.ClientError, asyncio.TimeoutError, Exception) as e:
attempt += 1 attempt += 1
logger.error(f"Attempt {attempt} failed: {e}") logger.error(f"Attempt {attempt} failed: {e}")
if attempt == self.retry_limit: if attempt == self.retry_limit:
raise RuntimeError(f"Failed to scrape after {self.retry_limit} attempts: {str(e)}") raise RuntimeError(f"Failed to scrape after {self.retry_limit} attempts: {str(e)}")
finally:
await browser.close()
async def ascrape_with_js_support(self, url: str, browser_name: str = "chromium") -> str: async def ascrape_with_js_support(self, url: str, browser_name: str = "chromium") -> str:
""" """