CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: fetch_node improved
Some checks failed
/ build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-09-07 09:33:57 +02:00
parent 8883bced7d
commit 167f97040f
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapegraphai/nodes/fetch_node.py
View File

@ -285,8 +285,14 @@ class FetchNode(BaseNode):
metadata={"source": source}) for content in data] metadata={"source": source}) for content in data]
elif self.scrape_do is not None: elif self.scrape_do is not None:
from ..docloaders.scrape_do import scrape_do_fetch from ..docloaders.scrape_do import scrape_do_fetch
data = scrape_do_fetch(self.scrape_do.get("api_key"), if self.scrape_do.get("use_proxy") is None or self.scrape_do.get("geoCode") is None or self.scrape_do.get("super_proxy") is None:
source) data = scrape_do_fetch(self.scrape_do.get("api_key"),
source)
else:
data = scrape_do_fetch(self.scrape_do.get("api_key"),
source, self.scrape_do.get("use_proxy"),
self.scrape_do.get("geoCode"),
self.scrape_do.get("super_proxy"))
document = [Document(page_content=data, document = [Document(page_content=data,
metadata={"source": source})] metadata={"source": source})]