CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-23 21:00:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add integration on the fetch node

Browse Source
This commit is contained in:
VinciGit00 2024-04-16 12:19:23 +02:00
parent 9661c77ebe
commit 4233430518
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapegraphai/nodes/fetch_node.py
View File

@ -6,6 +6,7 @@ from typing import List
from langchain_community.document_loaders import AsyncHtmlLoader
from langchain_core.documents import Document
from .base_node import BaseNode
from ..utils.remover import remover
class FetchNode(BaseNode):
@ -71,7 +72,7 @@ class FetchNode(BaseNode):
# if it is a local directory
if not source.startswith("http"):
document = [Document(page_content=source, metadata={
document = [Document(page_content=remover(source), metadata={
"source": "local_dir"
})]
@ -79,6 +80,5 @@ class FetchNode(BaseNode):
else:
loader = AsyncHtmlLoader(source)
document = loader.load()
state.update({self.output[0]: document})
return state