diff --git a/scrapegraphai/nodes/fetch_html_node.py b/scrapegraphai/nodes/fetch_html_node.py index 7c949749..e68a5f26 100644 --- a/scrapegraphai/nodes/fetch_html_node.py +++ b/scrapegraphai/nodes/fetch_html_node.py @@ -1,12 +1,26 @@ """ Module for fetching the HTML node """ +from typing import Any from langchain_community.document_loaders import AsyncHtmlLoader +from langchain_core.documents import Document from .base_node import BaseNode - from ..utils.remover import remover +def _build_metadata(soup: Any, url: str) -> dict: + """Build metadata from BeautifulSoup output.""" + metadata = {"source": url} + if title := soup.find("title"): + metadata["title"] = title.get_text() + if description := soup.find("meta", attrs={"name": "description"}): + metadata["description"] = description.get( + "content", "No description found.") + if html := soup.find("html"): + metadata["language"] = html.get("lang", "No language found.") + return metadata + + class FetchHTMLNode(BaseNode): """ A node responsible for fetching the HTML content of a specified URL and updating @@ -67,9 +81,10 @@ class FetchHTMLNode(BaseNode): loader = AsyncHtmlLoader(url) document = loader.load() + metadata = document[0].metadata + document = remover(str(document[0])) - document = remover(document, only_body=True) - - state["document"] = document + state["document"] = [ + Document(page_content=document, metadata=metadata)] return state