diff --git a/scrapegraphai/nodes/fetch_html_node.py b/scrapegraphai/nodes/fetch_html_node.py
index 7c949749..e68a5f26 100644
--- a/scrapegraphai/nodes/fetch_html_node.py
+++ b/scrapegraphai/nodes/fetch_html_node.py
@@ -1,12 +1,26 @@
"""
Module for fetching the HTML node
"""
+from typing import Any
from langchain_community.document_loaders import AsyncHtmlLoader
+from langchain_core.documents import Document
from .base_node import BaseNode
-
from ..utils.remover import remover
+def _build_metadata(soup: Any, url: str) -> dict:
+ """Build metadata from BeautifulSoup output."""
+ metadata = {"source": url}
+ if title := soup.find("title"):
+ metadata["title"] = title.get_text()
+ if description := soup.find("meta", attrs={"name": "description"}):
+ metadata["description"] = description.get(
+ "content", "No description found.")
+ if html := soup.find("html"):
+ metadata["language"] = html.get("lang", "No language found.")
+ return metadata
+
+
class FetchHTMLNode(BaseNode):
"""
A node responsible for fetching the HTML content of a specified URL and updating
@@ -67,9 +81,10 @@ class FetchHTMLNode(BaseNode):
loader = AsyncHtmlLoader(url)
document = loader.load()
+ metadata = document[0].metadata
+ document = remover(str(document[0]))
- document = remover(document, only_body=True)
-
- state["document"] = document
+ state["document"] = [
+ Document(page_content=document, metadata=metadata)]
return state