CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

updated parse node

Browse Source
This commit is contained in:
Matteo Vedovati 2024-10-02 12:28:48 +02:00
parent 2bdb01b07a
commit f755d56bb1
3 changed files with 33 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapegraphai/graphs/depth_search_graph.py
View File

@ -8,7 +8,8 @@ from .base_graph import BaseGraph
from .abstract_graph import AbstractGraph
from ..utils.save_code_to_file import save_code_to_file
from ..nodes import (
FetchNodeLevelK
FetchNodeLevelK,
ParseNodeDepthK
)
class DepthSearchGraph(AbstractGraph):
@ -71,12 +72,23 @@ class DepthSearchGraph(AbstractGraph):
"only_inside_links": self.config.get("only_inside_links", False)
}
)
parse_node = ParseNodeDepthK(
input="docs",
output=["docs"],
node_config={
"verbose": self.config.get("verbose", False)
}
)
return BaseGraph(
nodes=[
fetch_node
fetch_node,
parse_node
],
edges=[
(fetch_node, parse_node),
],
edges=[],
entry_point=fetch_node,
graph_name=self.__class__.__name__
)

1
scrapegraphai/nodes/__init__.py
View File

@ -31,3 +31,4 @@ from .reasoning_node import ReasoningNode
from .fetch_node_level_k import FetchNodeLevelK
from .generate_answer_node_k_level import GenerateAnswerNodeKLevel
from .description_node import DescriptionNode
from .parse_node_depth_k import ParseNodeDepthK

120
scrapegraphai/nodes/parse_node_depth_k.py
View File

@ -1,19 +1,14 @@
"""
ParseNode Module
ParseNodeDepthK Module
"""
import re
from typing import List, Optional, Tuple
from urllib.parse import urljoin
from langchain_community.document_transformers import Html2TextTransformer
from langchain_core.documents import Document
from .base_node import BaseNode
from ..utils.split_text_into_chunks import split_text_into_chunks
from ..helpers import default_filters
from ..utils.convert_to_md import convert_to_md
class ParseNode(BaseNode):
class ParseNodeDepthK(BaseNode):
"""
A node responsible for parsing HTML content from a document.
The parsed content is split into chunks for further processing.
A node responsible for parsing HTML content from a series of documents.
This node enhances the scraping workflow by allowing for targeted extraction of
content, thereby optimizing the processing of large HTML documents.
@ -33,26 +28,17 @@ class ParseNode(BaseNode):
input: str,
output: List[str],
node_config: Optional[dict] = None,
node_name: str = "ParseNode",
node_name: str = "ParseNodeDepthK",
):
super().__init__(node_name, "node", input, output, 1, node_config)
self.verbose = (
False if node_config is None else node_config.get("verbose", False)
)
self.parse_html = (
True if node_config is None else node_config.get("parse_html", True)
)
self.parse_urls = (
False if node_config is None else node_config.get("parse_urls", False)
)
self.llm_model = node_config.get("llm_model")
self.chunk_size = node_config.get("chunk_size")
def execute(self, state: dict) -> dict:
"""
Executes the node's logic to parse the HTML document content and split it into chunks.
Executes the node's logic to parse the HTML documents content.
Args:
state (dict): The current state of the graph. The input keys will be used to fetch the
@ -67,90 +53,18 @@ class ParseNode(BaseNode):
"""
self.logger.info(f"--- Executing {self.node_name} Node ---")
# Interpret input keys based on the provided input expression
input_keys = self.get_input_keys(state)
# Fetching data from the state based on the input keys
input_data = [state[key] for key in input_keys]
docs_transformed = input_data[0]
source = input_data[1] if self.parse_urls else None
if self.parse_html:
docs_transformed = Html2TextTransformer(ignore_links=False).transform_documents(input_data[0])
docs_transformed = docs_transformed[0]
link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
chunks = split_text_into_chunks(text=docs_transformed.page_content,
chunk_size=self.chunk_size-250, model=self.llm_model)
else:
docs_transformed = docs_transformed[0]
link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
chunk_size = self.chunk_size
chunk_size = min(chunk_size - 500, int(chunk_size * 0.75))
if isinstance(docs_transformed, Document):
chunks = split_text_into_chunks(text=docs_transformed.page_content,
chunk_size=chunk_size,
model=self.llm_model)
else:
chunks = split_text_into_chunks(text=docs_transformed,
chunk_size=chunk_size,
model=self.llm_model)
state.update({self.output[0]: chunks})
if self.parse_urls:
state.update({self.output[1]: link_urls})
state.update({self.output[2]: img_urls})
documents = input_data[0]
for doc in documents:
document_md = convert_to_md(doc["document"])
doc["document_md"] = document_md
state.update({self.output[0]: documents})
return state
def _extract_urls(self, text: str, source: str) -> Tuple[List[str], List[str]]:
"""
Extracts URLs from the given text.
Args:
text (str): The text to extract URLs from.
Returns:
Tuple[List[str], List[str]]: A tuple containing the extracted link URLs and image URLs.
"""
if not self.parse_urls:
return [], []
image_extensions = default_filters.filter_dict["img_exts"]
image_extension_seq = '|'.join(image_extensions).replace('.','')
url_pattern = re.compile(r'(https?://[^\s]+|\S+\.(?:' + image_extension_seq + '))')
all_urls = url_pattern.findall(text)
all_urls = self._clean_urls(all_urls)
if not source.startswith("http"):
all_urls = [url for url in all_urls if url.startswith("http")]
else:
all_urls = [urljoin(source, url) for url in all_urls]
images = [url for url in all_urls if any(url.endswith(ext) for ext in image_extensions)]
links = [url for url in all_urls if url not in images]
return links, images
def _clean_urls(self, urls: List[str]) -> List[str]:
"""
Cleans the URLs extracted from the text.
Args:
urls (List[str]): The list of URLs to clean.
Returns:
List[str]: The cleaned URLs.
"""
cleaned_urls = []
for url in urls:
url = re.sub(r'.*?\]\(', '', url)
url = url.rstrip(').')
cleaned_urls.append(url)
return cleaned_urls