From f755d56bb1e2406668c5114e649953adbbff6748 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Wed, 2 Oct 2024 12:28:48 +0200 Subject: [PATCH] updated parse node --- scrapegraphai/graphs/depth_search_graph.py | 18 +++- scrapegraphai/nodes/__init__.py | 1 + scrapegraphai/nodes/parse_node_depth_k.py | 120 +++------------------ 3 files changed, 33 insertions(+), 106 deletions(-) diff --git a/scrapegraphai/graphs/depth_search_graph.py b/scrapegraphai/graphs/depth_search_graph.py index fa6294a0..6ad3b245 100644 --- a/scrapegraphai/graphs/depth_search_graph.py +++ b/scrapegraphai/graphs/depth_search_graph.py @@ -8,7 +8,8 @@ from .base_graph import BaseGraph from .abstract_graph import AbstractGraph from ..utils.save_code_to_file import save_code_to_file from ..nodes import ( - FetchNodeLevelK + FetchNodeLevelK, + ParseNodeDepthK ) class DepthSearchGraph(AbstractGraph): @@ -71,12 +72,23 @@ class DepthSearchGraph(AbstractGraph): "only_inside_links": self.config.get("only_inside_links", False) } ) + + parse_node = ParseNodeDepthK( + input="docs", + output=["docs"], + node_config={ + "verbose": self.config.get("verbose", False) + } + ) return BaseGraph( nodes=[ - fetch_node + fetch_node, + parse_node + ], + edges=[ + (fetch_node, parse_node), ], - edges=[], entry_point=fetch_node, graph_name=self.__class__.__name__ ) diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py index 7b994746..edb195a5 100644 --- a/scrapegraphai/nodes/__init__.py +++ b/scrapegraphai/nodes/__init__.py @@ -31,3 +31,4 @@ from .reasoning_node import ReasoningNode from .fetch_node_level_k import FetchNodeLevelK from .generate_answer_node_k_level import GenerateAnswerNodeKLevel from .description_node import DescriptionNode +from .parse_node_depth_k import ParseNodeDepthK diff --git a/scrapegraphai/nodes/parse_node_depth_k.py b/scrapegraphai/nodes/parse_node_depth_k.py index fd2f3810..30afa23c 100644 --- a/scrapegraphai/nodes/parse_node_depth_k.py +++ b/scrapegraphai/nodes/parse_node_depth_k.py @@ -1,19 +1,14 @@ """ -ParseNode Module +ParseNodeDepthK Module """ import re from typing import List, Optional, Tuple -from urllib.parse import urljoin -from langchain_community.document_transformers import Html2TextTransformer -from langchain_core.documents import Document from .base_node import BaseNode -from ..utils.split_text_into_chunks import split_text_into_chunks -from ..helpers import default_filters +from ..utils.convert_to_md import convert_to_md -class ParseNode(BaseNode): +class ParseNodeDepthK(BaseNode): """ - A node responsible for parsing HTML content from a document. - The parsed content is split into chunks for further processing. + A node responsible for parsing HTML content from a series of documents. This node enhances the scraping workflow by allowing for targeted extraction of content, thereby optimizing the processing of large HTML documents. @@ -33,26 +28,17 @@ class ParseNode(BaseNode): input: str, output: List[str], node_config: Optional[dict] = None, - node_name: str = "ParseNode", + node_name: str = "ParseNodeDepthK", ): super().__init__(node_name, "node", input, output, 1, node_config) self.verbose = ( False if node_config is None else node_config.get("verbose", False) ) - self.parse_html = ( - True if node_config is None else node_config.get("parse_html", True) - ) - self.parse_urls = ( - False if node_config is None else node_config.get("parse_urls", False) - ) - - self.llm_model = node_config.get("llm_model") - self.chunk_size = node_config.get("chunk_size") def execute(self, state: dict) -> dict: """ - Executes the node's logic to parse the HTML document content and split it into chunks. + Executes the node's logic to parse the HTML documents content. Args: state (dict): The current state of the graph. The input keys will be used to fetch the @@ -67,90 +53,18 @@ class ParseNode(BaseNode): """ self.logger.info(f"--- Executing {self.node_name} Node ---") - + + # Interpret input keys based on the provided input expression input_keys = self.get_input_keys(state) - + # Fetching data from the state based on the input keys input_data = [state[key] for key in input_keys] - docs_transformed = input_data[0] - source = input_data[1] if self.parse_urls else None - - if self.parse_html: - docs_transformed = Html2TextTransformer(ignore_links=False).transform_documents(input_data[0]) - docs_transformed = docs_transformed[0] - - link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) - - chunks = split_text_into_chunks(text=docs_transformed.page_content, - chunk_size=self.chunk_size-250, model=self.llm_model) - else: - docs_transformed = docs_transformed[0] - - link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) - - chunk_size = self.chunk_size - chunk_size = min(chunk_size - 500, int(chunk_size * 0.75)) - - if isinstance(docs_transformed, Document): - chunks = split_text_into_chunks(text=docs_transformed.page_content, - chunk_size=chunk_size, - model=self.llm_model) - else: - chunks = split_text_into_chunks(text=docs_transformed, - chunk_size=chunk_size, - model=self.llm_model) - - state.update({self.output[0]: chunks}) - if self.parse_urls: - state.update({self.output[1]: link_urls}) - state.update({self.output[2]: img_urls}) + documents = input_data[0] + + for doc in documents: + document_md = convert_to_md(doc["document"]) + doc["document_md"] = document_md + + state.update({self.output[0]: documents}) + return state - - def _extract_urls(self, text: str, source: str) -> Tuple[List[str], List[str]]: - """ - Extracts URLs from the given text. - - Args: - text (str): The text to extract URLs from. - - Returns: - Tuple[List[str], List[str]]: A tuple containing the extracted link URLs and image URLs. - """ - if not self.parse_urls: - return [], [] - - image_extensions = default_filters.filter_dict["img_exts"] - image_extension_seq = '|'.join(image_extensions).replace('.','') - url_pattern = re.compile(r'(https?://[^\s]+|\S+\.(?:' + image_extension_seq + '))') - - all_urls = url_pattern.findall(text) - all_urls = self._clean_urls(all_urls) - - if not source.startswith("http"): - all_urls = [url for url in all_urls if url.startswith("http")] - else: - all_urls = [urljoin(source, url) for url in all_urls] - - images = [url for url in all_urls if any(url.endswith(ext) for ext in image_extensions)] - links = [url for url in all_urls if url not in images] - - return links, images - - def _clean_urls(self, urls: List[str]) -> List[str]: - """ - Cleans the URLs extracted from the text. - - Args: - urls (List[str]): The list of URLs to clean. - - Returns: - List[str]: The cleaned URLs. - """ - cleaned_urls = [] - for url in urls: - url = re.sub(r'.*?\]\(', '', url) - url = url.rstrip(').') - - cleaned_urls.append(url) - - return cleaned_urls