diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py index bfb825f6..579d34ea 100644 --- a/scrapegraphai/nodes/parse_node.py +++ b/scrapegraphai/nodes/parse_node.py @@ -3,14 +3,13 @@ ParseNode Module """ from typing import Tuple, List, Optional from urllib.parse import urljoin +import re from semchunk import chunk from langchain_community.document_transformers import Html2TextTransformer from langchain_core.documents import Document from .base_node import BaseNode from ..helpers import default_filters -import re - class ParseNode(BaseNode): """ A node responsible for parsing HTML content from a document. @@ -61,14 +60,12 @@ class ParseNode(BaseNode): """ cleaned_urls = [] for url in urls: - # Remove any leading 'thumbnail](' or similar patterns url = re.sub(r'.*?\]\(', '', url) - - # Remove any trailing parentheses or brackets + url = url.rstrip(').') - + cleaned_urls.append(url) - + return cleaned_urls def extract_urls(self, text: str, source: str) -> Tuple[List[str], List[str]]: @@ -81,26 +78,21 @@ class ParseNode(BaseNode): Returns: Tuple[List[str], List[str]]: A tuple containing the extracted link URLs and image URLs. """ - # Return empty lists if the URLs are not to be parsed if not self.parse_urls: return [], [] - - # Regular expression to find URLs (both links and images) + image_extensions = default_filters.filter_dict["img_exts"] image_extension_seq = '|'.join(image_extensions).replace('.','') url_pattern = re.compile(r'(https?://[^\s]+|\S+\.(?:' + image_extension_seq + '))') - # Find all URLs in the string all_urls = url_pattern.findall(text) all_urls = self._clean_urls(all_urls) if not source.startswith("http"): - # Remove any URLs that is not complete all_urls = [url for url in all_urls if url.startswith("http")] else: - # Add to local URLs the source URL all_urls = [urljoin(source, url) for url in all_urls] - + images = [url for url in all_urls if any(url.endswith(ext) for ext in image_extensions)] links = [url for url in all_urls if url not in images] @@ -136,7 +128,7 @@ class ParseNode(BaseNode): return token_count(text, self.llm_model.model_name) if self.parse_html: - docs_transformed = Html2TextTransformer().transform_documents(input_data[0]) + docs_transformed = Html2TextTransformer(ignore_links=False).transform_documents(input_data[0]) docs_transformed = docs_transformed[0] link_urls, img_urls = self.extract_urls(docs_transformed.page_content, source)