From 89de5b6cba988421e3f12581707cdbc98a03e289 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:10:40 +0200 Subject: [PATCH 01/12] Stating anew --- scrapegraphai/nodes/description_node.py | 42 ----------------------- scrapegraphai/nodes/fetch_node_level_k.py | 42 ----------------------- 2 files changed, 84 deletions(-) delete mode 100644 scrapegraphai/nodes/description_node.py delete mode 100644 scrapegraphai/nodes/fetch_node_level_k.py diff --git a/scrapegraphai/nodes/description_node.py b/scrapegraphai/nodes/description_node.py deleted file mode 100644 index 49ab941f..00000000 --- a/scrapegraphai/nodes/description_node.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -DescriptionNode Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class DescriptionNode(BaseNode): - """ - A node responsible for compressing the input tokens and storing the document - in a vector database for retrieval. Relevant chunks are stored in the state. - - It allows scraping of big documents without exceeding the token limit of the language model. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "RAG", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py deleted file mode 100644 index 18a0d435..00000000 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -FetchNodelevelK Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class FetchNodelevelK(BaseNode): - """ - A node responsible for compressing the input tokens and storing the document - in a vector database for retrieval. Relevant chunks are stored in the state. - - It allows scraping of big documents without exceeding the token limit of the language model. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "RAG", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass From 336bf705ec6f8200987b9a10f1210d732a35c7b0 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:18:30 +0200 Subject: [PATCH 02/12] initial creation of FetchNodeLevelK and DescriptionNode --- scrapegraphai/nodes/description_node.py | 42 +++++++++++++++++++++++ scrapegraphai/nodes/fetch_node_level_K.py | 39 +++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 scrapegraphai/nodes/description_node.py create mode 100644 scrapegraphai/nodes/fetch_node_level_K.py diff --git a/scrapegraphai/nodes/description_node.py b/scrapegraphai/nodes/description_node.py new file mode 100644 index 00000000..200d7032 --- /dev/null +++ b/scrapegraphai/nodes/description_node.py @@ -0,0 +1,42 @@ +""" +DescriptionNode Module +""" +from typing import List, Optional +from .base_node import BaseNode + +class DescriptionNode(BaseNode): + """ + A node responsible for generating a description of a given document. This description is + generated using a language model and is used for retrieving the right documents. + + It allows scraping of big documents without exceeding the token limit of the language model. + + Attributes: + llm_model: An instance of a language model client, configured for generating answers. + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "Description", + ): + super().__init__(node_name, "node", input, output, 2, node_config) + + self.llm_model = node_config["llm_model"] + self.embedder_model = node_config.get("embedder_model", None) + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + self.cache_path = node_config.get("cache_path", False) + + def execute(self, state: dict) -> dict: + pass diff --git a/scrapegraphai/nodes/fetch_node_level_K.py b/scrapegraphai/nodes/fetch_node_level_K.py new file mode 100644 index 00000000..2fd3aa8b --- /dev/null +++ b/scrapegraphai/nodes/fetch_node_level_K.py @@ -0,0 +1,39 @@ +""" +FetchNodeLevelK Module +""" +from typing import List, Optional +from .base_node import BaseNode + +class FetchNodeLevelK(BaseNode): + """ + A node responsible for fetching all the pages at a certain level of hyperlink the graph. + + Attributes: + llm_model: An instance of a language model client, configured for generating answers. + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "FetchLevelK", + ): + super().__init__(node_name, "node", input, output, 2, node_config) + + self.llm_model = node_config["llm_model"] + self.embedder_model = node_config.get("embedder_model", None) + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + self.cache_path = node_config.get("cache_path", False) + + def execute(self, state: dict) -> dict: + pass From 7411ff061c9ea74ddcd043574da1d968f6abaf99 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:21:26 +0200 Subject: [PATCH 03/12] Revert "initial creation of FetchNodeLevelK and DescriptionNode" This reverts commit 336bf705ec6f8200987b9a10f1210d732a35c7b0. --- scrapegraphai/nodes/description_node.py | 42 ----------------------- scrapegraphai/nodes/fetch_node_level_K.py | 39 --------------------- 2 files changed, 81 deletions(-) delete mode 100644 scrapegraphai/nodes/description_node.py delete mode 100644 scrapegraphai/nodes/fetch_node_level_K.py diff --git a/scrapegraphai/nodes/description_node.py b/scrapegraphai/nodes/description_node.py deleted file mode 100644 index 200d7032..00000000 --- a/scrapegraphai/nodes/description_node.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -DescriptionNode Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class DescriptionNode(BaseNode): - """ - A node responsible for generating a description of a given document. This description is - generated using a language model and is used for retrieving the right documents. - - It allows scraping of big documents without exceeding the token limit of the language model. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "Description", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass diff --git a/scrapegraphai/nodes/fetch_node_level_K.py b/scrapegraphai/nodes/fetch_node_level_K.py deleted file mode 100644 index 2fd3aa8b..00000000 --- a/scrapegraphai/nodes/fetch_node_level_K.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -FetchNodeLevelK Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class FetchNodeLevelK(BaseNode): - """ - A node responsible for fetching all the pages at a certain level of hyperlink the graph. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "FetchLevelK", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass From 462b27bc1d7ac29d0f668fe478867a4b357cb656 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:21:33 +0200 Subject: [PATCH 04/12] Revert "Stating anew" This reverts commit 89de5b6cba988421e3f12581707cdbc98a03e289. --- scrapegraphai/nodes/description_node.py | 42 +++++++++++++++++++++++ scrapegraphai/nodes/fetch_node_level_k.py | 42 +++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 scrapegraphai/nodes/description_node.py create mode 100644 scrapegraphai/nodes/fetch_node_level_k.py diff --git a/scrapegraphai/nodes/description_node.py b/scrapegraphai/nodes/description_node.py new file mode 100644 index 00000000..49ab941f --- /dev/null +++ b/scrapegraphai/nodes/description_node.py @@ -0,0 +1,42 @@ +""" +DescriptionNode Module +""" +from typing import List, Optional +from .base_node import BaseNode + +class DescriptionNode(BaseNode): + """ + A node responsible for compressing the input tokens and storing the document + in a vector database for retrieval. Relevant chunks are stored in the state. + + It allows scraping of big documents without exceeding the token limit of the language model. + + Attributes: + llm_model: An instance of a language model client, configured for generating answers. + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "RAG", + ): + super().__init__(node_name, "node", input, output, 2, node_config) + + self.llm_model = node_config["llm_model"] + self.embedder_model = node_config.get("embedder_model", None) + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + self.cache_path = node_config.get("cache_path", False) + + def execute(self, state: dict) -> dict: + pass diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py new file mode 100644 index 00000000..18a0d435 --- /dev/null +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -0,0 +1,42 @@ +""" +FetchNodelevelK Module +""" +from typing import List, Optional +from .base_node import BaseNode + +class FetchNodelevelK(BaseNode): + """ + A node responsible for compressing the input tokens and storing the document + in a vector database for retrieval. Relevant chunks are stored in the state. + + It allows scraping of big documents without exceeding the token limit of the language model. + + Attributes: + llm_model: An instance of a language model client, configured for generating answers. + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "RAG", + ): + super().__init__(node_name, "node", input, output, 2, node_config) + + self.llm_model = node_config["llm_model"] + self.embedder_model = node_config.get("embedder_model", None) + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + self.cache_path = node_config.get("cache_path", False) + + def execute(self, state: dict) -> dict: + pass From 6915f3edfd3e18d9c3fdedb677decb14f30afb49 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:22:09 +0200 Subject: [PATCH 05/12] start form scratch --- scrapegraphai/nodes/fetch_node_level_k.py | 42 ----------------------- 1 file changed, 42 deletions(-) delete mode 100644 scrapegraphai/nodes/fetch_node_level_k.py diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py deleted file mode 100644 index 18a0d435..00000000 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -FetchNodelevelK Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class FetchNodelevelK(BaseNode): - """ - A node responsible for compressing the input tokens and storing the document - in a vector database for retrieval. Relevant chunks are stored in the state. - - It allows scraping of big documents without exceeding the token limit of the language model. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "RAG", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass From 57bf572ab4a243a6d79155218bcc0d9d00dc3753 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:23:11 +0200 Subject: [PATCH 06/12] initial code for fetch nodel level K --- scrapegraphai/nodes/fetch_node_level_k.py | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 scrapegraphai/nodes/fetch_node_level_k.py diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py new file mode 100644 index 00000000..2fd3aa8b --- /dev/null +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -0,0 +1,39 @@ +""" +FetchNodeLevelK Module +""" +from typing import List, Optional +from .base_node import BaseNode + +class FetchNodeLevelK(BaseNode): + """ + A node responsible for fetching all the pages at a certain level of hyperlink the graph. + + Attributes: + llm_model: An instance of a language model client, configured for generating answers. + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "FetchLevelK", + ): + super().__init__(node_name, "node", input, output, 2, node_config) + + self.llm_model = node_config["llm_model"] + self.embedder_model = node_config.get("embedder_model", None) + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + self.cache_path = node_config.get("cache_path", False) + + def execute(self, state: dict) -> dict: + pass From d80b792e1529af8d87bb4534b777693e09b62feb Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:42:26 +0200 Subject: [PATCH 07/12] fetching first level --- scrapegraphai/nodes/fetch_node_level_k.py | 80 +++++++++++++++++++- scrapegraphai/utils/1_manual.py | 92 +++++++++++++++++++++++ 2 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 scrapegraphai/utils/1_manual.py diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py index 2fd3aa8b..bbaafded 100644 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -3,10 +3,17 @@ FetchNodeLevelK Module """ from typing import List, Optional from .base_node import BaseNode +from ..docloaders import ChromiumLoader +from ..utils.cleanup_html import cleanup_html +from ..utils.convert_to_md import convert_to_md +from langchain_core.documents import Document class FetchNodeLevelK(BaseNode): """ - A node responsible for fetching all the pages at a certain level of hyperlink the graph. + A node responsible for fetching the HTML content of a specified URL and all its sub-links + recursively up to a certain level of hyperlink the graph. This content is then used to update + the graph's state. It uses ChromiumLoader to fetch the content from a web page asynchronously + (with proxy protection). Attributes: llm_model: An instance of a language model client, configured for generating answers. @@ -29,11 +36,80 @@ class FetchNodeLevelK(BaseNode): super().__init__(node_name, "node", input, output, 2, node_config) self.llm_model = node_config["llm_model"] + self.embedder_model = node_config.get("embedder_model", None) + self.verbose = ( False if node_config is None else node_config.get("verbose", False) ) + self.cache_path = node_config.get("cache_path", False) + + self.headless = ( + True if node_config is None else node_config.get("headless", True) + ) + + self.loader_kwargs = ( + {} if node_config is None else node_config.get("loader_kwargs", {}) + ) + + self.browser_base = ( + None if node_config is None else node_config.get("browser_base", None) + ) def execute(self, state: dict) -> dict: - pass + """ + Executes the node's logic to fetch the HTML content of a specified URL and all its sub-links + and update the graph's state with the content. + + Args: + state (dict): The current state of the graph. The input keys will be used + to fetch the correct data types from the state. + + Returns: + dict: The updated state with a new output key containing the fetched HTML content. + + Raises: + KeyError: If the input key is not found in the state, indicating that the + necessary information to perform the operation is missing. + """ + + self.logger.info(f"--- Executing {self.node_name} Node ---") + + # Interpret input keys based on the provided input expression + input_keys = self.get_input_keys(state) + # Fetching data from the state based on the input keys + input_data = [state[key] for key in input_keys] + + source = input_data[0] + + self.logger.info(f"--- (Fetching HTML from: {source}) ---") + + loader_kwargs = {} + + if self.node_config is not None: + loader_kwargs = self.node_config.get("loader_kwargs", {}) + + if self.browser_base is not None: + try: + from ..docloaders.browser_base import browser_base_fetch + except ImportError: + raise ImportError("""The browserbase module is not installed. + Please install it using `pip install browserbase`.""") + + data = browser_base_fetch(self.browser_base.get("api_key"), + self.browser_base.get("project_id"), [source]) + + document = [Document(page_content=content, + metadata={"source": source}) for content in data] + + else: + loader = ChromiumLoader([source], headless=self.headless, **loader_kwargs) + + document = loader.load() + + if not document or not document[0].page_content.strip(): + raise ValueError("""No HTML body content found in + the document fetched by ChromiumLoader.""") + + parsed_content = document[0].page_content \ No newline at end of file diff --git a/scrapegraphai/utils/1_manual.py b/scrapegraphai/utils/1_manual.py new file mode 100644 index 00000000..21703b7b --- /dev/null +++ b/scrapegraphai/utils/1_manual.py @@ -0,0 +1,92 @@ +import requests +import logging +import time +from urllib.parse import quote, urljoin +from typing import Optional +from bs4 import BeautifulSoup +from dotenv import load_dotenv +import os +import json +import markdownify + +load_dotenv() + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +def fetch_content(token: str, target_url: str, max_retries: int = 5, retry_delay: int = 3) -> Optional[str]: + encoded_url = quote(target_url) + url = f"http://api.scrape.do?url={encoded_url}&token={token}&render=true&waitUntil=networkidle0" + + for attempt in range(max_retries): + try: + response = requests.get(url) + if response.status_code == 200: + logging.info(f"Successfully fetched content from {target_url}") + return response.text + logging.warning(f"Failed with status {response.status_code}. Retrying in {retry_delay}s...") + except requests.RequestException as e: + logging.error(f"Error fetching {target_url}: {e}. Retrying in {retry_delay}s...") + time.sleep(retry_delay) + + logging.error(f"Failed to fetch {target_url} after {max_retries} attempts.") + return None + +def extract_links(html_content: str) -> list: + soup = BeautifulSoup(html_content, 'html.parser') + links = [link['href'] for link in soup.find_all('a', href=True)] + logging.info(f"Extracted {len(links)} links.") + return links + +def process_links(token: str, base_url: str, links: list, depth: int, current_depth: int = 1) -> dict: + content_dict = {} + for idx, link in enumerate(links, start=1): + full_link = link if link.startswith("http") else urljoin(base_url, link) + logging.info(f"Processing link {idx}: {full_link}") + link_content = fetch_content(token, full_link) + if link_content: + markdown_content = markdownify.markdownify(link_content, heading_style="ATX") + content_dict[full_link] = markdown_content + save_content_to_json(content_dict, idx) + + if current_depth < depth: + new_links = extract_links(link_content) + content_dict.update(process_links(token, full_link, new_links, depth, current_depth + 1)) + else: + logging.warning(f"Failed to fetch content for {full_link}") + return content_dict + +def save_content_to_json(content_dict: dict, idx: int): + if not os.path.exists("downloaded_pages"): + os.makedirs("downloaded_pages") + + file_name = f"scraped_content_{idx}.json" + file_path = os.path.join("downloaded_pages", file_name) + + with open(file_path, "w", encoding="utf-8") as json_file: + json.dump(content_dict, json_file, ensure_ascii=False, indent=4) + + logging.info(f"Content saved to {file_path}") + +if __name__ == "__main__": + token = os.getenv("TOKEN") + target_url = "https://www.wired.com" + depth = 2 + + if not token or not target_url: + logging.error("Please set the TOKEN and TARGET_URL environment variables.") + exit(1) + + html_content = fetch_content(token, target_url) + + if html_content: + links = extract_links(html_content) + logging.info("Links found:") + for link in links: + logging.info(link) + + content_dict = process_links(token, target_url, links, depth) + for link, content in content_dict.items(): + logging.info(f"Link: {link}") + logging.info(f"Content: {content[:500]}...") + else: + logging.error("Failed to fetch the content.") From 4cb621feab7e014cba13798c0dd7d4f42b9938db Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Wed, 2 Oct 2024 10:22:21 +0200 Subject: [PATCH 08/12] fetch node level k implementation --- examples/openai/fetch_multiple_links.py | 21 +++++ scrapegraphai/graphs/__init__.py | 1 + scrapegraphai/graphs/depth_search_graph.py | 96 ++++++++++++++++++++++ scrapegraphai/nodes/__init__.py | 2 +- scrapegraphai/nodes/fetch_node_level_k.py | 80 ++++++++++++++++-- 5 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 examples/openai/fetch_multiple_links.py create mode 100644 scrapegraphai/graphs/depth_search_graph.py diff --git a/examples/openai/fetch_multiple_links.py b/examples/openai/fetch_multiple_links.py new file mode 100644 index 00000000..53e246de --- /dev/null +++ b/examples/openai/fetch_multiple_links.py @@ -0,0 +1,21 @@ + +from scrapegraphai.graphs import DepthSearchGraph + +graph_config = { + "llm": { + "api_key":"YOUR_API_KEY", + "model": "openai/gpt-4o-mini", + }, + "verbose": True, + "headless": False, + "depth": 2, +} + +search_graph = DepthSearchGraph( + prompt="List me all the projects with their description", + source="https://perinim.github.io/projects/", + config=graph_config +) + +result = search_graph.run() +print(result) \ No newline at end of file diff --git a/scrapegraphai/graphs/__init__.py b/scrapegraphai/graphs/__init__.py index efd6bd7e..b5ffcc47 100644 --- a/scrapegraphai/graphs/__init__.py +++ b/scrapegraphai/graphs/__init__.py @@ -26,3 +26,4 @@ from .search_link_graph import SearchLinkGraph from .screenshot_scraper_graph import ScreenshotScraperGraph from .smart_scraper_multi_concat_graph import SmartScraperMultiConcatGraph from .code_generator_graph import CodeGeneratorGraph +from .depth_search_graph import DepthSearchGraph diff --git a/scrapegraphai/graphs/depth_search_graph.py b/scrapegraphai/graphs/depth_search_graph.py new file mode 100644 index 00000000..a96d96a7 --- /dev/null +++ b/scrapegraphai/graphs/depth_search_graph.py @@ -0,0 +1,96 @@ +""" +... Module +""" +from typing import Optional +import logging +from pydantic import BaseModel +from .base_graph import BaseGraph +from .abstract_graph import AbstractGraph +from ..utils.save_code_to_file import save_code_to_file +from ..nodes import ( + FetchNodeLevelK +) + +class DepthSearchGraph(AbstractGraph): + """ + CodeGeneratorGraph is a script generator pipeline that generates the function extract_data(html: str) -> dict() for + extracting the wanted information from a HTML page. The code generated is in Python and uses the library BeautifulSoup. + It requires a user prompt, a source URL, and an output schema. + + Attributes: + prompt (str): The prompt for the graph. + source (str): The source of the graph. + config (dict): Configuration parameters for the graph. + schema (BaseModel): The schema for the graph output. + llm_model: An instance of a language model client, configured for generating answers. + embedder_model: An instance of an embedding model client, + configured for generating embeddings. + verbose (bool): A flag indicating whether to show print statements during execution. + headless (bool): A flag indicating whether to run the graph in headless mode. + library (str): The library used for web scraping (beautiful soup). + + Args: + prompt (str): The prompt for the graph. + source (str): The source of the graph. + config (dict): Configuration parameters for the graph. + schema (BaseModel): The schema for the graph output. + + Example: + >>> code_gen = CodeGeneratorGraph( + ... "List me all the attractions in Chioggia.", + ... "https://en.wikipedia.org/wiki/Chioggia", + ... {"llm": {"model": "openai/gpt-3.5-turbo"}} + ... ) + >>> result = code_gen.run() + ) + """ + + def __init__(self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None): + + super().__init__(prompt, config, source, schema) + + self.input_key = "url" if source.startswith("http") else "local_dir" + + def _create_graph(self) -> BaseGraph: + """ + Creates the graph of nodes representing the workflow for web scraping. + + Returns: + BaseGraph: A graph instance representing the web scraping workflow. + """ + + fetch_node = FetchNodeLevelK( + input="url| local_dir", + output=["docs"], + node_config={ + "loader_kwargs": self.config.get("loader_kwargs", {}), + "force": self.config.get("force", False), + "cut": self.config.get("cut", True), + "browser_base": self.config.get("browser_base"), + "depth": self.config.get("depth", 1) + } + ) + + return BaseGraph( + nodes=[ + fetch_node + ], + edges=[], + entry_point=fetch_node, + graph_name=self.__class__.__name__ + ) + + def run(self) -> str: + """ + Executes the scraping process and returns the generated code. + + Returns: + str: The generated code. + """ + + inputs = {"user_prompt": self.prompt, self.input_key: self.source} + self.final_state, self.execution_info = self.graph.execute(inputs) + + docs = self.final_state.get("docs", "No docs") + + return docs \ No newline at end of file diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py index e5fafb87..7b994746 100644 --- a/scrapegraphai/nodes/__init__.py +++ b/scrapegraphai/nodes/__init__.py @@ -28,6 +28,6 @@ from .html_analyzer_node import HtmlAnalyzerNode from .generate_code_node import GenerateCodeNode from .search_node_with_context import SearchLinksWithContext from .reasoning_node import ReasoningNode -from .fetch_node_level_k import FetchNodelevelK +from .fetch_node_level_k import FetchNodeLevelK from .generate_answer_node_k_level import GenerateAnswerNodeKLevel from .description_node import DescriptionNode diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py index bbaafded..f9fd57a8 100644 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -7,6 +7,8 @@ from ..docloaders import ChromiumLoader from ..utils.cleanup_html import cleanup_html from ..utils.convert_to_md import convert_to_md from langchain_core.documents import Document +from bs4 import BeautifulSoup +from urllib.parse import quote, urljoin class FetchNodeLevelK(BaseNode): """ @@ -34,8 +36,6 @@ class FetchNodeLevelK(BaseNode): node_name: str = "FetchLevelK", ): super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] self.embedder_model = node_config.get("embedder_model", None) @@ -56,6 +56,16 @@ class FetchNodeLevelK(BaseNode): self.browser_base = ( None if node_config is None else node_config.get("browser_base", None) ) + + self.depth = ( + 1 if node_config is None else node_config.get("depth", 1) + ) + + self.only_inside_links = ( + False if node_config is None else node_config.get("only_inside_links", False) + ) + + self.min_input_len = 1 def execute(self, state: dict) -> dict: """ @@ -83,6 +93,8 @@ class FetchNodeLevelK(BaseNode): source = input_data[0] + documents = [{"source": source}] + self.logger.info(f"--- (Fetching HTML from: {source}) ---") loader_kwargs = {} @@ -90,6 +102,12 @@ class FetchNodeLevelK(BaseNode): if self.node_config is not None: loader_kwargs = self.node_config.get("loader_kwargs", {}) + for _ in range(self.depth): + documents = self.obtain_content(documents, loader_kwargs) + + return {self.output_keys[0]: documents} + + def fetch_content(self, source: str, loader_kwargs) -> Optional[str]: if self.browser_base is not None: try: from ..docloaders.browser_base import browser_base_fetch @@ -108,8 +126,58 @@ class FetchNodeLevelK(BaseNode): document = loader.load() - if not document or not document[0].page_content.strip(): - raise ValueError("""No HTML body content found in - the document fetched by ChromiumLoader.""") + return document + + def extract_links(self, html_content: str) -> list: + soup = BeautifulSoup(html_content, 'html.parser') + links = [link['href'] for link in soup.find_all('a', href=True)] + self.logger.info(f"Extracted {len(links)} links.") + return links + + def get_full_links(self, base_url: str, links: list) -> list: + full_links = [] + for link in links: + if self.only_inside_links and link.startswith("http"): + continue + full_link = link if link.startswith("http") else urljoin(base_url, link) + full_links.append(full_link) + return full_links + + def obtain_content(self, documents: List, loader_kwargs) -> List: + for doc in documents: + source = doc['source'] + if 'document' not in doc: + document = self.fetch_content(source, loader_kwargs) - parsed_content = document[0].page_content \ No newline at end of file + if not document or not document[0].page_content.strip(): + self.logger.warning(f"Failed to fetch content for {source}") + documents.remove(doc) + continue + + doc['document'] = document[0].page_content + + links = self.extract_links(doc['document']) + full_links = self.get_full_links(source, links) + + # Check if the links are already present in other documents + for link in full_links: + # Check if any document is from the same link + if not any(d.get('source', '') == link for d in documents): + # Add the document + documents.append({"source": link}) + + return documents + + def process_links(self, base_url: str, links: list, loader_kwargs, depth: int, current_depth: int = 1) -> dict: + content_dict = {} + for idx, link in enumerate(links, start=1): + full_link = link if link.startswith("http") else urljoin(base_url, link) + self.logger.info(f"Processing link {idx}: {full_link}") + link_content = self.fetch_content(full_link, loader_kwargs) + + if current_depth < depth: + new_links = self.extract_links(link_content) + content_dict.update(self.process_links(full_link, new_links, depth, current_depth + 1)) + else: + self.logger.warning(f"Failed to fetch content for {full_link}") + return content_dict \ No newline at end of file From ea3ae1fd6d2406a0b1b4c3337eab24cea44c9656 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Wed, 2 Oct 2024 11:01:23 +0200 Subject: [PATCH 09/12] fetch multiple links fix --- examples/openai/fetch_multiple_links.py | 1 + scrapegraphai/graphs/depth_search_graph.py | 3 ++- scrapegraphai/nodes/fetch_node_level_k.py | 12 +++++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/openai/fetch_multiple_links.py b/examples/openai/fetch_multiple_links.py index 53e246de..c9c07877 100644 --- a/examples/openai/fetch_multiple_links.py +++ b/examples/openai/fetch_multiple_links.py @@ -9,6 +9,7 @@ graph_config = { "verbose": True, "headless": False, "depth": 2, + "only_inside_links": True, } search_graph = DepthSearchGraph( diff --git a/scrapegraphai/graphs/depth_search_graph.py b/scrapegraphai/graphs/depth_search_graph.py index a96d96a7..fa6294a0 100644 --- a/scrapegraphai/graphs/depth_search_graph.py +++ b/scrapegraphai/graphs/depth_search_graph.py @@ -67,7 +67,8 @@ class DepthSearchGraph(AbstractGraph): "force": self.config.get("force", False), "cut": self.config.get("cut", True), "browser_base": self.config.get("browser_base"), - "depth": self.config.get("depth", 1) + "depth": self.config.get("depth", 1), + "only_inside_links": self.config.get("only_inside_links", False) } ) diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py index f9fd57a8..ff329a39 100644 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -105,7 +105,11 @@ class FetchNodeLevelK(BaseNode): for _ in range(self.depth): documents = self.obtain_content(documents, loader_kwargs) - return {self.output_keys[0]: documents} + filtered_documents = [doc for doc in documents if 'document' in doc] + + state.update({self.output[0]: filtered_documents}) + + return state def fetch_content(self, source: str, loader_kwargs) -> Optional[str]: if self.browser_base is not None: @@ -144,6 +148,7 @@ class FetchNodeLevelK(BaseNode): return full_links def obtain_content(self, documents: List, loader_kwargs) -> List: + new_documents = [] for doc in documents: source = doc['source'] if 'document' not in doc: @@ -162,10 +167,11 @@ class FetchNodeLevelK(BaseNode): # Check if the links are already present in other documents for link in full_links: # Check if any document is from the same link - if not any(d.get('source', '') == link for d in documents): + if not any(d.get('source', '') == link for d in documents) and not any(d.get('source', '') == link for d in new_documents): # Add the document - documents.append({"source": link}) + new_documents.append({"source": link}) + documents.extend(new_documents) return documents def process_links(self, base_url: str, links: list, loader_kwargs, depth: int, current_depth: int = 1) -> dict: From 2bdb01b07a7011564c23f1117fe524f9238fae1b Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Wed, 2 Oct 2024 11:04:17 +0200 Subject: [PATCH 10/12] Create parse_node_depth_k.py --- scrapegraphai/nodes/parse_node_depth_k.py | 156 ++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 scrapegraphai/nodes/parse_node_depth_k.py diff --git a/scrapegraphai/nodes/parse_node_depth_k.py b/scrapegraphai/nodes/parse_node_depth_k.py new file mode 100644 index 00000000..fd2f3810 --- /dev/null +++ b/scrapegraphai/nodes/parse_node_depth_k.py @@ -0,0 +1,156 @@ +""" +ParseNode Module +""" +import re +from typing import List, Optional, Tuple +from urllib.parse import urljoin +from langchain_community.document_transformers import Html2TextTransformer +from langchain_core.documents import Document +from .base_node import BaseNode +from ..utils.split_text_into_chunks import split_text_into_chunks +from ..helpers import default_filters + +class ParseNode(BaseNode): + """ + A node responsible for parsing HTML content from a document. + The parsed content is split into chunks for further processing. + + This node enhances the scraping workflow by allowing for targeted extraction of + content, thereby optimizing the processing of large HTML documents. + + Attributes: + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (dict): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Parse". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "ParseNode", + ): + super().__init__(node_name, "node", input, output, 1, node_config) + + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + self.parse_html = ( + True if node_config is None else node_config.get("parse_html", True) + ) + self.parse_urls = ( + False if node_config is None else node_config.get("parse_urls", False) + ) + + self.llm_model = node_config.get("llm_model") + self.chunk_size = node_config.get("chunk_size") + + def execute(self, state: dict) -> dict: + """ + Executes the node's logic to parse the HTML document content and split it into chunks. + + Args: + state (dict): The current state of the graph. The input keys will be used to fetch the + correct data from the state. + + Returns: + dict: The updated state with the output key containing the parsed content chunks. + + Raises: + KeyError: If the input keys are not found in the state, indicating that the + necessary information for parsing the content is missing. + """ + + self.logger.info(f"--- Executing {self.node_name} Node ---") + + input_keys = self.get_input_keys(state) + + input_data = [state[key] for key in input_keys] + docs_transformed = input_data[0] + source = input_data[1] if self.parse_urls else None + + if self.parse_html: + docs_transformed = Html2TextTransformer(ignore_links=False).transform_documents(input_data[0]) + docs_transformed = docs_transformed[0] + + link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) + + chunks = split_text_into_chunks(text=docs_transformed.page_content, + chunk_size=self.chunk_size-250, model=self.llm_model) + else: + docs_transformed = docs_transformed[0] + + link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) + + chunk_size = self.chunk_size + chunk_size = min(chunk_size - 500, int(chunk_size * 0.75)) + + if isinstance(docs_transformed, Document): + chunks = split_text_into_chunks(text=docs_transformed.page_content, + chunk_size=chunk_size, + model=self.llm_model) + else: + chunks = split_text_into_chunks(text=docs_transformed, + chunk_size=chunk_size, + model=self.llm_model) + + state.update({self.output[0]: chunks}) + if self.parse_urls: + state.update({self.output[1]: link_urls}) + state.update({self.output[2]: img_urls}) + + return state + + def _extract_urls(self, text: str, source: str) -> Tuple[List[str], List[str]]: + """ + Extracts URLs from the given text. + + Args: + text (str): The text to extract URLs from. + + Returns: + Tuple[List[str], List[str]]: A tuple containing the extracted link URLs and image URLs. + """ + if not self.parse_urls: + return [], [] + + image_extensions = default_filters.filter_dict["img_exts"] + image_extension_seq = '|'.join(image_extensions).replace('.','') + url_pattern = re.compile(r'(https?://[^\s]+|\S+\.(?:' + image_extension_seq + '))') + + all_urls = url_pattern.findall(text) + all_urls = self._clean_urls(all_urls) + + if not source.startswith("http"): + all_urls = [url for url in all_urls if url.startswith("http")] + else: + all_urls = [urljoin(source, url) for url in all_urls] + + images = [url for url in all_urls if any(url.endswith(ext) for ext in image_extensions)] + links = [url for url in all_urls if url not in images] + + return links, images + + def _clean_urls(self, urls: List[str]) -> List[str]: + """ + Cleans the URLs extracted from the text. + + Args: + urls (List[str]): The list of URLs to clean. + + Returns: + List[str]: The cleaned URLs. + """ + cleaned_urls = [] + for url in urls: + url = re.sub(r'.*?\]\(', '', url) + url = url.rstrip(').') + + cleaned_urls.append(url) + + return cleaned_urls From f755d56bb1e2406668c5114e649953adbbff6748 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Wed, 2 Oct 2024 12:28:48 +0200 Subject: [PATCH 11/12] updated parse node --- scrapegraphai/graphs/depth_search_graph.py | 18 +++- scrapegraphai/nodes/__init__.py | 1 + scrapegraphai/nodes/parse_node_depth_k.py | 120 +++------------------ 3 files changed, 33 insertions(+), 106 deletions(-) diff --git a/scrapegraphai/graphs/depth_search_graph.py b/scrapegraphai/graphs/depth_search_graph.py index fa6294a0..6ad3b245 100644 --- a/scrapegraphai/graphs/depth_search_graph.py +++ b/scrapegraphai/graphs/depth_search_graph.py @@ -8,7 +8,8 @@ from .base_graph import BaseGraph from .abstract_graph import AbstractGraph from ..utils.save_code_to_file import save_code_to_file from ..nodes import ( - FetchNodeLevelK + FetchNodeLevelK, + ParseNodeDepthK ) class DepthSearchGraph(AbstractGraph): @@ -71,12 +72,23 @@ class DepthSearchGraph(AbstractGraph): "only_inside_links": self.config.get("only_inside_links", False) } ) + + parse_node = ParseNodeDepthK( + input="docs", + output=["docs"], + node_config={ + "verbose": self.config.get("verbose", False) + } + ) return BaseGraph( nodes=[ - fetch_node + fetch_node, + parse_node + ], + edges=[ + (fetch_node, parse_node), ], - edges=[], entry_point=fetch_node, graph_name=self.__class__.__name__ ) diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py index 7b994746..edb195a5 100644 --- a/scrapegraphai/nodes/__init__.py +++ b/scrapegraphai/nodes/__init__.py @@ -31,3 +31,4 @@ from .reasoning_node import ReasoningNode from .fetch_node_level_k import FetchNodeLevelK from .generate_answer_node_k_level import GenerateAnswerNodeKLevel from .description_node import DescriptionNode +from .parse_node_depth_k import ParseNodeDepthK diff --git a/scrapegraphai/nodes/parse_node_depth_k.py b/scrapegraphai/nodes/parse_node_depth_k.py index fd2f3810..30afa23c 100644 --- a/scrapegraphai/nodes/parse_node_depth_k.py +++ b/scrapegraphai/nodes/parse_node_depth_k.py @@ -1,19 +1,14 @@ """ -ParseNode Module +ParseNodeDepthK Module """ import re from typing import List, Optional, Tuple -from urllib.parse import urljoin -from langchain_community.document_transformers import Html2TextTransformer -from langchain_core.documents import Document from .base_node import BaseNode -from ..utils.split_text_into_chunks import split_text_into_chunks -from ..helpers import default_filters +from ..utils.convert_to_md import convert_to_md -class ParseNode(BaseNode): +class ParseNodeDepthK(BaseNode): """ - A node responsible for parsing HTML content from a document. - The parsed content is split into chunks for further processing. + A node responsible for parsing HTML content from a series of documents. This node enhances the scraping workflow by allowing for targeted extraction of content, thereby optimizing the processing of large HTML documents. @@ -33,26 +28,17 @@ class ParseNode(BaseNode): input: str, output: List[str], node_config: Optional[dict] = None, - node_name: str = "ParseNode", + node_name: str = "ParseNodeDepthK", ): super().__init__(node_name, "node", input, output, 1, node_config) self.verbose = ( False if node_config is None else node_config.get("verbose", False) ) - self.parse_html = ( - True if node_config is None else node_config.get("parse_html", True) - ) - self.parse_urls = ( - False if node_config is None else node_config.get("parse_urls", False) - ) - - self.llm_model = node_config.get("llm_model") - self.chunk_size = node_config.get("chunk_size") def execute(self, state: dict) -> dict: """ - Executes the node's logic to parse the HTML document content and split it into chunks. + Executes the node's logic to parse the HTML documents content. Args: state (dict): The current state of the graph. The input keys will be used to fetch the @@ -67,90 +53,18 @@ class ParseNode(BaseNode): """ self.logger.info(f"--- Executing {self.node_name} Node ---") - + + # Interpret input keys based on the provided input expression input_keys = self.get_input_keys(state) - + # Fetching data from the state based on the input keys input_data = [state[key] for key in input_keys] - docs_transformed = input_data[0] - source = input_data[1] if self.parse_urls else None - - if self.parse_html: - docs_transformed = Html2TextTransformer(ignore_links=False).transform_documents(input_data[0]) - docs_transformed = docs_transformed[0] - - link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) - - chunks = split_text_into_chunks(text=docs_transformed.page_content, - chunk_size=self.chunk_size-250, model=self.llm_model) - else: - docs_transformed = docs_transformed[0] - - link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source) - - chunk_size = self.chunk_size - chunk_size = min(chunk_size - 500, int(chunk_size * 0.75)) - - if isinstance(docs_transformed, Document): - chunks = split_text_into_chunks(text=docs_transformed.page_content, - chunk_size=chunk_size, - model=self.llm_model) - else: - chunks = split_text_into_chunks(text=docs_transformed, - chunk_size=chunk_size, - model=self.llm_model) - - state.update({self.output[0]: chunks}) - if self.parse_urls: - state.update({self.output[1]: link_urls}) - state.update({self.output[2]: img_urls}) + documents = input_data[0] + + for doc in documents: + document_md = convert_to_md(doc["document"]) + doc["document_md"] = document_md + + state.update({self.output[0]: documents}) + return state - - def _extract_urls(self, text: str, source: str) -> Tuple[List[str], List[str]]: - """ - Extracts URLs from the given text. - - Args: - text (str): The text to extract URLs from. - - Returns: - Tuple[List[str], List[str]]: A tuple containing the extracted link URLs and image URLs. - """ - if not self.parse_urls: - return [], [] - - image_extensions = default_filters.filter_dict["img_exts"] - image_extension_seq = '|'.join(image_extensions).replace('.','') - url_pattern = re.compile(r'(https?://[^\s]+|\S+\.(?:' + image_extension_seq + '))') - - all_urls = url_pattern.findall(text) - all_urls = self._clean_urls(all_urls) - - if not source.startswith("http"): - all_urls = [url for url in all_urls if url.startswith("http")] - else: - all_urls = [urljoin(source, url) for url in all_urls] - - images = [url for url in all_urls if any(url.endswith(ext) for ext in image_extensions)] - links = [url for url in all_urls if url not in images] - - return links, images - - def _clean_urls(self, urls: List[str]) -> List[str]: - """ - Cleans the URLs extracted from the text. - - Args: - urls (List[str]): The list of URLs to clean. - - Returns: - List[str]: The cleaned URLs. - """ - cleaned_urls = [] - for url in urls: - url = re.sub(r'.*?\]\(', '', url) - url = url.rstrip(').') - - cleaned_urls.append(url) - - return cleaned_urls From 015c6fd90504b03981d6e259e2f1aa5b16fa2472 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Wed, 2 Oct 2024 13:06:00 +0200 Subject: [PATCH 12/12] remove link from markdown --- scrapegraphai/nodes/fetch_node_level_k.py | 9 +++++---- scrapegraphai/nodes/parse_node_depth_k.py | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scrapegraphai/nodes/fetch_node_level_k.py b/scrapegraphai/nodes/fetch_node_level_k.py index ff329a39..5cdd6571 100644 --- a/scrapegraphai/nodes/fetch_node_level_k.py +++ b/scrapegraphai/nodes/fetch_node_level_k.py @@ -95,8 +95,6 @@ class FetchNodeLevelK(BaseNode): documents = [{"source": source}] - self.logger.info(f"--- (Fetching HTML from: {source}) ---") - loader_kwargs = {} if self.node_config is not None: @@ -112,6 +110,8 @@ class FetchNodeLevelK(BaseNode): return state def fetch_content(self, source: str, loader_kwargs) -> Optional[str]: + self.logger.info(f"--- (Fetching HTML from: {source}) ---") + if self.browser_base is not None: try: from ..docloaders.browser_base import browser_base_fetch @@ -159,9 +159,10 @@ class FetchNodeLevelK(BaseNode): documents.remove(doc) continue - doc['document'] = document[0].page_content + #doc['document'] = document[0].page_content + doc['document'] = document - links = self.extract_links(doc['document']) + links = self.extract_links(doc['document'][0].page_content) full_links = self.get_full_links(source, links) # Check if the links are already present in other documents diff --git a/scrapegraphai/nodes/parse_node_depth_k.py b/scrapegraphai/nodes/parse_node_depth_k.py index 30afa23c..7b7ab194 100644 --- a/scrapegraphai/nodes/parse_node_depth_k.py +++ b/scrapegraphai/nodes/parse_node_depth_k.py @@ -5,6 +5,7 @@ import re from typing import List, Optional, Tuple from .base_node import BaseNode from ..utils.convert_to_md import convert_to_md +from langchain_community.document_transformers import Html2TextTransformer class ParseNodeDepthK(BaseNode): """ @@ -62,8 +63,9 @@ class ParseNodeDepthK(BaseNode): documents = input_data[0] for doc in documents: - document_md = convert_to_md(doc["document"]) - doc["document_md"] = document_md + document_md = Html2TextTransformer(ignore_links=True).transform_documents(doc["document"]) + #document_md = convert_to_md(doc["document"]) + doc["document"] = document_md[0].page_content state.update({self.output[0]: documents})