From 7411ff061c9ea74ddcd043574da1d968f6abaf99 Mon Sep 17 00:00:00 2001 From: Matteo Vedovati Date: Mon, 30 Sep 2024 12:21:26 +0200 Subject: [PATCH] Revert "initial creation of FetchNodeLevelK and DescriptionNode" This reverts commit 336bf705ec6f8200987b9a10f1210d732a35c7b0. --- scrapegraphai/nodes/description_node.py | 42 ----------------------- scrapegraphai/nodes/fetch_node_level_K.py | 39 --------------------- 2 files changed, 81 deletions(-) delete mode 100644 scrapegraphai/nodes/description_node.py delete mode 100644 scrapegraphai/nodes/fetch_node_level_K.py diff --git a/scrapegraphai/nodes/description_node.py b/scrapegraphai/nodes/description_node.py deleted file mode 100644 index 200d7032..00000000 --- a/scrapegraphai/nodes/description_node.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -DescriptionNode Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class DescriptionNode(BaseNode): - """ - A node responsible for generating a description of a given document. This description is - generated using a language model and is used for retrieving the right documents. - - It allows scraping of big documents without exceeding the token limit of the language model. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "Description", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass diff --git a/scrapegraphai/nodes/fetch_node_level_K.py b/scrapegraphai/nodes/fetch_node_level_K.py deleted file mode 100644 index 2fd3aa8b..00000000 --- a/scrapegraphai/nodes/fetch_node_level_K.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -FetchNodeLevelK Module -""" -from typing import List, Optional -from .base_node import BaseNode - -class FetchNodeLevelK(BaseNode): - """ - A node responsible for fetching all the pages at a certain level of hyperlink the graph. - - Attributes: - llm_model: An instance of a language model client, configured for generating answers. - verbose (bool): A flag indicating whether to show print statements during execution. - - Args: - input (str): Boolean expression defining the input keys needed from the state. - output (List[str]): List of output keys to be updated in the state. - node_config (dict): Additional configuration for the node. - node_name (str): The unique identifier name for the node, defaulting to "Parse". - """ - - def __init__( - self, - input: str, - output: List[str], - node_config: Optional[dict] = None, - node_name: str = "FetchLevelK", - ): - super().__init__(node_name, "node", input, output, 2, node_config) - - self.llm_model = node_config["llm_model"] - self.embedder_model = node_config.get("embedder_model", None) - self.verbose = ( - False if node_config is None else node_config.get("verbose", False) - ) - self.cache_path = node_config.get("cache_path", False) - - def execute(self, state: dict) -> dict: - pass