CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Revert "initial creation of FetchNodeLevelK and DescriptionNode"

Browse Source 
This reverts commit 336bf705ec.
This commit is contained in:
Matteo Vedovati 2024-09-30 12:21:26 +02:00
parent 336bf705ec
commit 7411ff061c
2 changed files with 0 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
scrapegraphai/nodes/description_node.py
View File

@ -1,42 +0,0 @@
"""
DescriptionNode Module
"""
from typing import List, Optional
from .base_node import BaseNode
class DescriptionNode(BaseNode):
"""
A node responsible for generating a description of a given document. This description is
generated using a language model and is used for retrieving the right documents.
It allows scraping of big documents without exceeding the token limit of the language model.
Attributes:
llm_model: An instance of a language model client, configured for generating answers.
verbose (bool): A flag indicating whether to show print statements during execution.
Args:
input (str): Boolean expression defining the input keys needed from the state.
output (List[str]): List of output keys to be updated in the state.
node_config (dict): Additional configuration for the node.
node_name (str): The unique identifier name for the node, defaulting to "Parse".
"""
def __init__(
self,
input: str,
output: List[str],
node_config: Optional[dict] = None,
node_name: str = "Description",
):
super().__init__(node_name, "node", input, output, 2, node_config)
self.llm_model = node_config["llm_model"]
self.embedder_model = node_config.get("embedder_model", None)
self.verbose = (
False if node_config is None else node_config.get("verbose", False)
)
self.cache_path = node_config.get("cache_path", False)
def execute(self, state: dict) -> dict:
pass

39
scrapegraphai/nodes/fetch_node_level_K.py
View File

@ -1,39 +0,0 @@
"""
FetchNodeLevelK Module
"""
from typing import List, Optional
from .base_node import BaseNode
class FetchNodeLevelK(BaseNode):
"""
A node responsible for fetching all the pages at a certain level of hyperlink the graph.
Attributes:
llm_model: An instance of a language model client, configured for generating answers.
verbose (bool): A flag indicating whether to show print statements during execution.
Args:
input (str): Boolean expression defining the input keys needed from the state.
output (List[str]): List of output keys to be updated in the state.
node_config (dict): Additional configuration for the node.
node_name (str): The unique identifier name for the node, defaulting to "Parse".
"""
def __init__(
self,
input: str,
output: List[str],
node_config: Optional[dict] = None,
node_name: str = "FetchLevelK",
):
super().__init__(node_name, "node", input, output, 2, node_config)
self.llm_model = node_config["llm_model"]
self.embedder_model = node_config.get("embedder_model", None)
self.verbose = (
False if node_config is None else node_config.get("verbose", False)
)
self.cache_path = node_config.get("cache_path", False)
def execute(self, state: dict) -> dict:
pass