CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-06 21:11:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: parse_node

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-07-17 22:58:21 +02:00
parent 68f58cc4dd
commit 07f1e23d23
4 changed files with 29 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/local_models/smart_scraper_ollama.py
View File

@ -29,7 +29,7 @@ graph_config = {
smart_scraper_graph = SmartScraperGraph(
prompt="List me all the titles",
source="https://sport.sky.it/nba?gr=www",
source="https://perinim.github.io/projects",
config=graph_config
)

1
scrapegraphai/nodes/base_node.py
View File

@ -88,7 +88,6 @@ class BaseNode(ABC):
param (dict): The dictionary to update node_config with.
overwrite (bool): Flag indicating if the values of node_config should be overwritten if their value is not None.
"""
for key, val in params.items():
if hasattr(self, key) and not overwrite:
continue

2
scrapegraphai/nodes/generate_answer_node.py
View File

@ -121,7 +121,7 @@ class GenerateAnswerNode(BaseNode):
answer = chain.invoke({"question": user_prompt})
break
prompt = PromptTemplate(
prompt = PromptTemplate(
template=template_chunks_prompt,
input_variables=["question"],
partial_variables={"context": chunk,

41
scrapegraphai/nodes/parse_node.py
View File

@ -50,35 +50,48 @@ class ParseNode(BaseNode):
Args:
state (dict): The current state of the graph. The input keys will be used to fetch the
correct data from the state.
correct data from the state.
Returns:
dict: The updated state with the output key containing the parsed content chunks.
Raises:
KeyError: If the input keys are not found in the state.
KeyError: If the input keys are not found in the state, indicating that the
necessary information for parsing the content is missing.
"""
self.logger.info(f"--- Executing {self.node_name} Node ---")
# Fetch data using input keys
# Interpret input keys based on the provided input expression
input_keys = self.get_input_keys(state)
input_data = [state[key] for key in input_keys]
docs_transformed = input_data[0]
# Parse HTML if enabled
# Fetching data from the state based on the input keys
input_data = [state[key] for key in input_keys]
# Parse the document
docs_transformed = input_data[0]
if self.parse_html:
docs_transformed = Html2TextTransformer().transform_documents(input_data[0])
docs_transformed = docs_transformed[0]
# Get text content
text_content = docs_transformed.page_content if type(docs_transformed) == Document else docs_transformed
chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096),
token_counter=lambda x: len(x),
memoize=False)
else:
docs_transformed = docs_transformed[0]
# Chunk the text
chunk_size = self.node_config.get("chunk_size", 4096) - 250
chunks = chunk(text=text_content, chunk_size=chunk_size, token_counter=lambda x: len(x.split()), memoize=False)
# Update state with chunks
if type(docs_transformed) == Document:
chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096),
token_counter=lambda x: len(x),
memoize=False)
else:
chunks = chunk(text=docs_transformed,
chunk_size= self.node_config.get("chunk_size", 4096),
token_counter=lambda x: len(x),
memoize=False)
state.update({self.output[0]: chunks})
return state
return state