diff --git a/examples/local_models/smart_scraper_ollama.py b/examples/local_models/smart_scraper_ollama.py index e80413c2..0b3fcbfc 100644 --- a/examples/local_models/smart_scraper_ollama.py +++ b/examples/local_models/smart_scraper_ollama.py @@ -29,7 +29,7 @@ graph_config = { smart_scraper_graph = SmartScraperGraph( prompt="List me all the titles", - source="https://sport.sky.it/nba?gr=www", + source="https://perinim.github.io/projects", config=graph_config ) diff --git a/scrapegraphai/nodes/base_node.py b/scrapegraphai/nodes/base_node.py index bd95cd28..d1b59500 100644 --- a/scrapegraphai/nodes/base_node.py +++ b/scrapegraphai/nodes/base_node.py @@ -88,7 +88,6 @@ class BaseNode(ABC): param (dict): The dictionary to update node_config with. overwrite (bool): Flag indicating if the values of node_config should be overwritten if their value is not None. """ - for key, val in params.items(): if hasattr(self, key) and not overwrite: continue diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py index f764e58b..cd730bcc 100644 --- a/scrapegraphai/nodes/generate_answer_node.py +++ b/scrapegraphai/nodes/generate_answer_node.py @@ -121,7 +121,7 @@ class GenerateAnswerNode(BaseNode): answer = chain.invoke({"question": user_prompt}) break - prompt = PromptTemplate( + prompt = PromptTemplate( template=template_chunks_prompt, input_variables=["question"], partial_variables={"context": chunk, diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py index fba1da88..5c6c7cb3 100644 --- a/scrapegraphai/nodes/parse_node.py +++ b/scrapegraphai/nodes/parse_node.py @@ -50,35 +50,48 @@ class ParseNode(BaseNode): Args: state (dict): The current state of the graph. The input keys will be used to fetch the - correct data from the state. + correct data from the state. Returns: dict: The updated state with the output key containing the parsed content chunks. Raises: - KeyError: If the input keys are not found in the state. + KeyError: If the input keys are not found in the state, indicating that the + necessary information for parsing the content is missing. """ self.logger.info(f"--- Executing {self.node_name} Node ---") - # Fetch data using input keys + # Interpret input keys based on the provided input expression input_keys = self.get_input_keys(state) - input_data = [state[key] for key in input_keys] - docs_transformed = input_data[0] - # Parse HTML if enabled + # Fetching data from the state based on the input keys + input_data = [state[key] for key in input_keys] + # Parse the document + docs_transformed = input_data[0] if self.parse_html: docs_transformed = Html2TextTransformer().transform_documents(input_data[0]) docs_transformed = docs_transformed[0] - # Get text content - text_content = docs_transformed.page_content if type(docs_transformed) == Document else docs_transformed + chunks = chunk(text=docs_transformed.page_content, + chunk_size= self.node_config.get("chunk_size", 4096), + token_counter=lambda x: len(x), + memoize=False) + else: + docs_transformed = docs_transformed[0] - # Chunk the text - chunk_size = self.node_config.get("chunk_size", 4096) - 250 - chunks = chunk(text=text_content, chunk_size=chunk_size, token_counter=lambda x: len(x.split()), memoize=False) - - # Update state with chunks + if type(docs_transformed) == Document: + chunks = chunk(text=docs_transformed.page_content, + chunk_size= self.node_config.get("chunk_size", 4096), + token_counter=lambda x: len(x), + memoize=False) + else: + + chunks = chunk(text=docs_transformed, + chunk_size= self.node_config.get("chunk_size", 4096), + token_counter=lambda x: len(x), + memoize=False) + state.update({self.output[0]: chunks}) - return state + return state \ No newline at end of file