diff --git a/commit_and_push.sh b/commit_and_push.sh index 20be12fc..ff6dcc4c 100755 --- a/commit_and_push.sh +++ b/commit_and_push.sh @@ -21,7 +21,7 @@ commit_message="$1" # Run Pylint on the specified Python files pylint scrapegraphai/**/*.py scrapegraphai/*.py examples/**/*.py tests/**/*.py -#Maket the pull +#Make the pull git pull # Add the modified files to the Git repository diff --git a/examples/graph_examples/smart_scraper_example.py b/examples/graph_examples/smart_scraper_example.py index adf949c9..70ff06e9 100644 --- a/examples/graph_examples/smart_scraper_example.py +++ b/examples/graph_examples/smart_scraper_example.py @@ -16,8 +16,8 @@ llm_config = { } # Define URL and PROMPT -URL = "https://perinim.github.io/projects/" -PROMPT = "List me all the titles and project descriptions" +URL = "https://www.google.com/search?client=safari&rls=en&q=ristoranti+trento&ie=UTF-8&oe=UTF-8" +PROMPT = "List me all the https inside the page" # Create the SmartScraperGraph instance smart_scraper_graph = SmartScraperGraph(PROMPT, URL, llm_config) diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py index 2a3a1730..200e2ddb 100644 --- a/scrapegraphai/nodes/generate_answer_node.py +++ b/scrapegraphai/nodes/generate_answer_node.py @@ -11,6 +11,7 @@ from langchain_core.runnables import RunnableParallel # Imports from the library from .base_node import BaseNode +from langchain.text_splitter import RecursiveCharacterTextSplitter class GenerateAnswerNode(BaseNode): @@ -114,24 +115,30 @@ class GenerateAnswerNode(BaseNode): "chunk_id": i + 1, "format_instructions": format_instructions}, ) # Dynamically name the chains based on their index - chain_name = f"chunk{i+1}" - chains_dict[chain_name] = prompt | self.llm | output_parser + chains_dict[f"chunk{i+1}"] = prompt | self.llm | output_parser - # Use dictionary unpacking to pass the dynamically named chains to RunnableParallel - map_chain = RunnableParallel(**chains_dict) - # Chain - answer_map = map_chain.invoke({"question": user_input}) + text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=4000, + chunk_overlap=0, + ) + + chunks = text_splitter.split_text(str(chains_dict)) - # Merge the answers from the chunks merge_prompt = PromptTemplate( template=template_merge, input_variables=["context", "question"], partial_variables={"format_instructions": format_instructions}, ) merge_chain = merge_prompt | self.llm | output_parser - answer = merge_chain.invoke( - {"context": answer_map, "question": user_input}) - # Update the state with the generated answer + answer_lines = [] + for chunk in chunks: + answer_temp = merge_chain.invoke( + {"context": chunk, "question": user_input}) + answer_lines.append(answer_temp) + + unique_answer_lines = list(set(answer_lines)) + answer = '\n'.join(unique_answer_lines) + state.update({"answer": answer}) return state