CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-06 21:11:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: removed rag node

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-05-31 21:03:48 +02:00
parent 6d1d91a1e9
commit 930f673747
3 changed files with 7 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapegraphai/graphs/pdf_scraper_graph.py
View File

@ -1,3 +1,4 @@
""" """
PDFScraperGraph Module PDFScraperGraph Module
""" """
@ -9,7 +10,6 @@ from .abstract_graph import AbstractGraph
from ..nodes import ( from ..nodes import (
FetchNode, FetchNode,
RAGNode,
GenerateAnswerPDFNode GenerateAnswerPDFNode
) )
@ -63,14 +63,7 @@ class PDFScraperGraph(AbstractGraph):
input='pdf | pdf_dir', input='pdf | pdf_dir',
output=["doc"], output=["doc"],
) )
rag_node = RAGNode(
input="user_prompt & doc",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model
}
)
generate_answer_node_pdf = GenerateAnswerPDFNode( generate_answer_node_pdf = GenerateAnswerPDFNode(
input="user_prompt & (relevant_chunks | doc)", input="user_prompt & (relevant_chunks | doc)",
output=["answer"], output=["answer"],
@ -83,12 +76,10 @@ class PDFScraperGraph(AbstractGraph):
return BaseGraph( return BaseGraph(
nodes=[ nodes=[
fetch_node, fetch_node,
rag_node,
generate_answer_node_pdf, generate_answer_node_pdf,
], ],
edges=[ edges=[
(fetch_node, rag_node), (fetch_node, generate_answer_node_pdf)
(rag_node, generate_answer_node_pdf)
], ],
entry_point=fetch_node entry_point=fetch_node
) )
@ -104,4 +95,4 @@ class PDFScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source} inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs) self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.") return self.final_state.get("answer", "No answer found.")

2
scrapegraphai/graphs/smart_scraper_graph.py
View File

@ -117,4 +117,4 @@ class SmartScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source} inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs) self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.") return self.final_state.get("answer", "No answer found.")

6
scrapegraphai/nodes/generate_answer_pdf_node.py
View File

@ -95,9 +95,7 @@ class GenerateAnswerPDFNode(BaseNode):
output_parser = JsonOutputParser() output_parser = JsonOutputParser()
format_instructions = output_parser.get_format_instructions() format_instructions = output_parser.get_format_instructions()
chains_dict = {} chains_dict = {}
# Use tqdm to add progress bar # Use tqdm to add progress bar
for i, chunk in enumerate( for i, chunk in enumerate(
tqdm(doc, desc="Processing chunks", disable=not self.verbose) tqdm(doc, desc="Processing chunks", disable=not self.verbose)
@ -107,7 +105,7 @@ class GenerateAnswerPDFNode(BaseNode):
template=template_no_chunks_pdf, template=template_no_chunks_pdf,
input_variables=["question"], input_variables=["question"],
partial_variables={ partial_variables={
"context": chunk.page_content, "context":chunk,
"format_instructions": format_instructions, "format_instructions": format_instructions,
}, },
) )
@ -116,7 +114,7 @@ class GenerateAnswerPDFNode(BaseNode):
template=template_chunks_pdf, template=template_chunks_pdf,
input_variables=["question"], input_variables=["question"],
partial_variables={ partial_variables={
"context": chunk.page_content, "context":chunk,
"chunk_id": i + 1, "chunk_id": i + 1,
"format_instructions": format_instructions, "format_instructions": format_instructions,
}, },