CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-23 21:00:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Corrected graphs to use common params

Browse Source
This commit is contained in:
Eric Page 2024-05-05 14:36:16 +02:00
parent 8d0e109a70
commit a53e95cbf0
7 changed files with 69 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapegraphai/graphs/csv_scraper_graph.py
View File

@ -36,14 +36,24 @@ class CSVScraperGraph(AbstractGraph):
parse_node = ParseNode(
input="doc",
output=["parsed_doc"],
node_config={
"chunk_size": self.model_token,
}
)
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model,
}
)
generate_answer_node = GenerateAnswerCSVNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={
"llm_model": self.llm_model,
}
)
return BaseGraph(
@ -68,4 +78,4 @@ class CSVScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

11
scrapegraphai/graphs/json_scraper_graph.py
View File

@ -61,16 +61,23 @@ class JSONScraperGraph(AbstractGraph):
input="doc",
output=["parsed_doc"],
node_config={
"chunk_size": self.model_token,
"chunk_size": self.model_token
}
)
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model
}
)
generate_answer_node = GenerateAnswerNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={
"llm": self.llm_model
}
)
return BaseGraph(
@ -99,4 +106,4 @@ class JSONScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

7
scrapegraphai/graphs/script_creator_graph.py
View File

@ -71,10 +71,15 @@ class ScriptCreatorGraph(AbstractGraph):
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model
}
)
generate_scraper_node = GenerateScraperNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={"llm_model": self.llm_model},
library=self.library,
website=self.source
)
@ -105,4 +110,4 @@ class ScriptCreatorGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

16
scrapegraphai/graphs/search_graph.py
View File

@ -49,25 +49,35 @@ class SearchGraph(AbstractGraph):
search_internet_node = SearchInternetNode(
input="user_prompt",
output=["url"],
node_config={
"llm_model": self.llm_model
}
)
fetch_node = FetchNode(
input="url | local_dir",
output=["doc"],
output=["doc"]
)
parse_node = ParseNode(
input="doc",
output=["parsed_doc"],
node_config={
"chunk_size": self.model_token,
"chunk_size": self.model_token
}
)
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model
}
)
generate_answer_node = GenerateAnswerNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={
"llm_model": self.llm_model
}
)
return BaseGraph(
@ -98,4 +108,4 @@ class SearchGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

13
scrapegraphai/graphs/smart_scraper_graph.py
View File

@ -57,22 +57,29 @@ class SmartScraperGraph(AbstractGraph):
"""
fetch_node = FetchNode(
input="url | local_dir",
output=["doc"],
output=["doc"]
)
parse_node = ParseNode(
input="doc",
output=["parsed_doc"],
node_config={
"chunk_size": self.model_token,
"chunk_size": self.model_token
}
)
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model
}
)
generate_answer_node = GenerateAnswerNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={
"llm_model": self.llm_model
}
)
return BaseGraph(
@ -101,4 +108,4 @@ class SmartScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

14
scrapegraphai/graphs/speech_graph.py
View File

@ -56,28 +56,34 @@ class SpeechGraph(AbstractGraph):
fetch_node = FetchNode(
input="url | local_dir",
output=["doc"],
output=["doc"]
)
parse_node = ParseNode(
input="doc",
output=["parsed_doc"],
node_config={
"chunk_size": self.model_token,
"chunk_size": self.model_token
}
)
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model }
)
generate_answer_node = GenerateAnswerNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={
"llm_model": self.llm_model
}
)
text_to_speech_node = TextToSpeechNode(
input="answer",
output=["audio"],
node_config={
"tts_model": OpenAITextToSpeech(self.config["tts_model"]),
"tts_model": OpenAITextToSpeech(self.config["tts_model"])
}
)
@ -116,4 +122,4 @@ class SpeechGraph(AbstractGraph):
"output_path", "output.mp3"))
print(f"Audio saved to {self.config.get('output_path', 'output.mp3')}")
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

13
scrapegraphai/graphs/xml_scraper_graph.py
View File

@ -57,22 +57,29 @@ class XMLScraperGraph(AbstractGraph):
fetch_node = FetchNode(
input="xml_dir",
output=["doc"],
output=["doc"]
)
parse_node = ParseNode(
input="doc",
output=["parsed_doc"],
node_config={
"chunk_size": self.model_token,
"chunk_size": self.model_token
}
)
rag_node = RAGNode(
input="user_prompt & (parsed_doc | doc)",
output=["relevant_chunks"],
node_config={
"llm_model": self.llm_model,
"embedder_model": self.embedder_model
}
)
generate_answer_node = GenerateAnswerNode(
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
output=["answer"],
node_config={
"llm_model": self.llm_model
}
)
return BaseGraph(
@ -101,4 +108,4 @@ class XMLScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")