CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: update all nodes that were using MergeNode or IteratorNode

Browse Source
This commit is contained in:
Lorenzo Paleari 2024-09-13 04:19:40 +02:00
parent 66ea166438
commit a92dddb3e0
No known key found for this signature in database
GPG Key ID: 010F47E3CB681DED
8 changed files with 68 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapegraphai/graphs/csv_scraper_multi_graph.py
View File

@ -2,6 +2,7 @@
CSVScraperMultiGraph Module
"""
from copy import deepcopy
from typing import List, Optional
from pydantic import BaseModel
from .base_graph import BaseGraph
@ -48,6 +49,8 @@ class CSVScraperMultiGraph(AbstractGraph):
self.copy_config = safe_deepcopy(config)
self.copy_schema = deepcopy(schema)
super().__init__(prompt, config, source, schema)
def _create_graph(self) -> BaseGraph:
@ -58,17 +61,18 @@ class CSVScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
smart_scraper_instance = CSVScraperGraph(
prompt="",
source="",
config=self.copy_config,
)
# smart_scraper_instance = CSVScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],
node_config={
"graph_instance": smart_scraper_instance,
"graph_instance": CSVScraperGraph,
"scraper_config": self.copy_config,
}
)
@ -77,7 +81,7 @@ class CSVScraperMultiGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

20
scrapegraphai/graphs/json_scraper_multi_graph.py
View File

@ -61,19 +61,21 @@ class JSONScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
smart_scraper_instance = JSONScraperGraph(
prompt="",
source="",
config=self.copy_config,
schema=self.copy_schema
)
# smart_scraper_instance = JSONScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],
node_config={
"graph_instance": smart_scraper_instance,
}
"graph_instance": JSONScraperGraph,
"scraper_config": self.copy_config,
},
schema=self.copy_schema
)
merge_answers_node = MergeAnswersNode(
@ -81,7 +83,7 @@ class JSONScraperMultiGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

20
scrapegraphai/graphs/markdown_scraper_multi_graph.py
View File

@ -55,20 +55,22 @@ class MDScraperMultiGraph(AbstractGraph):
Returns:
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
smart_scraper_instance = MDScraperGraph(
prompt="",
source="",
config=self.copy_config,
schema=self.copy_schema
)
# smart_scraper_instance = MDScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
# Define the graph nodes
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],
node_config={
"graph_instance": smart_scraper_instance,
}
"graph_instance": MDScraperGraph,
"scraper_config": self.copy_config,
},
schema=self.copy_schema
)
merge_answers_node = MergeAnswersNode(
@ -76,7 +78,7 @@ class MDScraperMultiGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

2
scrapegraphai/graphs/omni_search_graph.py
View File

@ -92,7 +92,7 @@ class OmniSearchGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

20
scrapegraphai/graphs/pdf_scraper_multi_graph.py
View File

@ -59,19 +59,21 @@ class PdfScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
pdf_scraper_instance = PDFScraperGraph(
prompt="",
source="",
config=self.copy_config,
schema=self.copy_schema
)
# pdf_scraper_instance = PDFScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & pdfs",
output=["results"],
node_config={
"graph_instance": pdf_scraper_instance,
}
"graph_instance": PDFScraperGraph,
"scraper_config": self.copy_config,
},
schema=self.copy_schema
)
merge_answers_node = MergeAnswersNode(
@ -79,7 +81,7 @@ class PdfScraperMultiGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

2
scrapegraphai/graphs/search_graph.py
View File

@ -93,7 +93,7 @@ class SearchGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

20
scrapegraphai/graphs/smart_scraper_multi_graph.py
View File

@ -61,19 +61,21 @@ class SmartScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
smart_scraper_instance = SmartScraperGraph(
prompt="",
source="",
config=self.copy_config,
schema=self.copy_schema
)
# smart_scraper_instance = SmartScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & urls",
output=["results"],
node_config={
"graph_instance": smart_scraper_instance,
}
"graph_instance": SmartScraperGraph,
"scraper_config": self.copy_config,
},
schema=self.copy_schema
)
merge_answers_node = MergeAnswersNode(
@ -81,7 +83,7 @@ class SmartScraperMultiGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)

20
scrapegraphai/graphs/xml_scraper_multi_graph.py
View File

@ -59,19 +59,21 @@ class XMLScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
smart_scraper_instance = XMLScraperGraph(
prompt="",
source="",
config=self.copy_config,
schema=self.copy_schema
)
# smart_scraper_instance = XMLScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],
node_config={
"graph_instance": smart_scraper_instance,
}
"graph_instance": XMLScraperGraph,
"scaper_config": self.copy_config,
},
schema=self.copy_schema
)
merge_answers_node = MergeAnswersNode(
@ -79,7 +81,7 @@ class XMLScraperMultiGraph(AbstractGraph):
output=["answer"],
node_config={
"llm_model": self.llm_model,
"schema": self.schema
"schema": self.copy_schema
}
)