CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: update all nodes that were using MergeNode or IteratorNode

Browse Source
This commit is contained in:
Lorenzo Paleari 2024-09-13 04:19:40 +02:00
parent 66ea166438
commit a92dddb3e0
No known key found for this signature in database
GPG Key ID: 010F47E3CB681DED
8 changed files with 68 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapegraphai/graphs/csv_scraper_multi_graph.py
View File

@ -2,6 +2,7 @@
CSVScraperMultiGraph Module CSVScraperMultiGraph Module
""" """
from copy import deepcopy
from typing import List, Optional from typing import List, Optional
from pydantic import BaseModel from pydantic import BaseModel
from .base_graph import BaseGraph from .base_graph import BaseGraph
@ -48,6 +49,8 @@ class CSVScraperMultiGraph(AbstractGraph):
self.copy_config = safe_deepcopy(config) self.copy_config = safe_deepcopy(config)
self.copy_schema = deepcopy(schema)
super().__init__(prompt, config, source, schema) super().__init__(prompt, config, source, schema)
def _create_graph(self) -> BaseGraph: def _create_graph(self) -> BaseGraph:
@ -58,17 +61,18 @@ class CSVScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
smart_scraper_instance = CSVScraperGraph( # smart_scraper_instance = CSVScraperGraph(
prompt="", # prompt="",
source="", # source="",
config=self.copy_config, # config=self.copy_config,
) # )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons", input="user_prompt & jsons",
output=["results"], output=["results"],
node_config={ node_config={
"graph_instance": smart_scraper_instance, "graph_instance": CSVScraperGraph,
"scraper_config": self.copy_config,
} }
) )
@ -77,7 +81,7 @@ class CSVScraperMultiGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

20
scrapegraphai/graphs/json_scraper_multi_graph.py
View File

@ -61,19 +61,21 @@ class JSONScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
smart_scraper_instance = JSONScraperGraph( # smart_scraper_instance = JSONScraperGraph(
prompt="", # prompt="",
source="", # source="",
config=self.copy_config, # config=self.copy_config,
schema=self.copy_schema # schema=self.copy_schema
) # )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons", input="user_prompt & jsons",
output=["results"], output=["results"],
node_config={ node_config={
"graph_instance": smart_scraper_instance, "graph_instance": JSONScraperGraph,
} "scraper_config": self.copy_config,
},
schema=self.copy_schema
) )
merge_answers_node = MergeAnswersNode( merge_answers_node = MergeAnswersNode(
@ -81,7 +83,7 @@ class JSONScraperMultiGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

20
scrapegraphai/graphs/markdown_scraper_multi_graph.py
View File

@ -55,20 +55,22 @@ class MDScraperMultiGraph(AbstractGraph):
Returns: Returns:
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
smart_scraper_instance = MDScraperGraph( # smart_scraper_instance = MDScraperGraph(
prompt="", # prompt="",
source="", # source="",
config=self.copy_config, # config=self.copy_config,
schema=self.copy_schema # schema=self.copy_schema
) # )
# Define the graph nodes # Define the graph nodes
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons", input="user_prompt & jsons",
output=["results"], output=["results"],
node_config={ node_config={
"graph_instance": smart_scraper_instance, "graph_instance": MDScraperGraph,
} "scraper_config": self.copy_config,
},
schema=self.copy_schema
) )
merge_answers_node = MergeAnswersNode( merge_answers_node = MergeAnswersNode(
@ -76,7 +78,7 @@ class MDScraperMultiGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

2
scrapegraphai/graphs/omni_search_graph.py
View File

@ -92,7 +92,7 @@ class OmniSearchGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

20
scrapegraphai/graphs/pdf_scraper_multi_graph.py
View File

@ -59,19 +59,21 @@ class PdfScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
pdf_scraper_instance = PDFScraperGraph( # pdf_scraper_instance = PDFScraperGraph(
prompt="", # prompt="",
source="", # source="",
config=self.copy_config, # config=self.copy_config,
schema=self.copy_schema # schema=self.copy_schema
) # )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & pdfs", input="user_prompt & pdfs",
output=["results"], output=["results"],
node_config={ node_config={
"graph_instance": pdf_scraper_instance, "graph_instance": PDFScraperGraph,
} "scraper_config": self.copy_config,
},
schema=self.copy_schema
) )
merge_answers_node = MergeAnswersNode( merge_answers_node = MergeAnswersNode(
@ -79,7 +81,7 @@ class PdfScraperMultiGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

2
scrapegraphai/graphs/search_graph.py
View File

@ -93,7 +93,7 @@ class SearchGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

20
scrapegraphai/graphs/smart_scraper_multi_graph.py
View File

@ -61,19 +61,21 @@ class SmartScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
smart_scraper_instance = SmartScraperGraph( # smart_scraper_instance = SmartScraperGraph(
prompt="", # prompt="",
source="", # source="",
config=self.copy_config, # config=self.copy_config,
schema=self.copy_schema # schema=self.copy_schema
) # )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & urls", input="user_prompt & urls",
output=["results"], output=["results"],
node_config={ node_config={
"graph_instance": smart_scraper_instance, "graph_instance": SmartScraperGraph,
} "scraper_config": self.copy_config,
},
schema=self.copy_schema
) )
merge_answers_node = MergeAnswersNode( merge_answers_node = MergeAnswersNode(
@ -81,7 +83,7 @@ class SmartScraperMultiGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )

20
scrapegraphai/graphs/xml_scraper_multi_graph.py
View File

@ -59,19 +59,21 @@ class XMLScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
smart_scraper_instance = XMLScraperGraph( # smart_scraper_instance = XMLScraperGraph(
prompt="", # prompt="",
source="", # source="",
config=self.copy_config, # config=self.copy_config,
schema=self.copy_schema # schema=self.copy_schema
) # )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons", input="user_prompt & jsons",
output=["results"], output=["results"],
node_config={ node_config={
"graph_instance": smart_scraper_instance, "graph_instance": XMLScraperGraph,
} "scaper_config": self.copy_config,
},
schema=self.copy_schema
) )
merge_answers_node = MergeAnswersNode( merge_answers_node = MergeAnswersNode(
@ -79,7 +81,7 @@ class XMLScraperMultiGraph(AbstractGraph):
output=["answer"], output=["answer"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,
"schema": self.schema "schema": self.copy_schema
} }
) )