CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: script creator multi

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-09-26 10:09:36 +02:00
parent c181fea3bd
commit 9905be8a37
9 changed files with 23 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
examples/fireworks/script_multi_generator_fireworks.py
View File

@ -26,10 +26,9 @@ graph_config = {
# ************************************************ # ************************************************
# Create the ScriptCreatorGraph instance and run it # Create the ScriptCreatorGraph instance and run it
# ************************************************ # ************************************************
urls=[ urls=[
"https://perinim.github.io/", "https://schultzbergagency.com/emil-raste-karlsen/",
"https://perinim.github.io/cv/" "https://schultzbergagency.com/johanna-hedberg/",
] ]
# ************************************************ # ************************************************
@ -37,7 +36,8 @@ urls=[
# ************************************************ # ************************************************
script_creator_graph = ScriptCreatorMultiGraph( script_creator_graph = ScriptCreatorMultiGraph(
prompt="Who is Marco Perini?", prompt="Find information about actors",
# also accepts a string with the already downloaded HTML code
source=urls, source=urls,
config=graph_config config=graph_config
) )

7
examples/mistral/script_multi_generator_mistral.py
View File

@ -29,8 +29,8 @@ graph_config = {
# ************************************************ # ************************************************
urls=[ urls=[
"https://perinim.github.io/", "https://schultzbergagency.com/emil-raste-karlsen/",
"https://perinim.github.io/cv/" "https://schultzbergagency.com/johanna-hedberg/",
] ]
# ************************************************ # ************************************************
@ -38,7 +38,8 @@ urls=[
# ************************************************ # ************************************************
script_creator_graph = ScriptCreatorMultiGraph( script_creator_graph = ScriptCreatorMultiGraph(
prompt="Who is Marco Perini?", prompt="Find information about actors",
# also accepts a string with the already downloaded HTML code
source=urls, source=urls,
config=graph_config config=graph_config
) )

7
examples/nemotron/script_multi_generator_nemotron.py
View File

@ -29,8 +29,8 @@ graph_config = {
# ************************************************ # ************************************************
urls=[ urls=[
"https://perinim.github.io/", "https://schultzbergagency.com/emil-raste-karlsen/",
"https://perinim.github.io/cv/" "https://schultzbergagency.com/johanna-hedberg/",
] ]
# ************************************************ # ************************************************
@ -38,7 +38,8 @@ urls=[
# ************************************************ # ************************************************
script_creator_graph = ScriptCreatorMultiGraph( script_creator_graph = ScriptCreatorMultiGraph(
prompt="Who is Marco Perini?", prompt="Find information about actors",
# also accepts a string with the already downloaded HTML code
source=urls, source=urls,
config=graph_config config=graph_config
) )

7
examples/openai/script_multi_generator_openai.py
View File

@ -29,8 +29,8 @@ graph_config = {
# ************************************************ # ************************************************
urls=[ urls=[
"https://perinim.github.io/", "https://schultzbergagency.com/emil-raste-karlsen/",
"https://perinim.github.io/cv/" "https://schultzbergagency.com/johanna-hedberg/",
] ]
# ************************************************ # ************************************************
@ -38,7 +38,8 @@ urls=[
# ************************************************ # ************************************************
script_creator_graph = ScriptCreatorMultiGraph( script_creator_graph = ScriptCreatorMultiGraph(
prompt="Who is Marco Perini?", prompt="Find information about actors",
# also accepts a string with the already downloaded HTML code
source=urls, source=urls,
config=graph_config config=graph_config
) )

2
scrapegraphai/graphs/json_scraper_multi_graph.py
View File

@ -45,9 +45,7 @@ class JSONScraperMultiGraph(AbstractGraph):
config: dict, schema: Optional[BaseModel] = None): config: dict, schema: Optional[BaseModel] = None):
self.max_results = config.get("max_results", 3) self.max_results = config.get("max_results", 3)
self.copy_config = safe_deepcopy(config) self.copy_config = safe_deepcopy(config)
self.copy_schema = deepcopy(schema) self.copy_schema = deepcopy(schema)
super().__init__(prompt, config, source, schema) super().__init__(prompt, config, source, schema)

2
scrapegraphai/graphs/markdown_scraper_multi_graph.py
View File

@ -1,7 +1,7 @@
""" """
MDScraperMultiGraph Module MDScraperMultiGraph Module
""" """
from copy import copy, deepcopy from copy import deepcopy
from typing import List, Optional from typing import List, Optional
from pydantic import BaseModel from pydantic import BaseModel
from .base_graph import BaseGraph from .base_graph import BaseGraph

16
scrapegraphai/graphs/script_creator_multi_graph.py
View File

@ -1,6 +1,7 @@
""" """
ScriptCreatorMultiGraph Module ScriptCreatorMultiGraph Module
""" """
from copy import deepcopy
from typing import List, Optional from typing import List, Optional
from pydantic import BaseModel from pydantic import BaseModel
from .base_graph import BaseGraph from .base_graph import BaseGraph
@ -45,7 +46,7 @@ class ScriptCreatorMultiGraph(AbstractGraph):
self.max_results = config.get("max_results", 3) self.max_results = config.get("max_results", 3)
self.copy_config = safe_deepcopy(config) self.copy_config = safe_deepcopy(config)
self.copy_schema = deepcopy(schema)
super().__init__(prompt, config, source, schema) super().__init__(prompt, config, source, schema)
def _create_graph(self) -> BaseGraph: def _create_graph(self) -> BaseGraph:
@ -55,19 +56,14 @@ class ScriptCreatorMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
script_generator_instance = ScriptCreatorGraph(
prompt="",
source="",
config=self.copy_config,
schema=self.schema
)
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & urls", input="user_prompt & urls",
output=["scripts"], output=["scripts"],
node_config={ node_config={
"graph_instance": script_generator_instance, "graph_instance": ScriptCreatorGraph,
} "scraper_config": self.copy_config,
},
schema=self.copy_schema
) )
merge_scripts_node = MergeGeneratedScriptsNode( merge_scripts_node = MergeGeneratedScriptsNode(

7
scrapegraphai/graphs/smart_scraper_multi_graph.py
View File

@ -60,13 +60,6 @@ class SmartScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
# smart_scraper_instance = SmartScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & urls", input="user_prompt & urls",
output=["results"], output=["results"],

10
scrapegraphai/graphs/xml_scraper_multi_graph.py
View File

@ -45,9 +45,7 @@ class XMLScraperMultiGraph(AbstractGraph):
config: dict, schema: Optional[BaseModel] = None): config: dict, schema: Optional[BaseModel] = None):
self.copy_config = safe_deepcopy(config) self.copy_config = safe_deepcopy(config)
self.copy_schema = deepcopy(schema) self.copy_schema = deepcopy(schema)
super().__init__(prompt, config, source, schema) super().__init__(prompt, config, source, schema)
def _create_graph(self) -> BaseGraph: def _create_graph(self) -> BaseGraph:
@ -57,14 +55,6 @@ class XMLScraperMultiGraph(AbstractGraph):
Returns: Returns:
BaseGraph: A graph instance representing the web scraping and searching workflow. BaseGraph: A graph instance representing the web scraping and searching workflow.
""" """
# smart_scraper_instance = XMLScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode( graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons", input="user_prompt & jsons",
output=["results"], output=["results"],