CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add copy for smart_scraper_multi_concat

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-09-13 09:17:21 +02:00
parent 2ae26e96a0
commit 9e3171b9fa
5 changed files with 4 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapegraphai/graphs/csv_scraper_multi_graph.py
View File

@ -61,12 +61,6 @@ class CSVScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
# smart_scraper_instance = CSVScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],

7
scrapegraphai/graphs/json_scraper_multi_graph.py
View File

@ -61,13 +61,6 @@ class JSONScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
# smart_scraper_instance = JSONScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],

7
scrapegraphai/graphs/markdown_scraper_multi_graph.py
View File

@ -55,14 +55,7 @@ class MDScraperMultiGraph(AbstractGraph):
Returns:
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
# smart_scraper_instance = MDScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
# Define the graph nodes
graph_iterator_node = GraphIteratorNode(
input="user_prompt & jsons",
output=["results"],

7
scrapegraphai/graphs/pdf_scraper_multi_graph.py
View File

@ -59,13 +59,6 @@ class PdfScraperMultiGraph(AbstractGraph):
BaseGraph: A graph instance representing the web scraping and searching workflow.
"""
# pdf_scraper_instance = PDFScraperGraph(
# prompt="",
# source="",
# config=self.copy_config,
# schema=self.copy_schema
# )
graph_iterator_node = GraphIteratorNode(
input="user_prompt & pdfs",
output=["results"],

12
scrapegraphai/graphs/smart_scraper_multi_concat_graph.py
View File

@ -2,18 +2,17 @@
SmartScraperMultiGraph Module
"""
from copy import copy, deepcopy
from copy import deepcopy
from typing import List, Optional
from pydantic import BaseModel
from .base_graph import BaseGraph
from .abstract_graph import AbstractGraph
from .smart_scraper_graph import SmartScraperGraph
from ..nodes import (
GraphIteratorNode,
ConcatAnswersNode
)
from ..utils.copy import safe_deepcopy
class SmartScraperMultiConcatGraph(AbstractGraph):
@ -46,11 +45,8 @@ class SmartScraperMultiConcatGraph(AbstractGraph):
def __init__(self, prompt: str, source: List[str],
config: dict, schema: Optional[BaseModel] = None):
if all(isinstance(value, str) for value in config.values()):
self.copy_config = copy(config)
else:
self.copy_config = deepcopy(config)
self.copy_config = safe_deepcopy(config)
self.copy_schema = deepcopy(schema)