From 9e3171b9fa263aa4a5a6fba2d9c8079d4e918490 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Fri, 13 Sep 2024 09:17:21 +0200 Subject: [PATCH] feat: add copy for smart_scraper_multi_concat --- scrapegraphai/graphs/csv_scraper_multi_graph.py | 6 ------ scrapegraphai/graphs/json_scraper_multi_graph.py | 7 ------- scrapegraphai/graphs/markdown_scraper_multi_graph.py | 7 ------- scrapegraphai/graphs/pdf_scraper_multi_graph.py | 7 ------- .../graphs/smart_scraper_multi_concat_graph.py | 12 ++++-------- 5 files changed, 4 insertions(+), 35 deletions(-) diff --git a/scrapegraphai/graphs/csv_scraper_multi_graph.py b/scrapegraphai/graphs/csv_scraper_multi_graph.py index a7df9fd0..a2eb13db 100644 --- a/scrapegraphai/graphs/csv_scraper_multi_graph.py +++ b/scrapegraphai/graphs/csv_scraper_multi_graph.py @@ -61,12 +61,6 @@ class CSVScraperMultiGraph(AbstractGraph): BaseGraph: A graph instance representing the web scraping and searching workflow. """ - # smart_scraper_instance = CSVScraperGraph( - # prompt="", - # source="", - # config=self.copy_config, - # ) - graph_iterator_node = GraphIteratorNode( input="user_prompt & jsons", output=["results"], diff --git a/scrapegraphai/graphs/json_scraper_multi_graph.py b/scrapegraphai/graphs/json_scraper_multi_graph.py index eaf5c007..8e7c56e6 100644 --- a/scrapegraphai/graphs/json_scraper_multi_graph.py +++ b/scrapegraphai/graphs/json_scraper_multi_graph.py @@ -61,13 +61,6 @@ class JSONScraperMultiGraph(AbstractGraph): BaseGraph: A graph instance representing the web scraping and searching workflow. """ - # smart_scraper_instance = JSONScraperGraph( - # prompt="", - # source="", - # config=self.copy_config, - # schema=self.copy_schema - # ) - graph_iterator_node = GraphIteratorNode( input="user_prompt & jsons", output=["results"], diff --git a/scrapegraphai/graphs/markdown_scraper_multi_graph.py b/scrapegraphai/graphs/markdown_scraper_multi_graph.py index 32e0464b..fb7d6863 100644 --- a/scrapegraphai/graphs/markdown_scraper_multi_graph.py +++ b/scrapegraphai/graphs/markdown_scraper_multi_graph.py @@ -55,14 +55,7 @@ class MDScraperMultiGraph(AbstractGraph): Returns: BaseGraph: A graph instance representing the web scraping and searching workflow. """ - # smart_scraper_instance = MDScraperGraph( - # prompt="", - # source="", - # config=self.copy_config, - # schema=self.copy_schema - # ) - # Define the graph nodes graph_iterator_node = GraphIteratorNode( input="user_prompt & jsons", output=["results"], diff --git a/scrapegraphai/graphs/pdf_scraper_multi_graph.py b/scrapegraphai/graphs/pdf_scraper_multi_graph.py index 801db131..3b19516b 100644 --- a/scrapegraphai/graphs/pdf_scraper_multi_graph.py +++ b/scrapegraphai/graphs/pdf_scraper_multi_graph.py @@ -59,13 +59,6 @@ class PdfScraperMultiGraph(AbstractGraph): BaseGraph: A graph instance representing the web scraping and searching workflow. """ - # pdf_scraper_instance = PDFScraperGraph( - # prompt="", - # source="", - # config=self.copy_config, - # schema=self.copy_schema - # ) - graph_iterator_node = GraphIteratorNode( input="user_prompt & pdfs", output=["results"], diff --git a/scrapegraphai/graphs/smart_scraper_multi_concat_graph.py b/scrapegraphai/graphs/smart_scraper_multi_concat_graph.py index 4d1867f9..1adda1a8 100644 --- a/scrapegraphai/graphs/smart_scraper_multi_concat_graph.py +++ b/scrapegraphai/graphs/smart_scraper_multi_concat_graph.py @@ -2,18 +2,17 @@ SmartScraperMultiGraph Module """ -from copy import copy, deepcopy +from copy import deepcopy from typing import List, Optional from pydantic import BaseModel - from .base_graph import BaseGraph from .abstract_graph import AbstractGraph from .smart_scraper_graph import SmartScraperGraph - from ..nodes import ( GraphIteratorNode, ConcatAnswersNode ) +from ..utils.copy import safe_deepcopy class SmartScraperMultiConcatGraph(AbstractGraph): @@ -46,11 +45,8 @@ class SmartScraperMultiConcatGraph(AbstractGraph): def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None): - - if all(isinstance(value, str) for value in config.values()): - self.copy_config = copy(config) - else: - self.copy_config = deepcopy(config) + + self.copy_config = safe_deepcopy(config) self.copy_schema = deepcopy(schema)