From 5bda918a39e4b50d86d784b4c592cc2ea1a68986 Mon Sep 17 00:00:00 2001
From: Marco Vinciguerra <mvincig11@gmail.com>
Date: Sat, 1 Jun 2024 12:04:19 +0200
Subject: [PATCH] feat: add json multiscraper

---
 .../local_models/json_scraper_multi_ollama.py | 28 +++++++------------
 .../local_models/pdf_scraper_multi_ollama.py  |  1 -
 scrapegraphai/graphs/__init__.py              |  2 +-
 scrapegraphai/graphs/json_scraper_multi.py    |  6 ++--
 scrapegraphai/nodes/__init__.py               |  2 +-
 5 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/examples/local_models/json_scraper_multi_ollama.py b/examples/local_models/json_scraper_multi_ollama.py
index d3540301..2754425c 100644
--- a/examples/local_models/json_scraper_multi_ollama.py
+++ b/examples/local_models/json_scraper_multi_ollama.py
@@ -2,7 +2,8 @@
 Module for showing how PDFScraper multi works
 """
 import os 
-from scrapegraphai.graphs import PdfScraperMultiGraph
+import json
+from scrapegraphai.graphs import JSONScraperMultiGraph
 
 graph_config = {
     "llm": {
@@ -25,23 +26,14 @@ file_path = os.path.join(curr_dir, FILE_NAME)
 with open(file_path, 'r', encoding="utf-8") as file:
     text = file.read()
 
-    
-json_scraper_graph = JSONScraperGraph(
-    prompt="List me all the authors, title and genres of the books",
-    source=text,  # Pass the content of the file, not the file object
+sources = [text, text]
+
+multiple_search_graph = JSONScraperMultiGraph(
+    prompt= "List me all the authors, title and genres of the books",
+    source= sources,
+    schema=None,
     config=graph_config
 )
 
-
-
-results = []
-for source in sources:
-    pdf_scraper_graph = PdfScraperMultiGraph(
-        prompt=prompt,
-        source=source,
-        config=graph_config
-    )
-    result = pdf_scraper_graph.run()
-    results.append(result)
-
-print(results)
+result = multiple_search_graph.run()
+print(json.dumps(result, indent=4))
diff --git a/examples/local_models/pdf_scraper_multi_ollama.py b/examples/local_models/pdf_scraper_multi_ollama.py
index 77565918..c0b65a63 100644
--- a/examples/local_models/pdf_scraper_multi_ollama.py
+++ b/examples/local_models/pdf_scraper_multi_ollama.py
@@ -16,7 +16,6 @@ graph_config = {
         "temperature": 0,
     },
     "verbose": True,
-    "headless": False,
 }
 
 # Covert to list
diff --git a/scrapegraphai/graphs/__init__.py b/scrapegraphai/graphs/__init__.py
index b70686a7..37814cd1 100644
--- a/scrapegraphai/graphs/__init__.py
+++ b/scrapegraphai/graphs/__init__.py
@@ -17,4 +17,4 @@ from .omni_scraper_graph import OmniScraperGraph
 from .omni_search_graph import OmniSearchGraph
 from .smart_scraper_multi_graph import SmartScraperMultiGraph
 from .pdf_scraper_multi import PdfScraperMultiGraph
-from .json_scraper_multi import JsonScraperMultiGraph
+from .json_scraper_multi import JSONScraperMultiGraph
diff --git a/scrapegraphai/graphs/json_scraper_multi.py b/scrapegraphai/graphs/json_scraper_multi.py
index c7632d79..2010c856 100644
--- a/scrapegraphai/graphs/json_scraper_multi.py
+++ b/scrapegraphai/graphs/json_scraper_multi.py
@@ -1,5 +1,5 @@
 """ 
-JsonScraperMultiGraph Module
+JSONScraperMultiGraph Module
 """
 
 from copy import copy, deepcopy
@@ -15,9 +15,9 @@ from ..nodes import (
 )
 
 
-class JsonScraperMultiGraph(AbstractGraph):
+class JSONScraperMultiGraph(AbstractGraph):
     """ 
-    JsonScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    JSONScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:
diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py
index 4577ee86..5c54937c 100644
--- a/scrapegraphai/nodes/__init__.py
+++ b/scrapegraphai/nodes/__init__.py
@@ -19,4 +19,4 @@ from .generate_answer_csv_node import GenerateAnswerCSVNode
 from .generate_answer_pdf_node import GenerateAnswerPDFNode
 from .graph_iterator_node import GraphIteratorNode
 from .merge_answers_node import MergeAnswersNode
-from .generate_answer_omni_node import GenerateAnswerOmniNode
\ No newline at end of file
+from .generate_answer_omni_node import GenerateAnswerOmniNode