CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-06 21:11:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add json multiscraper

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-06-01 12:04:19 +02:00
parent 4d42d7bfc6
commit 5bda918a39
5 changed files with 15 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
examples/local_models/json_scraper_multi_ollama.py
View File

@ -2,7 +2,8 @@
Module for showing how PDFScraper multi works Module for showing how PDFScraper multi works
""" """
import os import os
from scrapegraphai.graphs import PdfScraperMultiGraph import json
from scrapegraphai.graphs import JSONScraperMultiGraph
graph_config = { graph_config = {
"llm": { "llm": {
@ -25,23 +26,14 @@ file_path = os.path.join(curr_dir, FILE_NAME)
with open(file_path, 'r', encoding="utf-8") as file: with open(file_path, 'r', encoding="utf-8") as file:
text = file.read() text = file.read()
sources = [text, text]
json_scraper_graph = JSONScraperGraph(
prompt="List me all the authors, title and genres of the books", multiple_search_graph = JSONScraperMultiGraph(
source=text, # Pass the content of the file, not the file object prompt= "List me all the authors, title and genres of the books",
source= sources,
schema=None,
config=graph_config config=graph_config
) )
result = multiple_search_graph.run()
print(json.dumps(result, indent=4))
results = []
for source in sources:
pdf_scraper_graph = PdfScraperMultiGraph(
prompt=prompt,
source=source,
config=graph_config
)
result = pdf_scraper_graph.run()
results.append(result)
print(results)

1
examples/local_models/pdf_scraper_multi_ollama.py
View File

@ -16,7 +16,6 @@ graph_config = {
"temperature": 0, "temperature": 0,
}, },
"verbose": True, "verbose": True,
"headless": False,
} }
# Covert to list # Covert to list

2
scrapegraphai/graphs/__init__.py
View File

@ -17,4 +17,4 @@ from .omni_scraper_graph import OmniScraperGraph
from .omni_search_graph import OmniSearchGraph from .omni_search_graph import OmniSearchGraph
from .smart_scraper_multi_graph import SmartScraperMultiGraph from .smart_scraper_multi_graph import SmartScraperMultiGraph
from .pdf_scraper_multi import PdfScraperMultiGraph from .pdf_scraper_multi import PdfScraperMultiGraph
from .json_scraper_multi import JsonScraperMultiGraph from .json_scraper_multi import JSONScraperMultiGraph

6
scrapegraphai/graphs/json_scraper_multi.py
View File

@ -1,5 +1,5 @@
""" """
JsonScraperMultiGraph Module JSONScraperMultiGraph Module
""" """
from copy import copy, deepcopy from copy import copy, deepcopy
@ -15,9 +15,9 @@ from ..nodes import (
) )
class JsonScraperMultiGraph(AbstractGraph): class JSONScraperMultiGraph(AbstractGraph):
""" """
JsonScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt. JSONScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
It only requires a user prompt and a list of URLs. It only requires a user prompt and a list of URLs.
Attributes: Attributes:

2
scrapegraphai/nodes/__init__.py
View File

@ -19,4 +19,4 @@ from .generate_answer_csv_node import GenerateAnswerCSVNode
from .generate_answer_pdf_node import GenerateAnswerPDFNode from .generate_answer_pdf_node import GenerateAnswerPDFNode
from .graph_iterator_node import GraphIteratorNode from .graph_iterator_node import GraphIteratorNode
from .merge_answers_node import MergeAnswersNode from .merge_answers_node import MergeAnswersNode
from .generate_answer_omni_node import GenerateAnswerOmniNode from .generate_answer_omni_node import GenerateAnswerOmniNode