CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-06 21:11:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add json multiscraper

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-06-01 12:04:19 +02:00
parent 4d42d7bfc6
commit 5bda918a39
5 changed files with 15 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
examples/local_models/json_scraper_multi_ollama.py
View File

@ -2,7 +2,8 @@
Module for showing how PDFScraper multi works Module for showing how PDFScraper multi works
""" """
import os import os
from scrapegraphai.graphs import PdfScraperMultiGraph import json
from scrapegraphai.graphs import JSONScraperMultiGraph
graph_config = { graph_config = {
"llm": { "llm": {
@ -25,23 +26,14 @@ file_path = os.path.join(curr_dir, FILE_NAME)
with open(file_path, 'r', encoding="utf-8") as file: with open(file_path, 'r', encoding="utf-8") as file:
text = file.read() text = file.read()
sources = [text, text]
json_scraper_graph = JSONScraperGraph( multiple_search_graph = JSONScraperMultiGraph(
prompt="List me all the authors, title and genres of the books", prompt= "List me all the authors, title and genres of the books",
source=text, # Pass the content of the file, not the file object source= sources,
schema=None,
config=graph_config config=graph_config
) )
result = multiple_search_graph.run()
print(json.dumps(result, indent=4))
results = []
for source in sources:
pdf_scraper_graph = PdfScraperMultiGraph(
prompt=prompt,
source=source,
config=graph_config
)
result = pdf_scraper_graph.run()
results.append(result)
print(results)

1
examples/local_models/pdf_scraper_multi_ollama.py
View File

@ -16,7 +16,6 @@ graph_config = {
"temperature": 0, "temperature": 0,
}, },
"verbose": True, "verbose": True,
"headless": False,
} }
# Covert to list # Covert to list

2
scrapegraphai/graphs/__init__.py
View File

@ -17,4 +17,4 @@ from .omni_scraper_graph import OmniScraperGraph
from .omni_search_graph import OmniSearchGraph from .omni_search_graph import OmniSearchGraph
from .smart_scraper_multi_graph import SmartScraperMultiGraph from .smart_scraper_multi_graph import SmartScraperMultiGraph
from .pdf_scraper_multi import PdfScraperMultiGraph from .pdf_scraper_multi import PdfScraperMultiGraph
from .json_scraper_multi import JsonScraperMultiGraph from .json_scraper_multi import JSONScraperMultiGraph

6
scrapegraphai/graphs/json_scraper_multi.py
View File

@ -1,5 +1,5 @@
""" """
JsonScraperMultiGraph Module JSONScraperMultiGraph Module
""" """
from copy import copy, deepcopy from copy import copy, deepcopy
@ -15,9 +15,9 @@ from ..nodes import (
) )
class JsonScraperMultiGraph(AbstractGraph): class JSONScraperMultiGraph(AbstractGraph):
""" """
JsonScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt. JSONScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
It only requires a user prompt and a list of URLs. It only requires a user prompt and a list of URLs.
Attributes: Attributes: