CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-23 21:00:30 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: add json integration

Browse Source
This commit is contained in:
VinciGit00 2024-05-09 21:07:07 +02:00
parent 28c9dce7cb
commit 0ab31c3fdb
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapegraphai/graphs/json_scraper_graph.py
View File

@ -54,7 +54,7 @@ class JSONScraperGraph(AbstractGraph):
"""
fetch_node = FetchNode(
input="json_dir",
input="json",
output=["doc"],
)
parse_node = ParseNode(
@ -106,4 +106,4 @@ class JSONScraperGraph(AbstractGraph):
inputs = {"user_prompt": self.prompt, self.input_key: self.source}
self.final_state, self.execution_info = self.graph.execute(inputs)
return self.final_state.get("answer", "No answer found.")
return self.final_state.get("answer", "No answer found.")

10
scrapegraphai/nodes/fetch_node.py
View File

@ -2,6 +2,7 @@
FetchNode Module
"""
import pandas as pd
import json
from typing import List, Optional
from langchain_community.document_loaders import AsyncChromiumLoader
from langchain_core.documents import Document
@ -75,8 +76,13 @@ class FetchNode(BaseNode):
compressed_document = loader.load()
elif self.input == "csv":
compressed_document = [Document(page_content=pd.read_csv(source), metadata={
"source": "xml"
compressed_document = [Document(page_content=str(pd.read_csv(source)), metadata={
"source": "csv"
})]
elif self.input == "json":
f = open(source)
compressed_document = [Document(page_content=str(json.load(f)), metadata={
"source": "json"
})]
elif self.input == "xml":
with open(source, 'r', encoding='utf-8') as f: