CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-09 21:19:20 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: local html handling

Browse Source
This commit is contained in:
PeriniM 2025-01-06 12:04:02 +01:00
parent 013a196629
commit 2a15581865
3 changed files with 8 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
examples/openai/xml_scraper_openai.py
View File

@ -4,7 +4,7 @@ Basic example of scraping pipeline using XMLScraperGraph from XML documents
import os import os
from dotenv import load_dotenv from dotenv import load_dotenv
from scrapegraphai.graphs import XMLScraperGraph from scrapegraphai.graphs import XMLScraperGraph
from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info from scrapegraphai.utils import prettify_exec_info
load_dotenv() load_dotenv()
@ -23,7 +23,7 @@ with open(file_path, 'r', encoding="utf-8") as file:
# Define the configuration for the graph # Define the configuration for the graph
# ************************************************ # ************************************************
openai_key = os.getenv("OPENAI_APIKEY") openai_key = os.getenv("OPENAI_API_KEY")
graph_config = { graph_config = {
"llm": { "llm": {
@ -53,6 +53,3 @@ print(result)
graph_exec_info = xml_scraper_graph.get_execution_info() graph_exec_info = xml_scraper_graph.get_execution_info()
print(prettify_exec_info(graph_exec_info)) print(prettify_exec_info(graph_exec_info))
# Save to json or csv
convert_to_csv(result, "result")
convert_to_json(result, "result")

2
scrapegraphai/graphs/smart_scraper_graph.py
View File

@ -91,7 +91,7 @@ class SmartScraperGraph(AbstractGraph):
return response return response
fetch_node = FetchNode( fetch_node = FetchNode(
input="url| local_dir", input="url | local_dir",
output=["doc"], output=["doc"],
node_config={ node_config={
"llm_model": self.llm_model, "llm_model": self.llm_model,

12
scrapegraphai/nodes/fetch_node.py
View File

@ -107,15 +107,13 @@ class FetchNode(BaseNode):
if input_type in handlers: if input_type in handlers:
return handlers[input_type](state, input_type, source) return handlers[input_type](state, input_type, source)
elif self.input == "pdf_dir": elif input_type == "local_dir":
return state return self.handle_local_source(state, source)
elif input_type == "url":
try:
return self.handle_web_source(state, source) return self.handle_web_source(state, source)
except ValueError as e: else:
raise raise ValueError(f"Invalid input type: {input_type}")
return self.handle_local_source(state, source)
def handle_directory(self, state, input_type, source): def handle_directory(self, state, input_type, source):
""" """