diff --git a/examples/openai/xml_scraper_openai.py b/examples/openai/xml_scraper_openai.py index 1d3b8d85..bac87ff9 100644 --- a/examples/openai/xml_scraper_openai.py +++ b/examples/openai/xml_scraper_openai.py @@ -4,7 +4,7 @@ Basic example of scraping pipeline using XMLScraperGraph from XML documents import os from dotenv import load_dotenv from scrapegraphai.graphs import XMLScraperGraph -from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info +from scrapegraphai.utils import prettify_exec_info load_dotenv() @@ -23,7 +23,7 @@ with open(file_path, 'r', encoding="utf-8") as file: # Define the configuration for the graph # ************************************************ -openai_key = os.getenv("OPENAI_APIKEY") +openai_key = os.getenv("OPENAI_API_KEY") graph_config = { "llm": { @@ -53,6 +53,3 @@ print(result) graph_exec_info = xml_scraper_graph.get_execution_info() print(prettify_exec_info(graph_exec_info)) -# Save to json or csv -convert_to_csv(result, "result") -convert_to_json(result, "result") diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py index 8f13e340..246c3194 100644 --- a/scrapegraphai/graphs/smart_scraper_graph.py +++ b/scrapegraphai/graphs/smart_scraper_graph.py @@ -91,7 +91,7 @@ class SmartScraperGraph(AbstractGraph): return response fetch_node = FetchNode( - input="url| local_dir", + input="url | local_dir", output=["doc"], node_config={ "llm_model": self.llm_model, diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index e58cfb84..e666b340 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -107,15 +107,13 @@ class FetchNode(BaseNode): if input_type in handlers: return handlers[input_type](state, input_type, source) - elif self.input == "pdf_dir": - return state - - try: + elif input_type == "local_dir": + return self.handle_local_source(state, source) + elif input_type == "url": return self.handle_web_source(state, source) - except ValueError as e: - raise + else: + raise ValueError(f"Invalid input type: {input_type}") - return self.handle_local_source(state, source) def handle_directory(self, state, input_type, source): """