From 39a029ed9a8cd7c2277ba1386b976738e99d231b Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 10 Oct 2024 11:36:48 +0200 Subject: [PATCH] feat: refactoring fetch_node --- scrapegraphai/nodes/fetch_node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py index a548e05b..fbd94a83 100644 --- a/scrapegraphai/nodes/fetch_node.py +++ b/scrapegraphai/nodes/fetch_node.py @@ -124,7 +124,7 @@ class FetchNode(BaseNode): return handlers[input_type](state, input_type, source) elif self.input == "pdf_dir": return state - elif not source.startswith("http"): + elif not source.startswith("http") or not source.startswith("www"): return self.handle_local_source(state, source) else: return self.handle_web_source(state, source) @@ -307,6 +307,7 @@ class FetchNode(BaseNode): if not document or not document[0].page_content.strip(): raise ValueError("""No HTML body content found in the document fetched by ChromiumLoader.""") + parsed_content = document[0].page_content if (isinstance(self.llm_model, ChatOpenAI) \