CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-12 21:01:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: refactoring fetch_node

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-10-10 11:36:48 +02:00
parent 12f2b9946b
commit 39a029ed9a
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapegraphai/nodes/fetch_node.py
View File

@ -124,7 +124,7 @@ class FetchNode(BaseNode):
return handlers[input_type](state, input_type, source) return handlers[input_type](state, input_type, source)
elif self.input == "pdf_dir": elif self.input == "pdf_dir":
return state return state
elif not source.startswith("http"): elif not source.startswith("http") or not source.startswith("www"):
return self.handle_local_source(state, source) return self.handle_local_source(state, source)
else: else:
return self.handle_web_source(state, source) return self.handle_web_source(state, source)
@ -307,6 +307,7 @@ class FetchNode(BaseNode):
if not document or not document[0].page_content.strip(): if not document or not document[0].page_content.strip():
raise ValueError("""No HTML body content found in raise ValueError("""No HTML body content found in
the document fetched by ChromiumLoader.""") the document fetched by ChromiumLoader.""")
parsed_content = document[0].page_content parsed_content = document[0].page_content
if (isinstance(self.llm_model, ChatOpenAI) \ if (isinstance(self.llm_model, ChatOpenAI) \