CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: parse_node

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-09-24 15:27:20 +02:00
parent bc881b4aaa
commit ceede46673
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapegraphai/nodes/parse_node.py
View File

@ -88,7 +88,7 @@ class ParseNode(BaseNode):
link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
chunk_size = self.chunk_size
chunk_size = min(chunk_size - 500, int(chunk_size * 0.9))
chunk_size = min(chunk_size - 500, int(chunk_size * 0.75))
if isinstance(docs_transformed, Document):
chunks = split_text_into_chunks(text=docs_transformed.page_content,