CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-06-28 21:01:55 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update parse_node.py
Some checks failed
/ build (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-07-17 23:06:05 +02:00
parent 07f1e23d23
commit 032a491605
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapegraphai/nodes/parse_node.py
View File

@ -74,7 +74,7 @@ class ParseNode(BaseNode):
docs_transformed = docs_transformed[0]
chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096),
chunk_size= self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda x: len(x),
memoize=False)
else:
@ -82,13 +82,13 @@ class ParseNode(BaseNode):
if type(docs_transformed) == Document:
chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096),
chunk_size= self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda x: len(x),
memoize=False)
else:
chunks = chunk(text=docs_transformed,
chunk_size= self.node_config.get("chunk_size", 4096),
chunk_size= self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda x: len(x),
memoize=False)