CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-06 21:11:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

update the chunk size
Some checks are pending
/ build (3.10) (push) Waiting to run
Details

Browse Source
This commit is contained in:
Marco Vinciguerra 2024-07-02 12:03:08 +02:00
parent 3bf5f570a8
commit ed2af51501
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapegraphai/nodes/parse_node.py
View File

@ -74,7 +74,7 @@ class ParseNode(BaseNode):
docs_transformed = docs_transformed[0]
chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096),
chunk_size= self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda x: len(x.split()),
memoize=False)
else:
@ -82,16 +82,16 @@ class ParseNode(BaseNode):
if type(docs_transformed) == Document:
chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096),
chunk_size= self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda x: len(x.split()),
memoize=False)
else:
chunks = chunk(text=docs_transformed,
chunk_size= self.node_config.get("chunk_size", 4096),
chunk_size= self.node_config.get("chunk_size", 4096)-250,
token_counter=lambda x: len(x.split()),
memoize=False)
state.update({self.output[0]: chunks})
return state