CodeCow/Scrapegraph-ai
1
0
Fork 0
mirror of https://github.com/VinciGit00/Scrapegraph-ai.git synced 2026-07-01 21:00:48 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix(chunking): count tokens from words instead of characters

Browse Source 
closes #513
This commit is contained in:
Federico Aguzzi 2024-08-08 10:47:08 +02:00
parent 7f1f7503f7
commit 5ec2de9e1a
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapegraphai/nodes/parse_node.py
View File

@ -74,22 +74,22 @@ class ParseNode(BaseNode):
docs_transformed = docs_transformed[0] docs_transformed = docs_transformed[0]
chunks = chunk(text=docs_transformed.page_content, chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096)-250, chunk_size=self.node_config.get("chunk_size", 4096)-250,
token_counter= lambda x: len(x), token_counter=lambda text: len(text.split()),
memoize=False) memoize=False)
else: else:
docs_transformed = docs_transformed[0] docs_transformed = docs_transformed[0]
if isinstance(docs_transformed, Document): if isinstance(docs_transformed, Document):
chunks = chunk(text=docs_transformed.page_content, chunks = chunk(text=docs_transformed.page_content,
chunk_size= self.node_config.get("chunk_size", 4096)-250, chunk_size=self.node_config.get("chunk_size", 4096)-250,
token_counter= lambda x: len(x), token_counter=lambda text: len(text.split()),
memoize=False) memoize=False)
else: else:
chunks = chunk(text=docs_transformed, chunks = chunk(text=docs_transformed,
chunk_size= self.node_config.get("chunk_size", 4096)-250, chunk_size=self.node_config.get("chunk_size", 4096)-250,
token_counter= lambda x: len(x), token_counter=lambda text: len(text.split()),
memoize=False) memoize=False)
state.update({self.output[0]: chunks}) state.update({self.output[0]: chunks})