diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py index 9c24edb6..d2d9caad 100644 --- a/scrapegraphai/nodes/parse_node.py +++ b/scrapegraphai/nodes/parse_node.py @@ -74,7 +74,7 @@ class ParseNode(BaseNode): docs_transformed = docs_transformed[0] chunks = chunk(text=docs_transformed.page_content, - chunk_size= self.node_config.get("chunk_size", 4096), + chunk_size= self.node_config.get("chunk_size", 4096)-250, token_counter=lambda x: len(x.split()), memoize=False) else: @@ -82,16 +82,16 @@ class ParseNode(BaseNode): if type(docs_transformed) == Document: chunks = chunk(text=docs_transformed.page_content, - chunk_size= self.node_config.get("chunk_size", 4096), + chunk_size= self.node_config.get("chunk_size", 4096)-250, token_counter=lambda x: len(x.split()), memoize=False) else: - + chunks = chunk(text=docs_transformed, - chunk_size= self.node_config.get("chunk_size", 4096), + chunk_size= self.node_config.get("chunk_size", 4096)-250, token_counter=lambda x: len(x.split()), memoize=False) - + state.update({self.output[0]: chunks}) return state