update the chunk size

Author: VinciGit00

scrapegraphai/nodes/parse_node.py

@@ -74,24 +74,24 @@ def execute(self, state: dict) -> dict:
             docs_transformed = docs_transformed[0]
 
             chunks = chunk(text=docs_transformed.page_content,
-                            chunk_size= self.node_config.get("chunk_size", 4096),
+                            chunk_size= self.node_config.get("chunk_size", 4096)-250,
                             token_counter=lambda x: len(x.split()),
                             memoize=False)
         else:
             docs_transformed = docs_transformed[0]
 
             if type(docs_transformed) == Document:
                 chunks = chunk(text=docs_transformed.page_content,
-                            chunk_size= self.node_config.get("chunk_size", 4096),
+                            chunk_size= self.node_config.get("chunk_size", 4096)-250,
                             token_counter=lambda x: len(x.split()),
                             memoize=False)
             else:
-                
+
                 chunks = chunk(text=docs_transformed,
-                                chunk_size= self.node_config.get("chunk_size", 4096),
+                                chunk_size= self.node_config.get("chunk_size", 4096)-250,
                                 token_counter=lambda x: len(x.split()),
                                 memoize=False)
-                          
+
         state.update({self.output[0]: chunks})
 
         return state