flush based on bytes

Author: tsmathis

mp_api/client/core/client.py

@@ -637,9 +637,10 @@ def _flush(accumulator, group):
                     accumulator = []
                     for page in iterator:
                         # arro3 rb to pyarrow rb for compat w/ pyarrow ds writer
-                        accumulator.append(pa.record_batch(page))
+                        rg = pa.record_batch(page)
+                        accumulator.append(rg)
                         page_size = page.num_rows
-                        size += page_size
+                        size += rg.get_total_buffer_size()
 
                         if pbar is not None:
                             pbar.update(page_size)

mp_api/client/core/settings.py

@@ -95,8 +95,8 @@ class MAPIClientSettings(BaseSettings):
     )
 
     DATASET_FLUSH_THRESHOLD: int = Field(
-        100000,
-        description="Threshold number of rows to accumulate in memory before flushing dataset to disk",
+        int(2.75 * 1024**3),
+        description="Threshold bytes to accumulate in memory before flushing dataset to disk",
     )
 
     model_config = SettingsConfigDict(env_prefix="MPRESTER_")