Skip to content

Commit 71001a2

Browse files
committed
wip, adding other consolidation modes
1 parent d99e88d commit 71001a2

File tree

1 file changed

+16
-2
lines changed

1 file changed

+16
-2
lines changed

src/silvimetric/resources/storage.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ def ts_overlap(first, second):
2121
return False
2222
return True
2323

24+
2425
class Storage:
2526
"""Handles storage of shattered data in a TileDB Database."""
2627

@@ -479,10 +480,23 @@ def consolidate_shatter(self, timestamp, retries=0) -> None:
479480
"""
480481
self.config.log.info('Consolidating db.')
481482
try:
482-
tiledb.consolidate(self.config.tdb_dir, timestamp=timestamp)
483+
commits = tiledb.Config({'sm.consolidation.mode': 'commits'})
484+
tiledb.consolidate(
485+
self.config.tdb_dir, timestamp=timestamp, config=commits
486+
)
487+
fragments = tiledb.Config({'sm.consolidation.mode': 'fragments'})
488+
tiledb.consolidate(
489+
self.config.tdb_dir, timestamp=timestamp, config=fragments
490+
)
491+
metadata = tiledb.Config({'sm.consolidation.mode': 'metadata'})
492+
tiledb.consolidate(
493+
self.config.tdb_dir, timestamp=timestamp, config=metadata
494+
)
495+
483496
c = tiledb.Config({'sm.vacuum.mode': 'fragments'})
484-
tiledb.vacuum(self.config.tdb_dir, c)
497+
tiledb.vacuum(self.config.tdb_dir, timestamp=timestamp, config=c)
485498
self.config.log.info(f'Consolidated time slot {timestamp}.')
499+
486500
except Exception as e:
487501
if retries >= 3:
488502
self.config.log.warning(

0 commit comments

Comments
 (0)