Merge pull request #2853 from cta-observatory/remove-index-tables

Remove write_index_tables option of DataWriter, seems to corrupt files

Author: kosack

docs/changes/2853.api.rst

@@ -0,0 +1,4 @@
+Remove the option ``DataWriter.write_index_tables`` along with its alias
+in ``ctapipe-process --write-index-tables``.
+The option seems to corrupt output files for large data sizes, is not used
+in ctapipe itself and was not tested.

src/ctapipe/io/datawriter.py

@@ -218,16 +218,6 @@ class DataWriter(Component):
         default_value="blosc:zstd",
     ).tag(config=True)
 
-    write_index_tables = Bool(
-        help=(
-            "Generate PyTables index datasets for all tables that contain an "
-            "event_id or tel_id. These speed up in-kernel pytables operations,"
-            "but add some overhead to the file. They can also be generated "
-            "and attached after the file is written "
-        ),
-        default_value=False,
-    ).tag(config=True)
-
     overwrite = Bool(help="overwrite output file if it exists").tag(config=True)
 
     transform_waveform = Bool(default_value=False).tag(config=True)
@@ -374,9 +364,6 @@ def finish(self):
         if not self._at_least_one_event:
             self.log.warning("No events have been written to the output file")
 
-        if self.write_index_tables:
-            self._generate_indices()
-
         write_reference_metadata_headers(
             subarray=self._subarray,
             obs_ids=self.event_source.obs_ids,
@@ -719,46 +706,6 @@ def _write_dl2_stereo_event(self, event: ArrayEventContainer):
                     containers=[event.index, container],
                 )
 
-    def _generate_table_indices(self, h5file, start_node):
-        """helper to generate PyTables index tabnles for common columns"""
-        for node in h5file.iter_nodes(start_node):
-            if not isinstance(node, tables.group.Group):
-                self.log.debug("generating indices for node: %s", node)
-                if "event_id" in node.colnames:
-                    node.cols.event_id.create_index()
-                    self.log.debug("generated event_id index")
-                if "tel_id" in node.colnames:
-                    node.cols.tel_id.create_index()
-                    self.log.debug("generated tel_id index")
-                if "obs_id" in node.colnames:
-                    self.log.debug("generated obs_id index")
-                    node.cols.obs_id.create_index(kind="ultralight")
-            else:
-                # recurse
-                self._generate_table_indices(h5file, node)
-
-    def _generate_indices(self):
-        """generate PyTables index tables for common columns"""
-        self.log.debug("Writing index tables")
-        if self.write_dl1_images:
-            self._generate_table_indices(
-                self._writer.h5file, "/dl1/event/telescope/images"
-            )
-            if self._is_simulation:
-                self._generate_table_indices(
-                    self._writer.h5file, "/simulation/event/telescope/images"
-                )
-        if self.write_dl1_parameters:
-            self._generate_table_indices(
-                self._writer.h5file, "/dl1/event/telescope/parameters"
-            )
-            if self._is_simulation:
-                self._generate_table_indices(
-                    self._writer.h5file, "/simulation/event/telescope/parameters"
-                )
-
-        self._generate_table_indices(self._writer.h5file, "/dl1/event/subarray")
-
     def _write_context_metadata_headers(self):
         """write out any user-defined metadata in the context_metadata field to the
         headers.

src/ctapipe/tools/process.py

@@ -151,11 +151,6 @@ class ProcessorTool(Tool):
             "store DL2/Event parameters in output",
             "don't DL2/Event parameters in output",
         ),
-        **flag(
-            "write-index-tables",
-            "DataWriter.write_index_tables",
-            "generate PyTables index tables for the parameter and image datasets",
-        ),
         **flag(
             "write-muon-parameters",
             "DataWriter.write_muon_parameters",