adds tqdm and new row inversion method

Author: jmbhughes

CHANGELOG.md

@@ -1,8 +1,22 @@
 # Changelog
 
+Follows [Keep A Changelog format](https://keepachangelog.com/en/1.1.0/)
+
+## 0.0.3
+
+### Added
+
+- Uses tqdm for progress bar tracking
+- Adds better row mode instead of starting chunked and instantly switching to row
+
+### Changed
+
+- Prints elapsed seconds as integer instead of float
+- Expands documentation for missing parameters
+
 ## 0.0.2
 
-### New features
+### Added
 
 - Writes scores to a text file
 

example_config.toml

@@ -12,14 +12,14 @@ overwrite = true
 
 [inversion]
 solution_fov_width = 2
-detector_row_range = [300, 400] #[0, 792]
+detector_row_range = [0, 50]
 field_angle_range = [-1227, 1227]
 response_dependency_name = "logt"
 response_dependency_list = [5.7, 5.8, 5.9, 6.0 , 6.1, 6.2, 6.3, 6.4, 6.5, 6.6, 6.7, 6.8]
 smooth_over = 'dependence'
 
 [model]
-alphas = [3E-5] #[0.2, 0.1, 0.01, 0.005]
+alphas = [3E-5, 4E-5, 0.1] #[0.2, 0.1, 0.01, 0.005]
 rhos = [0.1]
 warm_start = false
 tol = 1E-2

overlappogram/cli.py

@@ -21,7 +21,10 @@
 @click.command()
 @click.argument("config")
 def unfold(config):
-    """Unfold an overlappogram given a configuration toml file."""  # TODO improve message
+    """Unfold an overlappogram given a configuration toml file.
+
+    See https://eccco-mission.github.io/overlappogram/configuration.html for the configuration file format.
+    """
 
     with open(config) as f:
         config = toml.load(f)
@@ -42,6 +45,8 @@ def unfold(config):
 
     for alpha in config["model"]["alphas"]:
         for rho in config["model"]["rhos"]:
+            print(80*"-")
+            print(f"Beginning inversion for alpha={alpha}, rho={rho}.")
             start = time.time()
             em_cube, prediction, scores, unconverged_rows = inversion.invert(
                 overlappogram,
@@ -54,9 +59,9 @@ def unfold(config):
             )
             end = time.time()
             print(
-                f"Inversion Time for alpha={alpha}, rho={rho}:",
-                end - start,
-                f"; {len(unconverged_rows)} unconverged rows",
+                f"Inversion for alpha={alpha}, rho={rho} took",
+                int(end - start),
+                f"seconds; {len(unconverged_rows)} unconverged rows",
             )
 
             postfix = (

overlappogram/inversion.py

@@ -9,6 +9,7 @@
 from ndcube import NDCube
 from sklearn.exceptions import ConvergenceWarning
 from sklearn.linear_model import ElasticNet
+from tqdm import tqdm
 
 from overlappogram.response import prepare_response_function
 
@@ -60,6 +61,8 @@ def __init__(
             response_dependency_list=response_dependency_list,
         )
 
+        self._progress_bar = None  # initialized in invert call
+
     @property
     def is_inverted(self) -> bool:
         return not any(
@@ -107,7 +110,8 @@ def _progress_indicator(self, future):
         with self._thread_count_lock:
             if not future.cancelled():
                 self._completed_row_count += 1
-                print(f"{self._completed_row_count / self.total_row_count * 100:3.0f}% complete", end="\r")
+                #print(f"{self._completed_row_count / self.total_row_count * 100:3.0f}% complete", end="\r")
+                self._progress_bar.update(1)
 
     def _switch_to_row_inversion(self, model_config, alpha, rho, num_row_threads=50):
         self._mode = InversionMode.ROW
@@ -161,6 +165,29 @@ def _collect_results(self, mode_switch_thread_count, model_config, alpha, rho):
                 self._switch_to_row_inversion(model_config, alpha, rho)
                 break
 
+    def _start_row_inversion(self, model_config, alpha, rho, num_threads):
+        self.executors = [concurrent.futures.ThreadPoolExecutor(max_workers=num_threads)]
+
+        self.futures = {}
+        self._models = []
+        for i, row_index in enumerate(range(self._detector_row_range[0], self._detector_row_range[1])):
+            enet_model = ElasticNet(
+                alpha=alpha,
+                l1_ratio=rho,
+                tol=model_config["tol"],
+                max_iter=model_config["max_iter"],
+                precompute=False,  # setting this to true slows down performance dramatically
+                positive=True,
+                copy_X=False,
+                fit_intercept=False,
+                selection=model_config["selection"],
+                warm_start=False,  # warm start doesn't make sense in the row version
+            )
+            self._models.append(enet_model)
+            future = self.executors[-1].submit(self._invert_image_row, row_index, i)
+            future.add_done_callback(self._progress_indicator)
+            self.futures[future] = (row_index, i)
+
     def _start_chunk_inversion(self, model_config, alpha, rho, num_threads):
         starts = np.arange(
             self._detector_row_range[0],
@@ -255,6 +282,8 @@ def invert(
 
         self._mode = mode
 
+        self._progress_bar = tqdm(total=self.total_row_count, unit='row', delay=1, leave=False)
+
         self._models = []
         self._completed_row_count = 0
 
@@ -269,16 +298,16 @@ def invert(
             # mode never switches since count=0
             self._collect_results(0, model_config, alpha, rho)
         elif self._mode == InversionMode.ROW:
-            self._start_chunk_inversion(model_config, alpha, rho, num_threads)
-            # TODO: it would be better to have a mode to start in row but right now we fake it with a fast mode switch
-            self._collect_results(np.inf, model_config, alpha, rho)  # immediately switch mode
+            self._start_row_inversion(model_config, alpha, rho, num_threads)
             self._collect_results(np.inf, model_config, alpha, rho)
         else:
             raise ValueError("Invalid InversionMode.")
 
         for executor in self.executors:
             executor.shutdown()
 
+        self._progress_bar.close()
+
         return (
             np.transpose(self._em_data, (2, 0, 1)),
             self._inversion_prediction,

overlappogram/spectral.py

@@ -47,12 +47,8 @@ def create_spectrally_pure_images(image_list: list, gnt_path: str, rsp_dep_list:
             for index in range(len(image_list)):
                 # Create spectrally pure data cube.
                 for em_data in image_list:
-                    # with fits.open(image_list[index]) as em_hdul:
-                    # em_data_cube = em_hdul[0].data.astype(np.float64)
                     em_data_cube = em_data.astype(np.float64)
                     em_data_cube = np.transpose(em_data_cube, axes=(1, 2, 0))
-                    # em_dep_list = em_hdul[1].data['logt']
-                    # print(em_dep_list)
                     if index == 0:
                         image_height, num_slits, num_logts = np.shape(em_data_cube)
                         gnt_data_cube = np.zeros((image_height, num_slits, num_gnts), dtype=np.float64)

pyproject.toml

@@ -5,7 +5,7 @@ requires = ["setuptools",
 
 [project]
 name = "overlappogram"
-version = "0.0.2"
+version = "0.0.3"
 dependencies = ["numpy",
     "astropy",
     "scikit-learn",
@@ -16,7 +16,8 @@ dependencies = ["numpy",
     "scipy",
     "ndcube",
     "toml",
-    "click"
+    "click",
+    "tqdm"
 ]
 requires-python = ">=3.9"
 authors = [