tweaks

Author: kylemann16

src/silvimetric/commands/scan.py

@@ -4,7 +4,6 @@
 import dask
 import math
 import json
-import itertools
 
 from dask.diagnostics import ProgressBar
 
@@ -52,15 +51,24 @@ def scan(
             extents = Extents.from_sub(tdb_dir, data.bounds)
             logger.info('Gathering initial chunks...')
             count = dask.delayed(data.estimate_count)(extents.bounds).persist()
-
             cell_counts = extent_handle(
                 extents, data, resolution, point_count, depth, log
             )
 
-            num_cells = np.sum(cell_counts).item()
-            std = np.std(cell_counts)
-            mean = np.mean(cell_counts)
-            rec = int(mean)
+            np_cell_counts = np.array(cell_counts)
+            num_cells = np.sum(np_cell_counts).item()
+            q1, q3 = np.percentile(np_cell_counts, [25,75])
+            iqr = q3 - q1
+            low_bounds = q1 - (1.5 * iqr)
+            up_bounds = q3 + (1.5 * iqr)
+
+            adjusted = np_cell_counts[np_cell_counts > low_bounds]
+            adjusted = adjusted[adjusted < up_bounds]
+
+            std = np.std(adjusted)
+            mean = np.mean(adjusted)
+            median = np.median(adjusted)
+            rec = median
 
             pc_info = dict(
                 pc_info=dict(
@@ -75,6 +83,7 @@ def scan(
                     num_tiles=len(cell_counts),
                     mean=mean,
                     std_dev=std,
+                    median=median,
                     recommended=rec,
                 )
             )

src/silvimetric/commands/shatter.py

@@ -161,11 +161,12 @@ def kill_gracefully(signum, frame):
     if dc is not None:
         processes = []
         count = 0
-        for leaf_bunch in itertools.batched(leaves, consolidate_count):
+        for leaf in leaves:
             count = count + 1
-            processes.append(dc.map(do_one, leaf_bunch, config=config, storage=storage))
+            processes.append(dc.submit(do_one, leaf, config=config, storage=storage))
+            if count % consolidate_count == 0:
+                processes.append(dc.submit(storage.consolidate_shatter, timestamp=config.timestamp, key=f'consolidate_{count}'))
 
-            processes.append(dc.submit(storage.consolidate_shatter, config.timestamp))
         gathered = dc.gather(processes)
         point_count = 0
         for pc in gathered:
@@ -234,9 +235,8 @@ def shatter(config: ShatterConfig) -> int:
     if config.tile_size is not None:
         leaves = extents.get_leaf_children(config.tile_size)
     else:
-        leaves = extents.chunk(data)
+        leaves = itertools.chain(extents.chunk(data))
 
-    leaves = itertools.chain(leaves)
     # Begin main operations
     config.log.debug('Fetching and arranging data...')
     storage.save_shatter_meta(config)

src/silvimetric/resources/metrics/grid_metrics.py

@@ -52,14 +52,14 @@ def _get_grid_metrics(elev_key='Z'):
 
     # give profile_area separate pct_base so we can apply separate filters
     percentiles['profile_area'].attributes = [A[elev_key]]
+    percentiles['iq'].attributes = [A[elev_key], A['Intensity']]
     # pct_base_copy = copy.deepcopy(pct_base)
     # pct_base_copy.name = 'pct_base_profile_area'
     # percentiles['profile_area'].dependencies = [pct_base_copy]
 
     statistics['cumean'].attributes = [A[elev_key]]
     statistics['sqmean'].attributes = [A[elev_key]]
 
-    statistics['iq'].attributes = [A[elev_key], A['Intensity']]
     statistics['min'].attributes = [A[elev_key], A['Intensity']]
     statistics['max'].attributes = [A[elev_key], A['Intensity']]
     statistics['mode'].attributes = [A[elev_key], A['Intensity']]

src/silvimetric/resources/metrics/p_moments.py

@@ -11,28 +11,28 @@ def m_mean(data, *args):
 
 def m_variance(data, *args):
     # copy FUSION's variance approach
-    num = ((data - data.mean()) ** 2).sum()
     denom = (data.count() - 1)
     if denom == 0:
         return np.nan
+    num = ((data - data.mean()) ** 2).sum()
     return num / denom
 
 
 def m_skewness(data, *args):
     # copy FUSION's approximation of skewness
-    num = ((data - data.mean()) ** 3).sum()
     denom = ( (data.count() - 1) * np.std(data) ** 3)
     if denom == 0:
         return np.nan
+    num = ((data - data.mean()) ** 3).sum()
     return  num / denom
 
 
 def m_kurtosis(data, *args):
     # copy FUSION's approximation of kurtosis
-    num = ((data - data.mean()) ** 4).sum()
     denom = ((data.count() - 1) * np.std(data) ** 4)
     if denom == 0:
         return np.nan
+    num = ((data - data.mean()) ** 4).sum()
     return num / denom
 
 

src/silvimetric/resources/metrics/percentiles.py

@@ -75,6 +75,12 @@ def m_90m10(data, *args):
 def m_95m05(data, *args):
     return args[0][13] - args[0][1]
 
+def m_iq(data, *args):
+    q1 = args[0][4]
+    q3 = args[0][10]
+    return q3 - q1
+
+
 def m_profile_area(data, *args):
     # sanity check...must have valid heights/elevations
     p = np.percentile(data, range(0,100)).tolist()
@@ -125,4 +131,5 @@ def m_profile_area(data, *args):
 percentiles['p99'] = Metric('p99', np.float32, m_p99, [pct_base])
 percentiles['90m10'] = Metric('90m10', np.float32, m_90m10, [pct_base])
 percentiles['95m05'] = Metric('95m05', np.float32, m_95m05, [pct_base])
-percentiles['profile_area'] = Metric('profile_area', np.float32, m_profile_area)
+percentiles['profile_area'] = Metric('profile_area', np.float32, m_profile_area)
+percentiles['iq'] = Metric('iq', np.float32, m_iq, [pct_base])

src/silvimetric/resources/metrics/stats.py

@@ -42,11 +42,6 @@ def m_cv(data, *args):
         return np.nan
     return stddev / mean
 
-
-def m_iq(data):
-    return stats.iqr(data)
-
-
 def m_crr(data, *args):
     mean, minimum, maximum = args
     den = maximum - minimum
@@ -80,7 +75,6 @@ def m_mad_mode(data, *args):
 maximum = Metric('max', np.float32, m_max)
 stddev = Metric('stddev', np.float32, m_stddev)
 cv = Metric('cv', np.float32, m_cv, [stddev, mean])
-iq = Metric('iq', np.float32, m_iq)
 crr = Metric('canopy_relief_ratio', np.float32, m_crr, [mean, minimum, maximum])
 sqmean = Metric('sqmean', np.float32, m_sqmean)
 cumean = Metric('cumean', np.float32, m_cumean)
@@ -92,7 +86,6 @@ def m_mad_mode(data, *args):
     max=maximum,
     stddev=stddev,
     cv=cv,
-    iq=iq,
     crr=crr,
     sqmean=sqmean,
     cumean=cumean,

tests/test_data.py

@@ -89,7 +89,7 @@ def test_chunking(
             autzen_storage.alignment,
             autzen_storage.root,
         )
-        chs1 = ex.chunk(autzen_data, pc_threshold=600000)
+        chs1 = ex.chunk(autzen_data, pc_threshold=100000)
         chs2 = ex.get_leaf_children(50)
         for c in chs1:
             assert isinstance(c, Extents)