Merge pull request #269 from raphaelrpl/patch-landsat-0.8

Fix bug related package install and landsat C2 L2 cubes (confidence)

Author: raphaelrpl

CHANGES.rst

@@ -21,6 +21,14 @@ Changes
 =======
 
 
+Version 0.8.6 (2023-07-14)
+--------------------------
+
+- Fix generation of data cubes for Landsat program: confidence mask
+- Improve cube builder forms for extra parameters
+- Customize number of GDAL parallel threads to generate COGs
+
+
 Version 0.8.5 (2023-03-08)
 --------------------------
 

Dockerfile

@@ -27,14 +27,14 @@ LABEL "org.brazildatacube.description"="Docker image for Data Cube Builder appli
 LABEL "org.brazildatacube.git_commit"="${GIT_COMMIT}"
 
 # Build arguments
-ARG CUBE_BUILDER_VERSION="0.8.5"
+ARG CUBE_BUILDER_VERSION="0.8.6"
 ARG CUBE_BUILDER_INSTALL_PATH="/opt/cube-builder/${CUBE_BUILDER_VERSION}"
 
 ADD . ${CUBE_BUILDER_INSTALL_PATH}
 
 WORKDIR ${CUBE_BUILDER_INSTALL_PATH}
 
-RUN python3 -m pip install pip --upgrade "setuptools<67" wheel && \
+RUN python3 -m pip install -U pip "setuptools<67" wheel && \
     python3 -m pip install -e .[rabbitmq] && \
     python3 -m pip install gunicorn
 

cube_builder/celery/__init__.py

@@ -27,7 +27,7 @@
 from celery import Celery
 from flask import Flask
 
-from cube_builder.config import Config
+from ..config import Config
 from ..constants import to_bool
 
 CELERY_TASKS = [

cube_builder/config.py

@@ -19,8 +19,8 @@
 
 import os
 
-from .version import __version__
 from .constants import to_bool
+from .version import __version__
 
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 
@@ -72,6 +72,13 @@ class Config:
     )
     """The database URI that should be used for the database connection.
     Defaults to ``'postgresql://postgres:postgres@localhost:5432/bdc_catalog'``."""
+    SQLALCHEMY_ENGINE_OPTIONS = {
+        "pool_size": int(os.getenv("SQLALCHEMY_ENGINE_POOL_SIZE", 2)),
+        "max_overflow": int(os.getenv("SQLALCHEMY_ENGINE_MAX_OVERFLOW", 5)),
+        "poolclass": os.getenv("SQLALCHEMY_ENGINE_POOL_CLASS"),
+        "pool_recycle": int(os.getenv("SQLALCHEMY_ENGINE_POOL_RECYCLE", -1)),
+    }
+
     STAC_URL = os.environ.get('STAC_URL', 'https://brazildatacube.dpi.inpe.br/stac/')
     MAX_THREADS_IMAGE_VALIDATOR = int(os.environ.get('MAX_THREADS_IMAGE_VALIDATOR', os.cpu_count()))
     # rasterio

cube_builder/forms.py

@@ -108,6 +108,13 @@ class CustomMaskDefinition(Schema):
     bits = fields.Boolean(required=False, allow_none=False, dump_default=False)
 
 
+class CustomScaleSchema(Schema):
+    """Represent the Band attribute scale factor and scale mult."""
+
+    add = fields.Float(required=True, allow_none=False, allow_nan=False)
+    mult = fields.Float(required=True, allow_none=False, allow_nan=False)
+
+
 class CubeParametersSchema(Schema):
     """Represent the data cube parameters used to be attached to the cube execution."""
 
@@ -116,6 +123,15 @@ class CubeParametersSchema(Schema):
     histogram_matching = fields.Bool(required=False, allow_none=False)
     no_post_process = fields.Bool(required=False, allow_none=False)
     band_map = fields.Dict(required=False, allow_none=False)
+    channel_limits = fields.List(
+        fields.List(fields.Float, many=True, validate=fields.Length(equal=2)),
+        required=False,
+        allow_none=False,
+        validate=fields.Length(min=1, max=3)
+    )
+    combined = fields.Boolean(required=False, allow_none=False)
+    resampling = fields.String(required=False, allow_none=False, default="nearest")
+    scale = fields.Nested(CustomScaleSchema, required=False, allow_none=False)
 
 
 class DataCubeForm(Schema):

cube_builder/utils/image.py

@@ -30,7 +30,7 @@
 import rasterio.features
 import rasterio.warp
 import shapely.geometry
-from rasterio._warp import Affine
+from rasterio.warp import Affine, Resampling
 from rio_cogeo.cogeo import cog_translate
 from rio_cogeo.profiles import cog_profiles
 from sqlalchemy.engine.result import ResultProxy, RowProxy
@@ -354,9 +354,13 @@ def generate_cogs(input_data_set_path, file_path, profile='deflate', block_size=
         output_profile["blockxsize"] = block_size
         output_profile["blockysize"] = block_size
 
+    threads = os.getenv("GDAL_NUM_THREADS", "2")
+    if threads.isnumeric():
+        threads = int(threads)
+
     # Dataset Open option (see gdalwarp `-oo` option)
     config = dict(
-        GDAL_NUM_THREADS="ALL_CPUS",
+        GDAL_NUM_THREADS=threads,
         GDAL_TIFF_INTERNAL_MASK=True,
         GDAL_TIFF_OVR_BLOCKSIZE="128",
     )
@@ -730,3 +734,19 @@ def linear_raster_scale(array: ArrayType,
     data = data * (output_range[1] - output_range[0]) + output_range[0]
 
     return data
+
+
+def get_resample_method(name: str) -> Resampling:
+    """Retrieve a resampling method from name.
+
+    Note:
+        This method uses ``rasterio.warp.Resampling``.
+
+    Args:
+        name: The resampling name
+    """
+    supported = {entry.name: entry for entry in list(Resampling)}
+    lowered = name.lower()
+    if lowered not in supported:
+        raise RuntimeError(f"Invalid resampling method. Use one of {supported.keys()}")
+    return supported[lowered]

cube_builder/utils/processing.py

@@ -50,8 +50,8 @@
                          TOTAL_OBSERVATION_NAME)
 # Builder
 from . import get_srid_column
-from .image import (SmartDataSet, generate_cogs, linear_raster_scale, raster_convexhull, raster_extent, rescale,
-                    save_as_cog)
+from .image import (SmartDataSet, generate_cogs, get_resample_method, linear_raster_scale, raster_convexhull,
+                    raster_extent, rescale, save_as_cog)
 from .index_generator import generate_band_indexes
 from .strings import StringFormatter
 
@@ -236,7 +236,8 @@ def merge(merge_file: str, mask: dict, assets: List[dict], band: str,
 
     if quality_band == band:
         source_nodata = nodata = float(mask['nodata'])
-        # Only apply bilinear (change pixel values) for band values
+    elif "resampling" in kwargs:
+        resampling = get_resample_method(kwargs["resampling"])
     elif (mask and mask.get('saturated_band') != band) or quality_band is None:
         resampling = Resampling.bilinear
 
@@ -796,6 +797,16 @@ def blend(activity, band_map, quality_band, build_clear_observation=False, block
                 saturated = radsat_extract_bits(saturated, 1, 7).astype(numpy.bool_)
                 masked.mask[saturated] = True
 
+            # Get current observation file name
+            file_path = bandlist[order].name
+            file_date = datetime.strptime(merges_band_map[file_path], '%Y-%m-%d')
+            day_of_year = file_date.timetuple().tm_yday
+
+            if build_clear_observation and is_combined_collection:
+                datasource_block = provenance_merge_map[file_date.strftime('%Y-%m-%d')].dataset.read(1, window=window)
+                if mask_values['bits']:
+                    confidence.oli = numpy.isin(datasource_block, index_landsat_oli)
+
             if mask_values['bits']:
                 matched = get_qa_mask(masked,
                                       clear_data=clear_values,
@@ -828,16 +839,6 @@ def blend(activity, band_map, quality_band, build_clear_observation=False, block
 
             stack_total_observation[window.row_off: row_offset, window.col_off: col_offset] += copy_mask.astype(numpy.uint8)
 
-            # Get current observation file name
-            file_path = bandlist[order].name
-            file_date = datetime.strptime(merges_band_map[file_path], '%Y-%m-%d')
-            day_of_year = file_date.timetuple().tm_yday
-
-            if build_clear_observation and is_combined_collection:
-                datasource_block = provenance_merge_map[file_date.strftime('%Y-%m-%d')].dataset.read(1, window=window)
-                if mask_values['bits']:
-                    confidence.oli = datasource_block == index_landsat_oli
-
             # Find all no data in destination STACK image
             stack_raster_where_nodata = numpy.where(
                 stack_raster[window.row_off: row_offset, window.col_off: col_offset] == nodata
@@ -848,9 +849,6 @@ def blend(activity, band_map, quality_band, build_clear_observation=False, block
                                                               stack_raster[window.row_off: row_offset,
                                                               window.col_off: col_offset].shape)
 
-            if build_clear_observation and is_combined_collection:
-                datasource_block = provenance_merge_map[file_date.strftime('%Y-%m-%d')].dataset.read(1, window=window)
-
             # Find all valid/cloud in destination STACK image
             raster_where_data = numpy.where(raster != nodata)
             raster_data_pos = numpy.ravel_multi_index(raster_where_data, raster.shape)
@@ -952,7 +950,7 @@ def blend(activity, band_map, quality_band, build_clear_observation=False, block
             provenance_file = build_cube_path(datacube, period, tile_id, version=version,
                                               band=PROVENANCE_NAME, composed=True, **kwargs)
             provenance_profile = profile.copy()
-            provenance_profile.pop('nodata', -1)
+            provenance_profile['nodata'] = PROVENANCE_ATTRIBUTES['nodata']
             provenance_profile['dtype'] = PROVENANCE_ATTRIBUTES['data_type']
 
             save_as_cog(str(provenance_file), provenance_array, block_size=block_size, **provenance_profile)

cube_builder/version.py

@@ -23,4 +23,4 @@
 """
 
 
-__version__ = '0.8.5'
+__version__ = '0.8.6'

setup.py

@@ -77,7 +77,9 @@
     'SQLAlchemy-Utils>=0.34.2,<1',
     'pystac-client>=0.5',
     'MarkupSafe==2.0.1',
-    'bdc-auth-client @ git+https://github.com/brazil-data-cube/bdc-auth-client.git@v0.2.1#egg=bdc-auth-client'
+    'bdc-auth-client @ git+https://github.com/brazil-data-cube/bdc-auth-client.git@v0.2.1#egg=bdc-auth-client',
+    # 3rdparty deps
+    'pydantic<2',
 ]
 
 packages = find_packages()