diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d3fa4be..b408776 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -14,7 +14,6 @@ jobs: fail-fast: false matrix: python-version: - - '3.6' - '3.7' - '3.8' - '3.9' diff --git a/appveyor.yml b/appveyor.yml index 71468ed..e36e90e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,18 +1,20 @@ image: ubuntu1804 -install: - - sudo apt-get install -y python3-setuptools python3-wheel twine - build: off +install: + - sudo apt-get install -y python3-pip + build_script: + - pip3 install --user -U pip setuptools wheel flake8 + - export PATH="$(python3 -m site --user-base)/bin:${PATH}" - python3 setup.py build test_script: - - python3 setup.py flake8 + - flake8 . after_test: - - python3 setup.py bdist_wheel bdist_egg + - python3 setup.py bdist_wheel artifacts: - path: dist/* diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index f128f1f..f84420a 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -19,6 +19,14 @@ import softwarerenderbackend import zarr +from numcodecs.abc import Codec +from numcodecs.compat import \ + ensure_bytes, \ + ensure_contiguous_ndarray, \ + ndarray_copy +from numcodecs.registry import register_codec +import imagecodecs + from datetime import datetime from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait from threading import BoundedSemaphore @@ -81,7 +89,7 @@ class WriteTiles(object): def __init__( self, tile_width, tile_height, resolutions, max_workers, - batch_size, fill_color, nested, input_path, output_path + batch_size, fill_color, nested, input_path, output_path, psnr ): self.tile_width = tile_width self.tile_height = tile_height @@ -92,6 +100,7 @@ def __init__( self.nested = nested self.input_path = input_path self.slide_directory = output_path + self.psnr = psnr render_context = softwarerendercontext.SoftwareRenderContext() render_backend = softwarerenderbackend.SoftwareRenderBackend() @@ -388,6 +397,18 @@ def wait_any(self, regions): else: return self.pixel_engine.wait_any(regions) + def write_image_metadata(self, resolutions, series): + multiscales = [{ + 'metadata': { + 'method': 'pixelengine', + 'version': str(self.pixel_engine.version) + }, + 'version': '0.2', + 'datasets': [{'path': str(v)} for v in resolutions] + }] + z = self.zarr_group["%d" % series] + z.attrs['multiscales'] = multiscales + def write_metadata_json(self, metadata_file): '''write metadata to a JSON file''' @@ -491,9 +512,10 @@ def write_image_type(self, image_type, series): tile = self.zarr_group["%d/0" % series] tile.attrs['image type'] = image_type for channel in range(0, 3): - band = np.array(img.getdata(band=channel)) - band.shape = (height, width) - tile[0, 0, channel] = band + data = np.array(img.getdata()) + data.shape = (height, width, 3) + tile[0, 0, :] = data + self.write_image_metadata(range(1), series) log.info("wrote %s image" % image_type) @@ -512,21 +534,15 @@ def create_tile_directory(self, series, resolution, width, height): # important to explicitly set the chunk size to 1 for non-XY dims # setting to None may cause all planes to be chunked together - # ordering is TZCYX and hard-coded since Z and T are not present + # ordering is TZYXC (interleaved) and hard-coded since Z and T + # are not present self.zarr_group.create_dataset( "%s/%s" % (str(series), str(resolution)), - shape=(1, 1, 3, height, width), - chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B' + shape=(1, 1, height, width, 3), + chunks=(1, 1, self.tile_height, self.tile_width, 3), dtype='B', + compressor=j2k(self.psnr) ) - def make_planar(self, pixels, tile_width, tile_height): - r = pixels[0::3] - g = pixels[1::3] - b = pixels[2::3] - for v in (r, g, b): - v.shape = (tile_height, tile_width) - return np.array([r, g, b]) - def write_pyramid(self): '''write the slide's pyramid as a set of tiles''' pe_in = self.pixel_engine["in"] @@ -547,11 +563,9 @@ def write_tile( x_end = x_start + tile_width y_end = y_start + tile_height try: - # Zarr has a single n-dimensional array representation on - # disk (not interleaved RGB) - pixels = self.make_planar(pixels, tile_width, tile_height) + pixels.shape = (tile_height, tile_width, 3) z = self.zarr_group["0/%d" % resolution] - z[0, 0, :, y_start:y_end, x_start:x_end] = pixels + z[0, 0, y_start:y_end, x_start:x_end, :] = pixels except Exception: log.error( "Failed to write tile [:, %d:%d, %d:%d]" % ( @@ -656,6 +670,7 @@ def write_tile( x_start, y_start, width, height )) wait(jobs, return_when=ALL_COMPLETED) + self.write_image_metadata(resolutions, 0) def create_patch_list( self, dim_ranges, tiles, tile_size, tile_directory @@ -705,3 +720,36 @@ def create_patch_list( # order to identify the patches returned asynchronously patch_ids.append((x, y)) return patches, patch_ids + + +class j2k(Codec): + """Codec providing j2k compression via imagecodecs. + Parameters + ---------- + psnr : int + Compression peak signal noise ratio. + """ + + codec_id = "j2k" + + def __init__(self, psnr=50): + self.psnr = psnr + assert (self.psnr > 0 and self.psnr <= 100 + and isinstance(self.psnr, int)) + super().__init__() + + def encode(self, buf): + return imagecodecs.jpeg2k_encode(np.squeeze(buf), level=self.psnr) + + def decode(self, buf, out=None): + buf = ensure_bytes(buf) + decoded = imagecodecs.jpeg2k_decode(buf) + if out is not None: + out_view = ensure_contiguous_ndarray(out) + ndarray_copy(decoded, out_view) + else: + out = decoded + return out + + +register_codec(j2k) diff --git a/isyntax2raw/cli/isyntax2raw.py b/isyntax2raw/cli/isyntax2raw.py index 56a3152..dc215ca 100644 --- a/isyntax2raw/cli/isyntax2raw.py +++ b/isyntax2raw/cli/isyntax2raw.py @@ -64,16 +64,20 @@ def cli(): "--debug", is_flag=True, help="enable debugging", ) +@click.option( + "--psnr", default=50, show_default=True, + help="JPEG-2000 compression PSNR" +) @click.argument("input_path") @click.argument("output_path") def write_tiles( tile_width, tile_height, resolutions, max_workers, batch_size, - fill_color, nested, debug, input_path, output_path + fill_color, nested, debug, input_path, output_path, psnr ): setup_logging(debug) with WriteTiles( tile_width, tile_height, resolutions, max_workers, - batch_size, fill_color, nested, input_path, output_path + batch_size, fill_color, nested, input_path, output_path, psnr ) as wt: wt.write_metadata() wt.write_label_image() diff --git a/isyntax2raw/resources/ome_template.xml b/isyntax2raw/resources/ome_template.xml index b63a79b..5df7793 100644 --- a/isyntax2raw/resources/ome_template.xml +++ b/isyntax2raw/resources/ome_template.xml @@ -7,8 +7,8 @@ ${image['description']} - ${image['acquisitionDate']} - @@ -41,8 +41,8 @@ ${image['acquisitionDate']} - =0.9.0', ], tests_require=[