From d3eed69926dae01b476acea9c7ee7d403ccea49e Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 13:51:02 +0200 Subject: [PATCH 1/8] fix issue 171 that prevented to choose output shape of interpolated images --- dl1_data_handler/image_mapper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dl1_data_handler/image_mapper.py b/dl1_data_handler/image_mapper.py index 38c84cfa..15b30cbc 100644 --- a/dl1_data_handler/image_mapper.py +++ b/dl1_data_handler/image_mapper.py @@ -8,7 +8,7 @@ from collections import Counter, namedtuple from ctapipe.instrument.camera import PixelShape -from ctapipe.core import TelescopeComponent +from ctapipe.core import Component from ctapipe.core.traits import Bool, Int __all__ = [ @@ -23,7 +23,7 @@ "SquareMapper", ] -class ImageMapper(TelescopeComponent): +class ImageMapper(Component): """ Base component for mapping raw 1D vectors into 2D mapped images. @@ -86,6 +86,12 @@ def __init__( this is mutually exclusive with passing ``config`` """ + super().__init__( + config=config, + parent=parent, + **kwargs, + ) + # Camera types self.geometry = geometry self.camera_type = self.geometry.name From 3b3dd93f6948da6a4a709e8b30225eec45e3f92b Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 13:51:16 +0200 Subject: [PATCH 2/8] add unit tests --- dl1_data_handler/tests/test_image_mapper.py | 193 ++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 dl1_data_handler/tests/test_image_mapper.py diff --git a/dl1_data_handler/tests/test_image_mapper.py b/dl1_data_handler/tests/test_image_mapper.py new file mode 100644 index 00000000..9696bdf2 --- /dev/null +++ b/dl1_data_handler/tests/test_image_mapper.py @@ -0,0 +1,193 @@ +"""Tests for image_mapper module.""" +import pytest +import numpy as np +from ctapipe.instrument import CameraGeometry + +from dl1_data_handler.image_mapper import ( + BilinearMapper, + BicubicMapper, + NearestNeighborMapper, + RebinMapper, + AxialMapper, + OversamplingMapper, + ShiftingMapper, + SquareMapper, +) + + +@pytest.fixture +def lstcam_geometry(): + """Fixture to provide LSTCam geometry.""" + return CameraGeometry.from_name("LSTCam") + + +@pytest.fixture +def sample_image(lstcam_geometry): + """Fixture to provide a sample image for testing.""" + return np.random.rand(lstcam_geometry.n_pixels, 1).astype(np.float32) + + +class TestInterpolationImageShape: + """Test that interpolation_image_shape parameter works correctly (issue #171).""" + + @pytest.mark.parametrize( + "mapper_class", + [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], + ) + def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): + """Test that interpolation_image_shape can be set via kwarg. + + This is a regression test for issue #171 where passing + interpolation_image_shape directly to mapper constructors + was silently ignored. + """ + # Request a custom interpolation grid size + custom_size = 138 + mapper = mapper_class( + geometry=lstcam_geometry, interpolation_image_shape=custom_size + ) + + # Verify the trait is set correctly + assert ( + mapper.interpolation_image_shape == custom_size + ), f"{mapper_class.__name__}: interpolation_image_shape trait not set correctly" + + # Verify the image_shape is updated + assert ( + mapper.image_shape == custom_size + ), f"{mapper_class.__name__}: image_shape not updated to custom size" + + # Verify the mapping table has the correct shape + expected_mapping_cols = custom_size * custom_size + assert ( + mapper.mapping_table.shape[1] == expected_mapping_cols + ), f"{mapper_class.__name__}: mapping_table shape incorrect" + + @pytest.mark.parametrize( + "mapper_class", + [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], + ) + def test_interpolation_image_shape_output( + self, lstcam_geometry, sample_image, mapper_class + ): + """Test that the output image has the correct shape when interpolation_image_shape is set.""" + custom_size = 138 + mapper = mapper_class( + geometry=lstcam_geometry, interpolation_image_shape=custom_size + ) + + # Map the image + mapped_image = mapper.map_image(sample_image) + + # Verify output shape + expected_shape = (custom_size, custom_size, 1) + assert ( + mapped_image.shape == expected_shape + ), f"{mapper_class.__name__}: output shape incorrect. Expected {expected_shape}, got {mapped_image.shape}" + + @pytest.mark.parametrize( + "mapper_class", + [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], + ) + def test_default_image_shape(self, lstcam_geometry, mapper_class): + """Test that mappers use default image_shape when interpolation_image_shape is not set.""" + mapper = mapper_class(geometry=lstcam_geometry) + + # Default for LSTCam should be 110 + default_size = 110 + assert ( + mapper.image_shape == default_size + ), f"{mapper_class.__name__}: default image_shape incorrect" + assert ( + mapper.interpolation_image_shape is None + ), f"{mapper_class.__name__}: interpolation_image_shape should be None by default" + + +class TestMapperBasicFunctionality: + """Test basic functionality of all mapper classes.""" + + @pytest.mark.parametrize( + "mapper_class", + [ + BilinearMapper, + BicubicMapper, + NearestNeighborMapper, + RebinMapper, + AxialMapper, + OversamplingMapper, + ShiftingMapper, + ], + ) + def test_hexagonal_mapper_instantiation(self, lstcam_geometry, mapper_class): + """Test that hexagonal mappers can be instantiated.""" + mapper = mapper_class(geometry=lstcam_geometry) + assert mapper is not None + assert mapper.mapping_table is not None + + def test_square_mapper_requires_square_pixels(self, lstcam_geometry): + """Test that SquareMapper raises error for non-square pixel cameras.""" + # LSTCam has hexagonal pixels, should raise ValueError + with pytest.raises(ValueError, match="only available for square pixel cameras"): + SquareMapper(geometry=lstcam_geometry) + + @pytest.mark.parametrize( + "mapper_class", + [ + BilinearMapper, + BicubicMapper, + NearestNeighborMapper, + RebinMapper, + AxialMapper, + OversamplingMapper, + ShiftingMapper, + ], + ) + def test_mapper_output_shape(self, lstcam_geometry, sample_image, mapper_class): + """Test that mappers produce correctly shaped output.""" + mapper = mapper_class(geometry=lstcam_geometry) + mapped_image = mapper.map_image(sample_image) + + # Output should be square image with 1 channel + assert len(mapped_image.shape) == 3 + assert mapped_image.shape[0] == mapped_image.shape[1] + assert mapped_image.shape[2] == 1 + + @pytest.mark.parametrize( + "mapper_class", + [ + BilinearMapper, + BicubicMapper, + NearestNeighborMapper, + RebinMapper, + AxialMapper, + OversamplingMapper, + ShiftingMapper, + ], + ) + def test_mapper_multichannel(self, lstcam_geometry, mapper_class): + """Test that mappers work with multi-channel input.""" + # Create a 2-channel image + multichannel_image = np.random.rand(lstcam_geometry.n_pixels, 2).astype( + np.float32 + ) + mapper = mapper_class(geometry=lstcam_geometry) + mapped_image = mapper.map_image(multichannel_image) + + # Output should preserve the number of channels + assert mapped_image.shape[2] == 2 + + +class TestAxialMapperSpecific: + """Test AxialMapper specific functionality.""" + + def test_set_index_matrix_false(self, lstcam_geometry): + """Test AxialMapper with set_index_matrix=False (default).""" + mapper = AxialMapper(geometry=lstcam_geometry, set_index_matrix=False) + assert mapper.index_matrix is None + + def test_set_index_matrix_true(self, lstcam_geometry): + """Test AxialMapper with set_index_matrix=True.""" + mapper = AxialMapper(geometry=lstcam_geometry, set_index_matrix=True) + assert mapper.index_matrix is not None + # Index matrix should have the same shape as the output image + assert mapper.index_matrix.shape == (mapper.image_shape, mapper.image_shape) From fef9fab7b2a1fcb5ecc64abf72959b9f6dae37f4 Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 14:31:35 +0200 Subject: [PATCH 3/8] add unit test for square pixels geometries, testing with SCTCam --- dl1_data_handler/tests/test_image_mapper.py | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/dl1_data_handler/tests/test_image_mapper.py b/dl1_data_handler/tests/test_image_mapper.py index 9696bdf2..84debea9 100644 --- a/dl1_data_handler/tests/test_image_mapper.py +++ b/dl1_data_handler/tests/test_image_mapper.py @@ -124,11 +124,24 @@ def test_hexagonal_mapper_instantiation(self, lstcam_geometry, mapper_class): assert mapper is not None assert mapper.mapping_table is not None - def test_square_mapper_requires_square_pixels(self, lstcam_geometry): - """Test that SquareMapper raises error for non-square pixel cameras.""" - # LSTCam has hexagonal pixels, should raise ValueError - with pytest.raises(ValueError, match="only available for square pixel cameras"): - SquareMapper(geometry=lstcam_geometry) + def test_square_mapper_instantiation(self): + """Test that SquareMapper can be instantiated with square pixel camera.""" + # SCTCam has square pixels + square_geometry = CameraGeometry.from_name("SCTCam") + mapper = SquareMapper(geometry=square_geometry) + assert mapper is not None + assert mapper.mapping_table is not None + + # Test output shape + sample_square_image = np.random.rand(square_geometry.n_pixels, 1).astype(np.float32) + mapped_image = mapper.map_image(sample_square_image) + + # Output should be square image with 1 channel + assert len(mapped_image.shape) == 3 + assert mapped_image.shape[0] == mapped_image.shape[1] + assert mapped_image.shape[2] == 1 + assert mapped_image.shape[0] == mapper.image_shape + @pytest.mark.parametrize( "mapper_class", From f5c9b85768b2587753b6b63131ebaf4049deac46 Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 14:54:48 +0200 Subject: [PATCH 4/8] test with 55 pixels --- dl1_data_handler/tests/test_image_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dl1_data_handler/tests/test_image_mapper.py b/dl1_data_handler/tests/test_image_mapper.py index 84debea9..1fd40892 100644 --- a/dl1_data_handler/tests/test_image_mapper.py +++ b/dl1_data_handler/tests/test_image_mapper.py @@ -42,7 +42,7 @@ def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): was silently ignored. """ # Request a custom interpolation grid size - custom_size = 138 + custom_size = 55 mapper = mapper_class( geometry=lstcam_geometry, interpolation_image_shape=custom_size ) From 1411019b502162852362d4837ff2cc707be67e01 Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 16:01:17 +0200 Subject: [PATCH 5/8] remove RebinMapper from tests --- dl1_data_handler/tests/test_image_mapper.py | 39 ++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/dl1_data_handler/tests/test_image_mapper.py b/dl1_data_handler/tests/test_image_mapper.py index 1fd40892..d30a8af4 100644 --- a/dl1_data_handler/tests/test_image_mapper.py +++ b/dl1_data_handler/tests/test_image_mapper.py @@ -7,7 +7,6 @@ BilinearMapper, BicubicMapper, NearestNeighborMapper, - RebinMapper, AxialMapper, OversamplingMapper, ShiftingMapper, @@ -32,7 +31,7 @@ class TestInterpolationImageShape: @pytest.mark.parametrize( "mapper_class", - [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], + [BilinearMapper, BicubicMapper, NearestNeighborMapper], ) def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): """Test that interpolation_image_shape can be set via kwarg. @@ -40,6 +39,9 @@ def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): This is a regression test for issue #171 where passing interpolation_image_shape directly to mapper constructors was silently ignored. + + Note: RebinMapper is excluded due to excessive memory requirements + when testing with custom interpolation sizes. """ # Request a custom interpolation grid size custom_size = 55 @@ -65,12 +67,16 @@ def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): @pytest.mark.parametrize( "mapper_class", - [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], + [BilinearMapper, BicubicMapper, NearestNeighborMapper], ) def test_interpolation_image_shape_output( self, lstcam_geometry, sample_image, mapper_class ): - """Test that the output image has the correct shape when interpolation_image_shape is set.""" + """Test that the output image has the correct shape when interpolation_image_shape is set. + + Note: RebinMapper is excluded due to excessive memory requirements + when testing with custom interpolation sizes. + """ custom_size = 138 mapper = mapper_class( geometry=lstcam_geometry, interpolation_image_shape=custom_size @@ -87,10 +93,13 @@ def test_interpolation_image_shape_output( @pytest.mark.parametrize( "mapper_class", - [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], + [BilinearMapper, BicubicMapper, NearestNeighborMapper], ) def test_default_image_shape(self, lstcam_geometry, mapper_class): - """Test that mappers use default image_shape when interpolation_image_shape is not set.""" + """Test that mappers use default image_shape when interpolation_image_shape is not set. + + Note: RebinMapper is excluded due to excessive memory requirements. + """ mapper = mapper_class(geometry=lstcam_geometry) # Default for LSTCam should be 110 @@ -112,14 +121,16 @@ class TestMapperBasicFunctionality: BilinearMapper, BicubicMapper, NearestNeighborMapper, - RebinMapper, AxialMapper, OversamplingMapper, ShiftingMapper, ], ) def test_hexagonal_mapper_instantiation(self, lstcam_geometry, mapper_class): - """Test that hexagonal mappers can be instantiated.""" + """Test that hexagonal mappers can be instantiated. + + Note: RebinMapper is excluded due to excessive memory requirements. + """ mapper = mapper_class(geometry=lstcam_geometry) assert mapper is not None assert mapper.mapping_table is not None @@ -149,14 +160,16 @@ def test_square_mapper_instantiation(self): BilinearMapper, BicubicMapper, NearestNeighborMapper, - RebinMapper, AxialMapper, OversamplingMapper, ShiftingMapper, ], ) def test_mapper_output_shape(self, lstcam_geometry, sample_image, mapper_class): - """Test that mappers produce correctly shaped output.""" + """Test that mappers produce correctly shaped output. + + Note: RebinMapper is excluded due to excessive memory requirements. + """ mapper = mapper_class(geometry=lstcam_geometry) mapped_image = mapper.map_image(sample_image) @@ -171,14 +184,16 @@ def test_mapper_output_shape(self, lstcam_geometry, sample_image, mapper_class): BilinearMapper, BicubicMapper, NearestNeighborMapper, - RebinMapper, AxialMapper, OversamplingMapper, ShiftingMapper, ], ) def test_mapper_multichannel(self, lstcam_geometry, mapper_class): - """Test that mappers work with multi-channel input.""" + """Test that mappers work with multi-channel input. + + Note: RebinMapper is excluded due to excessive memory requirements. + """ # Create a 2-channel image multichannel_image = np.random.rand(lstcam_geometry.n_pixels, 2).astype( np.float32 From d07cc3ad7370855449e4a5f3aca758641885e024 Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 16:48:21 +0200 Subject: [PATCH 6/8] allow user to set a higher memory limit for RebinMapper. prevent default usage > 10GB --- dl1_data_handler/image_mapper.py | 42 ++++++++++++++++++++- dl1_data_handler/tests/test_image_mapper.py | 40 ++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/dl1_data_handler/image_mapper.py b/dl1_data_handler/image_mapper.py index 15b30cbc..9f5baba1 100644 --- a/dl1_data_handler/image_mapper.py +++ b/dl1_data_handler/image_mapper.py @@ -84,12 +84,22 @@ def __init__( parent : ctapipe.core.Component or ctapipe.core.Tool Parent of this component in the configuration hierarchy, this is mutually exclusive with passing ``config`` + **kwargs + Additional keyword arguments for traitlets. Non-traitlet kwargs + (like 'subarray') are filtered out for compatibility. """ + # Filter out non-traitlet kwargs before passing to Component + # This allows compatibility with ctapipe's reader which may pass extra kwargs + component_kwargs = { + key: value for key, value in kwargs.items() + if self.class_own_traits().get(key) is not None + } + super().__init__( config=config, parent=parent, - **kwargs, + **component_kwargs, ) # Camera types @@ -1172,6 +1182,16 @@ class RebinMapper(ImageMapper): ), ).tag(config=True) + max_memory_gb = Int( + default_value=10, + allow_none=True, + help=( + "Maximum memory in GB that RebinMapper is allowed to allocate. " + "Set to None to disable memory checks. Default is 10 GB. " + "Note: RebinMapper uses approximately (image_shape * 10)^2 * image_shape^2 * 4 bytes." + ), + ).tag(config=True) + def __init__( self, geometry, @@ -1211,6 +1231,26 @@ def __init__( self.image_shape = self.interpolation_image_shape self.internal_shape = self.image_shape + self.internal_pad * 2 self.rebinning_mult_factor = 10 + + # Validate memory requirements before proceeding (if max_memory_gb is set) + if self.max_memory_gb is not None: + # RebinMapper uses a fine grid (internal_shape * rebinning_mult_factor)^2 + # and creates a mapping matrix of shape (fine_grid_size, internal_shape, internal_shape) + fine_grid_size = (self.internal_shape * self.rebinning_mult_factor) ** 2 + estimated_memory_gb = ( + fine_grid_size * self.internal_shape * self.internal_shape * 4 + ) / (1024**3) # 4 bytes per float32 + + if estimated_memory_gb > self.max_memory_gb: + raise ValueError( + f"RebinMapper with image_shape={self.image_shape} would require " + f"approximately {estimated_memory_gb:.1f} GB of memory, which exceeds " + f"the limit of {self.max_memory_gb:.1f} GB. " + f"To allow this allocation, set max_memory_gb to a higher value or None. " + f"Alternatively, consider using a smaller interpolation_image_shape (recommended < 60) " + f"or use BilinearMapper or BicubicMapper instead, which are more memory-efficient." + ) + # Creating the hexagonal and the output grid for the conversion methods. input_grid, output_grid = super()._get_grids_for_interpolation() # Calculate the mapping table diff --git a/dl1_data_handler/tests/test_image_mapper.py b/dl1_data_handler/tests/test_image_mapper.py index d30a8af4..d5d31936 100644 --- a/dl1_data_handler/tests/test_image_mapper.py +++ b/dl1_data_handler/tests/test_image_mapper.py @@ -205,6 +205,46 @@ def test_mapper_multichannel(self, lstcam_geometry, mapper_class): assert mapped_image.shape[2] == 2 +class TestRebinMapperMemoryValidation: + """Test RebinMapper memory validation.""" + + def test_rebinmapper_default_size_exceeds_limit(self, lstcam_geometry): + """Test that RebinMapper default size also exceeds memory limit. + + RebinMapper's default behavior requires ~67 GB for LSTCam, which exceeds + the 10 GB safety limit. This is expected and demonstrates why RebinMapper + is excluded from general tests. + """ + from dl1_data_handler.image_mapper import RebinMapper + + # Default size (110) should also raise ValueError due to memory requirements + with pytest.raises(ValueError, match="would require approximately.*GB of memory"): + RebinMapper(geometry=lstcam_geometry) + + def test_rebinmapper_large_size_raises_error(self, lstcam_geometry): + """Test that RebinMapper raises ValueError for large interpolation_image_shape.""" + from dl1_data_handler.image_mapper import RebinMapper + + # Large size should raise ValueError with even more memory requirements + with pytest.raises(ValueError, match="would require approximately.*GB of memory"): + RebinMapper(geometry=lstcam_geometry, interpolation_image_shape=200) + + def test_rebinmapper_error_message_helpful(self, lstcam_geometry): + """Test that RebinMapper error message suggests alternatives.""" + from dl1_data_handler.image_mapper import RebinMapper + + try: + RebinMapper(geometry=lstcam_geometry, interpolation_image_shape=200) + pytest.fail("Should have raised ValueError") + except ValueError as e: + error_msg = str(e) + # Check that error message contains helpful information + assert "BilinearMapper" in error_msg or "BicubicMapper" in error_msg + assert "memory-efficient" in error_msg + assert "interpolation_image_shape" in error_msg or "image_shape" in error_msg + assert "GB of memory" in error_msg + + class TestAxialMapperSpecific: """Test AxialMapper specific functionality.""" From 745f2231408e98faf4224bcaf5e89905d9535ba0 Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 16:55:34 +0200 Subject: [PATCH 7/8] memory in a float --- dl1_data_handler/image_mapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dl1_data_handler/image_mapper.py b/dl1_data_handler/image_mapper.py index 9f5baba1..0992dbcf 100644 --- a/dl1_data_handler/image_mapper.py +++ b/dl1_data_handler/image_mapper.py @@ -9,7 +9,7 @@ from ctapipe.instrument.camera import PixelShape from ctapipe.core import Component -from ctapipe.core.traits import Bool, Int +from ctapipe.core.traits import Bool, Int, Float __all__ = [ "ImageMapper", @@ -1182,7 +1182,7 @@ class RebinMapper(ImageMapper): ), ).tag(config=True) - max_memory_gb = Int( + max_memory_gb = Float( default_value=10, allow_none=True, help=( From 08b1b44c68e9363edd9c0fe5089981a92f7195b3 Mon Sep 17 00:00:00 2001 From: Thomas Vuillaume Date: Mon, 13 Oct 2025 16:55:56 +0200 Subject: [PATCH 8/8] testing RebinMapper with memory usage fixes --- dl1_data_handler/tests/test_image_mapper.py | 119 +++++++++++++++----- 1 file changed, 90 insertions(+), 29 deletions(-) diff --git a/dl1_data_handler/tests/test_image_mapper.py b/dl1_data_handler/tests/test_image_mapper.py index d5d31936..5503eefb 100644 --- a/dl1_data_handler/tests/test_image_mapper.py +++ b/dl1_data_handler/tests/test_image_mapper.py @@ -7,6 +7,7 @@ BilinearMapper, BicubicMapper, NearestNeighborMapper, + RebinMapper, AxialMapper, OversamplingMapper, ShiftingMapper, @@ -31,7 +32,7 @@ class TestInterpolationImageShape: @pytest.mark.parametrize( "mapper_class", - [BilinearMapper, BicubicMapper, NearestNeighborMapper], + [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], ) def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): """Test that interpolation_image_shape can be set via kwarg. @@ -40,14 +41,19 @@ def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): interpolation_image_shape directly to mapper constructors was silently ignored. - Note: RebinMapper is excluded due to excessive memory requirements - when testing with custom interpolation sizes. + Note: RebinMapper uses a small size (10) and increased max_memory_gb + to avoid excessive memory requirements during testing. """ # Request a custom interpolation grid size - custom_size = 55 - mapper = mapper_class( - geometry=lstcam_geometry, interpolation_image_shape=custom_size - ) + # Use smaller size for RebinMapper due to memory requirements + custom_size = 10 if mapper_class == RebinMapper else 55 + + # RebinMapper needs max_memory_gb set higher to allow the allocation + kwargs = {"interpolation_image_shape": custom_size} + if mapper_class == RebinMapper: + kwargs["max_memory_gb"] = 100 + + mapper = mapper_class(geometry=lstcam_geometry, **kwargs) # Verify the trait is set correctly assert ( @@ -67,20 +73,25 @@ def test_interpolation_image_shape_kwarg(self, lstcam_geometry, mapper_class): @pytest.mark.parametrize( "mapper_class", - [BilinearMapper, BicubicMapper, NearestNeighborMapper], + [BilinearMapper, BicubicMapper, NearestNeighborMapper, RebinMapper], ) def test_interpolation_image_shape_output( self, lstcam_geometry, sample_image, mapper_class ): """Test that the output image has the correct shape when interpolation_image_shape is set. - Note: RebinMapper is excluded due to excessive memory requirements - when testing with custom interpolation sizes. + Note: RebinMapper uses a small size (10) and increased max_memory_gb + to avoid excessive memory requirements during testing. """ - custom_size = 138 - mapper = mapper_class( - geometry=lstcam_geometry, interpolation_image_shape=custom_size - ) + # Use smaller size for RebinMapper due to memory requirements + custom_size = 10 if mapper_class == RebinMapper else 138 + + # RebinMapper needs max_memory_gb set higher to allow the allocation + kwargs = {"interpolation_image_shape": custom_size} + if mapper_class == RebinMapper: + kwargs["max_memory_gb"] = 100 + + mapper = mapper_class(geometry=lstcam_geometry, **kwargs) # Map the image mapped_image = mapper.map_image(sample_image) @@ -98,7 +109,8 @@ def test_interpolation_image_shape_output( def test_default_image_shape(self, lstcam_geometry, mapper_class): """Test that mappers use default image_shape when interpolation_image_shape is not set. - Note: RebinMapper is excluded due to excessive memory requirements. + Note: RebinMapper is excluded from this test because its default size (110) + exceeds the default memory limit (10 GB), requiring ~67 GB. """ mapper = mapper_class(geometry=lstcam_geometry) @@ -129,7 +141,9 @@ class TestMapperBasicFunctionality: def test_hexagonal_mapper_instantiation(self, lstcam_geometry, mapper_class): """Test that hexagonal mappers can be instantiated. - Note: RebinMapper is excluded due to excessive memory requirements. + Note: RebinMapper is excluded from this test because its default size (110) + exceeds the default memory limit (10 GB). See test_rebinmapper_small_size_works + for RebinMapper instantiation test with appropriate parameters. """ mapper = mapper_class(geometry=lstcam_geometry) assert mapper is not None @@ -168,7 +182,9 @@ def test_square_mapper_instantiation(self): def test_mapper_output_shape(self, lstcam_geometry, sample_image, mapper_class): """Test that mappers produce correctly shaped output. - Note: RebinMapper is excluded due to excessive memory requirements. + Note: RebinMapper is excluded from this test because its default size (110) + exceeds the default memory limit (10 GB). See test_rebinmapper_small_size_works + for RebinMapper output shape test with appropriate parameters. """ mapper = mapper_class(geometry=lstcam_geometry) mapped_image = mapper.map_image(sample_image) @@ -192,7 +208,9 @@ def test_mapper_output_shape(self, lstcam_geometry, sample_image, mapper_class): def test_mapper_multichannel(self, lstcam_geometry, mapper_class): """Test that mappers work with multi-channel input. - Note: RebinMapper is excluded due to excessive memory requirements. + Note: RebinMapper is excluded from this test because its default size (110) + exceeds the default memory limit (10 GB). See test_rebinmapper_small_size_works + for RebinMapper multichannel test with appropriate parameters. """ # Create a 2-channel image multichannel_image = np.random.rand(lstcam_geometry.n_pixels, 2).astype( @@ -206,33 +224,26 @@ def test_mapper_multichannel(self, lstcam_geometry, mapper_class): class TestRebinMapperMemoryValidation: - """Test RebinMapper memory validation.""" + """Test RebinMapper memory validation and functionality.""" def test_rebinmapper_default_size_exceeds_limit(self, lstcam_geometry): - """Test that RebinMapper default size also exceeds memory limit. + """Test that RebinMapper default size exceeds the default memory limit. RebinMapper's default behavior requires ~67 GB for LSTCam, which exceeds - the 10 GB safety limit. This is expected and demonstrates why RebinMapper - is excluded from general tests. + the 10 GB default safety limit. This is expected behavior. """ - from dl1_data_handler.image_mapper import RebinMapper - - # Default size (110) should also raise ValueError due to memory requirements + # Default size (110) should raise ValueError due to memory requirements with pytest.raises(ValueError, match="would require approximately.*GB of memory"): RebinMapper(geometry=lstcam_geometry) def test_rebinmapper_large_size_raises_error(self, lstcam_geometry): """Test that RebinMapper raises ValueError for large interpolation_image_shape.""" - from dl1_data_handler.image_mapper import RebinMapper - # Large size should raise ValueError with even more memory requirements with pytest.raises(ValueError, match="would require approximately.*GB of memory"): RebinMapper(geometry=lstcam_geometry, interpolation_image_shape=200) def test_rebinmapper_error_message_helpful(self, lstcam_geometry): """Test that RebinMapper error message suggests alternatives.""" - from dl1_data_handler.image_mapper import RebinMapper - try: RebinMapper(geometry=lstcam_geometry, interpolation_image_shape=200) pytest.fail("Should have raised ValueError") @@ -244,6 +255,56 @@ def test_rebinmapper_error_message_helpful(self, lstcam_geometry): assert "interpolation_image_shape" in error_msg or "image_shape" in error_msg assert "GB of memory" in error_msg + def test_rebinmapper_disable_memory_check(self, lstcam_geometry): + """Test that RebinMapper memory check can be disabled with max_memory_gb=None.""" + # Small size that would normally pass, but we're testing the None behavior + # Note: We still use a small size to avoid actually allocating huge memory + mapper = RebinMapper( + geometry=lstcam_geometry, + interpolation_image_shape=10, + max_memory_gb=None + ) + assert mapper is not None + assert mapper.mapping_table is not None + + def test_rebinmapper_custom_memory_limit(self, lstcam_geometry): + """Test that RebinMapper respects custom max_memory_gb values.""" + # Size that requires ~0.13 GB should pass with 1 GB limit + mapper = RebinMapper( + geometry=lstcam_geometry, + interpolation_image_shape=10, + max_memory_gb=1 + ) + assert mapper is not None + + # Size that requires ~0.13 GB should fail with 0.01 GB limit + with pytest.raises(ValueError, match="would require approximately.*GB of memory"): + RebinMapper( + geometry=lstcam_geometry, + interpolation_image_shape=10, + max_memory_gb=0.01 + ) + + def test_rebinmapper_small_size_works(self, lstcam_geometry, sample_image): + """Test that RebinMapper works with small interpolation_image_shape and increased limit.""" + # Small size with increased memory limit should work + mapper = RebinMapper( + geometry=lstcam_geometry, + interpolation_image_shape=10, + max_memory_gb=100 + ) + assert mapper is not None + assert mapper.mapping_table is not None + + # Test that it can actually map an image + mapped_image = mapper.map_image(sample_image) + + # Output should be square image with 1 channel + assert len(mapped_image.shape) == 3 + assert mapped_image.shape[0] == mapped_image.shape[1] + assert mapped_image.shape[2] == 1 + assert mapped_image.shape[0] == 10 + class TestAxialMapperSpecific: """Test AxialMapper specific functionality."""