⚡️ Speed up method BlipImageProcessor.postprocess by 51%

[misrasaurabh1]

📄 51% (0.51x) speedup for BlipImageProcessor.postprocess in src/diffusers/pipelines/blip_diffusion/blip_image_processing.py

⏱️ Runtime : 201 milliseconds → 133 milliseconds (best of 27 runs)

📝 Explanation and details

Here’s a faster, more memory-efficient rewrite while preserving all return values and function signatures. The optimizations address.

• Avoid unnecessary copying/conversion during numpy->PIL conversion
• Remove redundant .cpu() calls when already on CPU
• Optimize numpy array handling to avoid memory overhead
• Only run squeeze when necessary and pull out constants where safe.

Optimizations made:

• Avoided unnecessary .cpu() calls and ensured direct use of .contiguous() before .numpy() to avoid memory bottlenecks on non-contiguous tensors.
• Used dictionary set-literal lookups for output_type (marginally faster for a fixed small set).
• Removed needless Image.fromarray squeeze (use [..., 0] indexing, never triggers for RGB).
• Used astype("uint8", copy=False) to avoid unnecessary array copying during data type conversion.
• Used .clamp_() for in-place operations to reduce memory and allow for better memory reuse.
• Moved size default initialization outside the function call for better micro-optimization and readability.

No changes to logic, outputs, or external side-effects or comments.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 84 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests Details

import pytest  # used for our unit tests
import torch
from PIL import Image
from src.diffusers.pipelines.blip_diffusion.blip_image_processing import \
    BlipImageProcessor

# function to test (already imported as BlipImageProcessor with postprocess)

# ---------------------------
# Unit tests for postprocess
# ---------------------------

# Helper function to create a batch of images as torch tensors
def make_tensor(batch_size, channels, height, width, fill_value=None, dtype=torch.float32):
    """
    Utility to create a batch of images as a torch tensor.
    If fill_value is not None, fills the tensor with that value.
    """
    shape = (batch_size, channels, height, width)
    if fill_value is not None:
        return torch.full(shape, fill_value, dtype=dtype)
    # Otherwise, random values in [-1, 1]
    return (torch.rand(shape, dtype=dtype) - 0.5) * 2

# 1. Basic Test Cases

def test_postprocess_pt_output_type_returns_tensor():
    # Test that output_type='pt' returns a torch.Tensor with values in [0,1]
    processor = BlipImageProcessor()
    sample = make_tensor(2, 3, 16, 16)
    codeflash_output = processor.postprocess(sample, output_type="pt"); out = codeflash_output

def test_postprocess_np_output_type_returns_numpy():
    # Test that output_type='np' returns a numpy array with correct shape and dtype
    processor = BlipImageProcessor()
    sample = make_tensor(1, 3, 8, 8)
    codeflash_output = processor.postprocess(sample, output_type="np"); out = codeflash_output

def test_postprocess_pil_output_type_returns_pil_images():
    # Test that output_type='pil' returns a list of PIL.Image.Image objects
    processor = BlipImageProcessor()
    sample = make_tensor(3, 3, 10, 10)
    codeflash_output = processor.postprocess(sample, output_type="pil"); out = codeflash_output
    for img in out:
        pass

def test_postprocess_grayscale_image():
    # Test grayscale (1 channel) image returns mode "L" PIL images
    processor = BlipImageProcessor()
    sample = make_tensor(2, 1, 7, 5)
    codeflash_output = processor.postprocess(sample, output_type="pil"); out = codeflash_output
    for img in out:
        pass

def test_postprocess_single_image_batch():
    # Test single image (batch size 1) returns a list of one PIL image
    processor = BlipImageProcessor()
    sample = make_tensor(1, 3, 12, 12)
    codeflash_output = processor.postprocess(sample, output_type="pil"); out = codeflash_output

def test_postprocess_value_range_mapping():
    # Test that input values of -1 and 1 are mapped to 0 and 1 after postprocess
    processor = BlipImageProcessor()
    sample = torch.tensor([[[[-1.0, 1.0], [0.0, 0.5]]]])
    # shape: (1,1,2,2)
    codeflash_output = processor.postprocess(sample, output_type="pt"); out = codeflash_output
    # -1 -> 0, 1 -> 1, 0 -> 0.5, 0.5 -> 0.75
    expected = torch.tensor([[[[0.0, 1.0], [0.5, 0.75]]]])

# 2. Edge Test Cases

def test_postprocess_invalid_output_type():
    # Test that an invalid output_type raises ValueError
    processor = BlipImageProcessor()
    sample = make_tensor(1, 3, 8, 8)
    with pytest.raises(ValueError):
        processor.postprocess(sample, output_type="badtype")

def test_postprocess_empty_batch():
    # Test that an empty batch (batch size 0) returns an empty list for PIL/np, tensor for pt
    processor = BlipImageProcessor()
    sample = make_tensor(0, 3, 8, 8)
    codeflash_output = processor.postprocess(sample, output_type="pt"); out_pt = codeflash_output
    codeflash_output = processor.postprocess(sample, output_type="np"); out_np = codeflash_output
    codeflash_output = processor.postprocess(sample, output_type="pil"); out_pil = codeflash_output

def test_postprocess_single_pixel_image():
    # Test 1x1 image (single pixel)
    processor = BlipImageProcessor()
    sample = make_tensor(1, 3, 1, 1)
    codeflash_output = processor.postprocess(sample, output_type="pil"); out = codeflash_output

def test_postprocess_max_min_values_clamping():
    # Test that values outside [-1, 1] are clamped correctly after postprocess
    processor = BlipImageProcessor()
    # Values: -10, 10, 0
    sample = torch.tensor([[[[-10.0, 10.0, 0.0]]]])
    codeflash_output = processor.postprocess(sample, output_type="pt"); out = codeflash_output
    # -10 -> 0, 10 -> 1, 0 -> 0.5
    expected = torch.tensor([[[[0.0, 1.0, 0.5]]]])

def test_postprocess_non_float_tensor():
    # Test that integer tensors are converted to float and processed correctly
    processor = BlipImageProcessor()
    sample = torch.ones((2, 3, 4, 4), dtype=torch.int32)
    codeflash_output = processor.postprocess(sample, output_type="pt"); out = codeflash_output

def test_postprocess_cpu_gpu_consistency():
    # Test that running on cpu and cuda (if available) gives the same result
    processor = BlipImageProcessor()
    sample = make_tensor(1, 3, 8, 8)
    codeflash_output = processor.postprocess(sample, output_type="pt"); out_cpu = codeflash_output
    if torch.cuda.is_available():
        sample_cuda = sample.cuda()
        codeflash_output = processor.postprocess(sample_cuda, output_type="pt"); out_cuda = codeflash_output

def test_postprocess_large_channel_number():
    # Test with a large number of channels (e.g., 10) for pt/np outputs
    processor = BlipImageProcessor()
    sample = make_tensor(1, 10, 5, 5)
    codeflash_output = processor.postprocess(sample, output_type="pt"); out_pt = codeflash_output
    codeflash_output = processor.postprocess(sample, output_type="np"); out_np = codeflash_output
    # PIL output should fail (since numpy_to_pil expects 1 or 3 channels)
    with pytest.raises(ValueError):
        processor.postprocess(sample, output_type="pil")

def test_postprocess_single_channel_np_output():
    # Test that single-channel image returns correct shape for np output
    processor = BlipImageProcessor()
    sample = make_tensor(2, 1, 6, 6)
    codeflash_output = processor.postprocess(sample, output_type="np"); out = codeflash_output

# 3. Large Scale Test Cases

def test_postprocess_large_batch():
    # Test with a large batch size (e.g., 512)
    processor = BlipImageProcessor()
    batch_size = 512
    sample = make_tensor(batch_size, 3, 8, 8)
    codeflash_output = processor.postprocess(sample, output_type="pil"); out_pil = codeflash_output
    for img in out_pil[:5]:  # spot check first 5
        pass

def test_postprocess_large_image():
    # Test with a large image size (e.g., 128x128, batch size 2)
    processor = BlipImageProcessor()
    sample = make_tensor(2, 3, 128, 128)
    codeflash_output = processor.postprocess(sample, output_type="np"); out_np = codeflash_output
    codeflash_output = processor.postprocess(sample, output_type="pil"); out_pil = codeflash_output
    for img in out_pil:
        pass

def test_postprocess_maximum_allowed_tensor_size():
    # Test with a tensor near the 100MB limit: batch=16, 3x256x256 float32 = ~12.6MB
    processor = BlipImageProcessor()
    batch_size = 16
    channels = 3
    height = 256
    width = 256
    sample = make_tensor(batch_size, channels, height, width)
    codeflash_output = processor.postprocess(sample, output_type="np"); out_np = codeflash_output
    codeflash_output = processor.postprocess(sample, output_type="pil"); out_pil = codeflash_output
    for img in out_pil[:3]:  # spot check
        pass

def test_postprocess_all_zero_input():
    # Test that all-zero input returns all 0.5 after postprocess
    processor = BlipImageProcessor()
    sample = torch.zeros((4, 3, 8, 8))
    codeflash_output = processor.postprocess(sample, output_type="pt"); out = codeflash_output

def test_postprocess_all_one_input():
    # Test that all-one input returns all 1.0 after postprocess
    processor = BlipImageProcessor()
    sample = torch.ones((3, 3, 8, 8))
    codeflash_output = processor.postprocess(sample, output_type="pt"); out = codeflash_output
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

from typing import Dict, List, Optional, Union

# imports
import pytest  # used for our unit tests
import torch
from PIL import Image
from src.diffusers.pipelines.blip_diffusion.blip_image_processing import \
    BlipImageProcessor
from transformers.image_processing_utils import (BaseImageProcessor,
                                                 get_size_dict)
from transformers.image_utils import (OPENAI_CLIP_MEAN, OPENAI_CLIP_STD,
                                      PILImageResampling)

# ----------- UNIT TESTS BEGIN HERE ------------

# Helper function to create a random tensor of the required shape and dtype
def make_tensor(batch, channels, height, width, dtype=torch.float32, fill_value=None):
    if fill_value is not None:
        t = torch.full((batch, channels, height, width), fill_value, dtype=dtype)
    else:
        t = torch.rand((batch, channels, height, width), dtype=dtype) * 2 - 1  # in [-1,1]
    return t

@pytest.fixture
def processor():
    # Provide a default processor instance for tests
    return BlipImageProcessor()

# ---------------- Basic Test Cases ----------------

def test_postprocess_pt_output_type(processor):
    # Test that output_type="pt" returns a torch.Tensor with expected value range and shape
    x = make_tensor(2, 3, 16, 16)
    codeflash_output = processor.postprocess(x, output_type="pt"); result = codeflash_output

def test_postprocess_np_output_type(processor):
    # Test that output_type="np" returns a numpy ndarray with correct shape and value range
    x = make_tensor(1, 3, 8, 8)
    codeflash_output = processor.postprocess(x, output_type="np"); result = codeflash_output

def test_postprocess_pil_output_type(processor):
    # Test that output_type="pil" returns a list of PIL Images with correct size and mode
    x = make_tensor(2, 3, 10, 12)
    codeflash_output = processor.postprocess(x, output_type="pil"); result = codeflash_output
    for img in result:
        pass

def test_postprocess_single_channel_grayscale(processor):
    # Test that a single channel (grayscale) image returns PIL images in L mode
    x = make_tensor(1, 1, 5, 7)
    codeflash_output = processor.postprocess(x, output_type="pil"); result = codeflash_output
    img = result[0]

def test_postprocess_batch_size_one(processor):
    # Test that batch size 1 works for all output types
    x = make_tensor(1, 3, 4, 4)
    codeflash_output = processor.postprocess(x, output_type="pt"); pt = codeflash_output
    codeflash_output = processor.postprocess(x, output_type="np"); np_out = codeflash_output
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output

# ---------------- Edge Test Cases ----------------

def test_postprocess_invalid_output_type(processor):
    # Test that an invalid output_type raises ValueError
    x = make_tensor(1, 3, 4, 4)
    with pytest.raises(ValueError):
        processor.postprocess(x, output_type="foo")

def test_postprocess_min_max_values(processor):
    # Test that input values at -1 and 1 are mapped to 0 and 1 after denormalization
    x = torch.tensor([
        [[[-1.0, 1.0], [0.0, -0.5]]],  # shape (1,1,2,2)
    ])
    codeflash_output = processor.postprocess(x, output_type="np"); result = codeflash_output
    # Denormalize: (x/2 + 0.5) => [-1,1] -> [0,1]
    expected = (((x / 2) + 0.5).clamp(0, 1)).cpu().permute(0, 2, 3, 1).numpy()

def test_postprocess_non_contiguous_tensor(processor):
    # Test that a non-contiguous tensor is handled correctly
    x = make_tensor(2, 3, 8, 8)
    x_t = x.transpose(2, 3)  # make non-contiguous
    codeflash_output = processor.postprocess(x_t, output_type="pt"); result = codeflash_output

def test_postprocess_on_cuda_if_available(processor):
    # Test that CUDA tensors are handled (if CUDA is available)
    if torch.cuda.is_available():
        x = make_tensor(1, 3, 8, 8).cuda()
        codeflash_output = processor.postprocess(x, output_type="pt"); result = codeflash_output

def test_postprocess_single_pixel_image(processor):
    # Test that a single pixel image is handled correctly
    x = make_tensor(1, 3, 1, 1)
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output
    img = pil[0]

def test_postprocess_empty_batch(processor):
    # Test that an empty batch (batch size 0) returns empty outputs
    x = make_tensor(0, 3, 8, 8)
    codeflash_output = processor.postprocess(x, output_type="pt"); pt = codeflash_output
    codeflash_output = processor.postprocess(x, output_type="np"); np_out = codeflash_output
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output

def test_postprocess_large_value_range(processor):
    # Test that values outside [-1,1] are clamped correctly
    x = torch.tensor([[[[2.0, -2.0], [10.0, -10.0]]]])  # shape (1,1,2,2)
    codeflash_output = processor.postprocess(x, output_type="np"); result = codeflash_output

def test_postprocess_different_dtypes(processor):
    # Test that float16 and float64 tensors are handled
    for dtype in [torch.float16, torch.float64]:
        x = make_tensor(1, 3, 8, 8, dtype=dtype)
        codeflash_output = processor.postprocess(x, output_type="pt"); pt = codeflash_output

def test_postprocess_grayscale_and_rgb_batch(processor):
    # Test that a batch of both grayscale and RGB images is not supported (should raise)
    # The function expects all images in a batch to have the same number of channels
    x = torch.cat([
        make_tensor(1, 1, 4, 4),
        make_tensor(1, 3, 4, 4)
    ], dim=0)
    # This should raise a RuntimeError due to mismatched channels
    with pytest.raises(RuntimeError):
        processor.postprocess(x, output_type="pil")

# ---------------- Large Scale Test Cases ----------------

def test_postprocess_large_batch(processor):
    # Test with a large batch size, but within memory limits
    batch_size = 64
    x = make_tensor(batch_size, 3, 32, 32)
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output
    for img in pil:
        pass

def test_postprocess_large_image(processor):
    # Test with a single large image (e.g., 512x512, 3 channels)
    x = make_tensor(1, 3, 512, 512)
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output
    img = pil[0]

def test_postprocess_max_elements(processor):
    # Test with a tensor close to the 100MB limit
    # 100MB / 4 bytes per float32 = 25,000,000 elements
    # For 3x256x256 images: 3*256*256 = 196608 per image
    # 25,000,000 // 196608 = ~127 images
    batch_size = 100  # Keep well below the limit for safety
    x = make_tensor(batch_size, 3, 256, 256)
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output
    for img in pil:
        pass

def test_postprocess_large_grayscale_batch(processor):
    # Test with a large batch of grayscale images
    batch_size = 100
    x = make_tensor(batch_size, 1, 64, 64)
    codeflash_output = processor.postprocess(x, output_type="pil"); pil = codeflash_output
    for img in pil:
        pass

def test_postprocess_large_np_output(processor):
    # Test that np output for a large batch is correct
    batch_size = 200
    x = make_tensor(batch_size, 3, 16, 16)
    codeflash_output = processor.postprocess(x, output_type="np"); np_out = codeflash_output
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

[asomoza]

thanks but we're going to deprecate this pipeline because of the low usage it has, so there no much benefit on updating it at this point.

Closing this for now.