Make interpolation and extrapolation behaviour configurable (#97)

This adds `interpolate` and `extrapolate` keyword arguments to to the
Item types (Image, MaskBinary, MaskMulti).

Resolves #60

Author: paulnovo

docs/src/projective/gallery.md

@@ -26,6 +26,45 @@ tfm = CenterCrop((196, 196))
 image = Image(imagedata)
 apply(tfm, image) |> itemdata
 ```
+Customization of how pixel values are interpolated and extrapolated during
+transformations is done with the Item types ([`Image`](@ref),
+[`MaskBinary`](@ref), [`MaskMulti`](@ref)). For example, if we scale the image
+we can see how the interpolation affects how values of projected pixels are
+calculated.
+```@example deps
+using Interpolations: BSpline, Constant, Linear
+tfm = ScaleFixed((2000, 2000)) |> CenterCrop((200, 200))
+showgrid(
+    [
+        # Default is linear interpolation for Image
+        apply(tfm, Image(imagedata)),
+        # Nearest neighbor interpolation
+        apply(tfm, Image(imagedata; interpolate=BSpline(Constant()))),
+        # Linear interpolation
+        apply(tfm, Image(imagedata; interpolate=BSpline(Linear()))),
+    ];
+    ncol=3,
+    npad=8,
+)
+```
+Similarly, if we crop to a larger region than the image, we can see how
+extrapolation affects how pixel values are calculated in the regions outside
+the original image bounds.
+```@example deps
+import Interpolations
+tfm = CenterCrop((400, 400))
+showgrid(
+    [
+        apply(tfm, Image(imagedata)),
+        apply(tfm, Image(imagedata; extrapolate=1)),
+        apply(tfm, Image(imagedata; extrapolate=Interpolations.Flat())),
+        apply(tfm, Image(imagedata; extrapolate=Interpolations.Periodic())),
+        apply(tfm, Image(imagedata; extrapolate=Interpolations.Reflect())),
+    ];
+    ncol=5,
+    npad=8,
+)
+```
 
 Now let's say we want to train a light house detector and have a bounding box for the light house. We can use the [`BoundingBox`](@ref) item to represent it. It takes the two corners of the bounding rectangle as the first argument. As the second argument we have to pass the size of the corresponding image.
 

src/items/image.jl

@@ -4,9 +4,21 @@
 # correspond to the image axes.
 
 """
-    Image(image[, bounds])
-
-Item representing an N-dimensional image with element type T.
+    Image(image[, bounds]; interpolate=BSpline(Linear()), extrapolate=zero(T))
+
+Item representing an N-dimensional image with element type T. Optionally, the
+interpolation and extrapolation method can be provided. Interpolation here
+refers to how the values of projected pixels that fall into the transformed
+content bounds are calculated. Extrapolation refers to how to assign values
+that fall outside the projected content bounds. The default is linear
+interpolation and to fill new regions with zero.
+
+!!! info
+    The `Interpolations` package provides numerous methods for use with
+    the `interpolate` and `extrapolate` keyword arguments.  For instance,
+    `BSpline(Linear())` and `BSpline(Constant())` provide linear and nearest
+    neighbor interpolation, respectively. In addition `Flat()`, `Reflect()` and
+    `Periodic()` boundary conditions are available for extrapolation.
 
 ## Examples
 
@@ -30,14 +42,24 @@ showitems(item)
 struct Image{N,T} <: AbstractArrayItem{N,T}
     data::AbstractArray{T,N}
     bounds::Bounds{N}
+    interpolate::Interpolations.InterpolationType
+    extrapolate::ImageTransformations.FillType
 end
 
-Image(data) = Image(data, Bounds(axes(data)))
-
-function Image(data::AbstractArray{T,N}, sz::NTuple{N,Int}) where {T,N}
-    return Image(data, Bounds(sz))
+function Image(
+    data::AbstractArray{T,N},
+    bounds::Bounds{N};
+    interpolate::Interpolations.InterpolationType=BSpline(Linear()),
+    extrapolate::ImageTransformations.FillType=zero(T),
+) where {T,N}
+    return Image(data, bounds, interpolate, extrapolate)
 end
 
+Image(data; kwargs...) = Image(data, Bounds(axes(data)); kwargs...)
+
+function Image(data::AbstractArray{T,N}, sz::NTuple{N,Int}; kwargs...) where {T,N}
+    return Image(data, Bounds(sz); kwargs...)
+end
 
 Base.show(io::IO, item::Image{N,T}) where {N,T} =
     print(io, "Image{$N, $T}() with bounds $(item.bounds)")
@@ -68,7 +90,8 @@ function project(P, image::Image{N, T}, bounds::Bounds) where {N, T}
         itemdata(image),
         inv(P),
         bounds.rs;
-        fillvalue = zero(T))
+        method=image.interpolate,
+        fillvalue=image.extrapolate)
     return Image(data_, bounds)
 end
 
@@ -79,7 +102,7 @@ function project!(bufimage::Image, P, image::Image{N, T}, bounds::Bounds{N}) whe
     a = OffsetArray(parent(itemdata(bufimage)), bounds.rs)
     res = warp!(
         a,
-        box_extrapolation(itemdata(image); fillvalue=zero(T)),
+        box_extrapolation(itemdata(image); method=image.interpolate, fillvalue=image.extrapolate),
         inv(P),
     )
     return Image(res, bounds)

src/items/mask.jl

@@ -1,17 +1,27 @@
 """
-    MaskMulti(a, [classes])
-
-An `N`-dimensional multilabel mask with labels `classes`.
+    MaskMulti(a, [classes]; interpolate=BSpline(Constant()), extrapolate=Flat())
+
+An `N`-dimensional multilabel mask with labels `classes`. Optionally, the
+interpolation and extrapolation method can be provided. Interpolation here
+refers to how the values of projected pixels that fall into the transformed
+content bounds are calculated. Extrapolation refers to how to assign values
+that fall outside the projected content bounds. The default is nearest neighbor
+interpolation and flat extrapolation of the edges into new regions.
+
+!!! info
+    The `Interpolations` package provides numerous methods for use with
+    the `interpolate` and `extrapolate` keyword arguments.  For instance,
+    `BSpline(Linear())` and `BSpline(Constant())` provide linear and nearest
+    neighbor interpolation, respectively. In addition `Flat()`, `Reflect()` and
+    `Periodic()` boundary conditions are available for extrapolation.
 
 ## Examples
 
-{cell=MaskMulti}
 ```julia
 using DataAugmentation
 
 mask = MaskMulti(rand(1:3, 100, 100))
 ```
-{cell=MaskMulti}
 ```julia
 showitems(mask)
 ```
@@ -20,19 +30,30 @@ struct MaskMulti{N, T<:Integer, U} <: AbstractArrayItem{N, T}
     data::AbstractArray{T, N}
     classes::AbstractVector{U}
     bounds::Bounds{N}
+    interpolate::Interpolations.InterpolationType
+    extrapolate::ImageTransformations.FillType
 end
 
+function MaskMulti(
+    data::AbstractArray{T,N},
+    classes::AbstractVector{U},
+    bounds::Bounds{N};
+    interpolate::Interpolations.InterpolationType=BSpline(Constant()),
+    extrapolate::ImageTransformations.FillType=Flat(),
+) where {N, T<:Integer, U}
+    return MaskMulti(data, classes, bounds, interpolate, extrapolate)
+end
 
-function MaskMulti(a::AbstractArray, classes = unique(a))
+function MaskMulti(a::AbstractArray, classes = unique(a); kwargs...)
     bounds = Bounds(size(a))
     minimum(a) >= 1 || error("Class values must start at 1")
-    return MaskMulti(a, classes, bounds)
+    return MaskMulti(a, classes, bounds; kwargs...)
 end
 
-MaskMulti(a::AbstractArray{<:Gray{T}}, args...) where T = MaskMulti(reinterpret(T, a), args...)
-MaskMulti(a::AbstractArray{<:Normed{T}}, args...) where T = MaskMulti(reinterpret(T, a), args...)
-MaskMulti(a::IndirectArray, classes = a.values, bounds = Bounds(size(a))) =
-    MaskMulti(a.index, classes, bounds)
+MaskMulti(a::AbstractArray{<:Gray{T}}, args...; kwargs...) where T = MaskMulti(reinterpret(T, a), args...; kwargs...)
+MaskMulti(a::AbstractArray{<:Normed{T}}, args...; kwargs...) where T = MaskMulti(reinterpret(T, a), args...; kwargs...)
+MaskMulti(a::IndirectArray, classes = a.values, bounds = Bounds(size(a)); kwargs...) =
+    MaskMulti(a.index, classes, bounds; kwargs...)
 
 Base.show(io::IO, mask::MaskMulti{N, T}) where {N, T} =
     print(io, "MaskMulti{$N, $T}() with size $(size(itemdata(mask))) and $(length(mask.classes)) classes")
@@ -41,12 +62,15 @@ Base.show(io::IO, mask::MaskMulti{N, T}) where {N, T} =
 getbounds(mask::MaskMulti) = mask.bounds
 
 
-function project(P, mask::MaskMulti, bounds::Bounds)
-    a = itemdata(mask)
-    etp = mask_extrapolation(a)
-    res = warp(etp, inv(P), bounds.rs)
+function project(P, mask::MaskMulti{N, T, U}, bounds::Bounds{N}) where {N, T, U}
+    res = warp(
+        itemdata(mask),
+        inv(P),
+        bounds.rs;
+        method=mask.interpolate,
+        fillvalue=mask.extrapolate)
     return MaskMulti(
-        res,
+        convert.(T, res),
         mask.classes,
         bounds
     )
@@ -57,7 +81,7 @@ function project!(bufmask::MaskMulti, P, mask::MaskMulti, bounds)
     a = OffsetArray(parent(itemdata(bufmask)), bounds.rs)
     warp!(
         a,
-        mask_extrapolation(itemdata(mask)),
+        box_extrapolation(itemdata(mask); method=mask.interpolate, fillvalue=mask.extrapolate),
         inv(P),
     )
     return MaskMulti(
@@ -80,30 +104,47 @@ end
 # ## Binary masks
 
 """
-    MaskBinary(a)
-
-An `N`-dimensional binary mask.
+    MaskBinary(a; interpolate=BSpline(Constant()), extrapolate=Flat())
+
+An `N`-dimensional binary mask. Optionally, the interpolation and extrapolation
+method can be provided. Interpolation here refers to how the values of
+projected pixels that fall into the transformed content bounds are calculated.
+Extrapolation refers to how to assign values that fall outside the projected
+content bounds. The default is nearest neighbor interpolation and flat
+extrapolation of the edges into new regions.
+
+!!! info
+    The `Interpolations` package provides numerous methods for use with
+    the `interpolate` and `extrapolate` keyword arguments.  For instance,
+    `BSpline(Linear())` and `BSpline(Constant())` provide linear and nearest
+    neighbor interpolation, respectively. In addition `Flat()`, `Reflect()` and
+    `Periodic()` boundary conditions are available for extrapolation.
 
 ## Examples
 
-{cell=MaskMulti}
 ```julia
 using DataAugmentation
 
 mask = MaskBinary(rand(Bool, 100, 100))
 ```
-{cell=MaskMulti}
 ```julia
 showitems(mask)
 ```
 """
 struct MaskBinary{N} <: AbstractArrayItem{N, Bool}
     data::AbstractArray{Bool, N}
     bounds::Bounds{N}
+    interpolate::Interpolations.InterpolationType
+    extrapolate::ImageTransformations.FillType
 end
 
-function MaskBinary(a::AbstractArray{Bool, N}, bounds = Bounds(size(a))) where N
-    return MaskBinary(a, bounds)
+function MaskBinary(
+    a::AbstractArray,
+    bounds = Bounds(size(a));
+    interpolate::Interpolations.InterpolationType=BSpline(Constant()),
+    extrapolate::ImageTransformations.FillType=Flat(),
+)
+    return MaskBinary(a, bounds, interpolate, extrapolate)
 end
 
 Base.show(io::IO, mask::MaskBinary{N}) where {N} =
@@ -112,19 +153,21 @@ Base.show(io::IO, mask::MaskBinary{N}) where {N} =
 getbounds(mask::MaskBinary) = mask.bounds
 
 function project(P, mask::MaskBinary, bounds::Bounds)
-    etp = mask_extrapolation(itemdata(mask))
-    return MaskBinary(
-        warp(etp, inv(P), bounds.rs),
-        bounds,
-    )
+    res = warp(
+        itemdata(mask),
+        inv(P),
+        bounds.rs;
+        method=mask.interpolate,
+        fillvalue=mask.extrapolate)
+    return MaskBinary(convert.(Bool, res), bounds)
 end
 
 
 function project!(bufmask::MaskBinary, P, mask::MaskBinary, bounds)
     a = OffsetArray(parent(itemdata(bufmask)), bounds.rs)
-    res = warp!(
+    warp!(
         a,
-        mask_extrapolation(itemdata(mask)),
+        box_extrapolation(itemdata(mask); method=mask.interpolate, fillvalue=mask.extrapolate),
         inv(P),
     )
     return MaskBinary(
@@ -136,15 +179,3 @@ end
 function showitem!(img, mask::MaskBinary)
     showimage!(img, colorview(Gray, itemdata(mask)))
 end
-# ## Helpers
-
-
-function mask_extrapolation(
-        mask::AbstractArray{T};
-        t = T,
-        degree = Constant(),
-        boundary = Flat()) where T
-    itp = interpolate(t, t, mask, BSpline(degree))
-    etp = extrapolate(itp, Flat())
-    return etp
-end

src/testing.jl

@@ -103,17 +103,19 @@ function testitem end
 testitem(::Type{ArrayItem}) = testitem(ArrayItem{2, Float32})
 testitem(::Type{ArrayItem{N, T}}) where {N, T} = ArrayItem(rand(T, ntuple(i -> 16, N)))
 
-testitem(::Type{Image}) = testitem(Image{2, RGB{N0f8}})
-testitem(::Type{Image{N, T}}) where {N, T} = Image(rand(T, ntuple(i -> 16, N)))
+testitem(::Type{Image}; kwargs...) = testitem(Image{2, RGB{N0f8}}; kwargs...)
+testitem(::Type{Image{N}}; kwargs...) where {N} = testitem(Image{N, RGB{N0f8}}; kwargs...)
+testitem(::Type{Image{N, T}}; kwargs...) where {N, T} = Image(rand(T, ntuple(i -> 16, N)); kwargs...)
 
-testitem(::Type{MaskBinary}) = testitem(MaskBinary{2})
-testitem(::Type{MaskBinary{N}}) where {N} = MaskBinary(rand(Bool, ntuple(i -> 16, N)))
+testitem(::Type{MaskBinary}; kwargs...) = testitem(MaskBinary{2}; kwargs...)
+testitem(::Type{MaskBinary{N}}; kwargs...) where {N} = MaskBinary(rand(Bool, ntuple(i -> 16, N)); kwargs...)
 
-testitem(::Type{MaskMulti}) = testitem(MaskMulti{2, UInt8})
-function testitem(::Type{MaskMulti{N, T}}) where {N, T}
+testitem(::Type{MaskMulti}; kwargs...) = testitem(MaskMulti{2, UInt8}; kwargs...)
+testitem(::Type{MaskMulti{N}}; kwargs...) where {N} = testitem(MaskMulti{N, UInt8}; kwargs...)
+function testitem(::Type{MaskMulti{N, T}}; kwargs...) where {N, T}
     n = rand(2:10)
     data = T.(rand(1:n, ntuple(i -> 16, N)))
-    MaskMulti(data, 1:n)
+    MaskMulti(data, 1:n; kwargs...)
 end
 
 

test/Project.toml

@@ -8,8 +8,11 @@ CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
 DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e"
 FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 ImageShow = "4e3cecfd-b093-5904-9786-8bbb286a6a31"
+Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
 Rotations = "6038ab10-8711-5258-84ad-4b1120ba62dc"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestSetExtensions = "98d24dd4-01ad-11ea-1b02-c9a08f80db04"

test/imports.jl

@@ -7,6 +7,9 @@ using Colors
 using FixedPointNumbers: N0f8
 using LinearAlgebra
 using Rotations
+using Statistics
+using OffsetArrays
+import Interpolations
 
 
 using DataAugmentation: Item, Transform, getrandstate, itemdata, setdata, ComposedProjectiveTransform,