Skip to content

Commit 98955a9

Browse files
hyazinthhhyazinthh
committed
Implement image remapping
1 parent 76cbc1e commit 98955a9

File tree

2 files changed

+215
-40
lines changed

2 files changed

+215
-40
lines changed

‎src/Aardvark.OpenCV.Tests/Tests/ImageTests.fs‎

Lines changed: 104 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
namespace Aardvark.OpenCV.Tests
22

3+
open System
34
open Aardvark.Base
45
open NUnit.Framework
56
open FsUnit
@@ -10,13 +11,35 @@ module ``Image Processing Tests`` =
1011

1112
module private PixImage =
1213

13-
let random (size : V2i) =
14-
let pi = PixImage<float32>(Col.Format.RGBA, size)
14+
let private randomValues : Type -> (unit -> obj) =
15+
LookupTable.lookup [
16+
typeof<float32>, rnd.UniformFloat >> box
17+
typeof<uint16>, rnd.UniformUInt >> uint16 >> box
18+
typeof<int32>, rnd.UniformInt >> box
19+
]
20+
21+
let private getRandomValue<'T>() =
22+
let get = typeof<'T> |> randomValues
23+
get() |> unbox<'T>
24+
25+
let random<'T> (format: Col.Format) (size : V2i) =
26+
let pi = PixImage<'T>(format, size)
1527
for c in pi.ChannelArray do
16-
c.SetByIndex(ignore >> rnd.UniformFloat) |> ignore
28+
c.SetByIndex(ignore >> getRandomValue<'T>) |> ignore
1729

1830
pi
1931

32+
let generate<'T> (format: Col.Format) (sub: bool) =
33+
let size = V2i(512) + rnd.UniformV2i(1024)
34+
let pi = random<'T> format size
35+
if sub then
36+
let min = 10
37+
let offset = rnd.UniformV2i(size - min - 1)
38+
let size = min + rnd.UniformV2i(size - offset - min - 1)
39+
new PixImage<'T>(pi.Format, pi.Volume.SubVolume(offset.XYO, V3i(size, pi.ChannelCount)))
40+
else
41+
pi
42+
2043
[<DatapointSource>]
2144
let filters = [|
2245
ImageInterpolation.Near
@@ -25,27 +48,93 @@ module ``Image Processing Tests`` =
2548
ImageInterpolation.Lanczos
2649
|]
2750

51+
[<DatapointSource>]
52+
let borderTypes = [|
53+
ImageBorderType.Const
54+
ImageBorderType.Repl
55+
ImageBorderType.Wrap
56+
ImageBorderType.Mirror
57+
|]
58+
2859
[<SetUp>]
2960
let init() =
3061
Aardvark.Init()
3162

3263
[<Theory>]
3364
let ``Scaling`` (filter: ImageInterpolation) (sub: bool) =
34-
let size = V2i(512) + rnd.UniformV2i(1024)
65+
let src = PixImage.generate<float32> Col.Format.RGBA sub
3566
let scaleFactor = V2d(0.2345, 1.6789)
3667

37-
let src =
38-
let pi = PixImage.random size
39-
if sub then
40-
let min = 10
41-
let offset = rnd.UniformV2i(size - min - 1)
42-
let size = min + rnd.UniformV2i(size - offset - min - 1)
43-
new PixImage<float32>(pi.Format, pi.Volume.SubVolume(offset.XYO, V3i(size, pi.ChannelCount)))
44-
else
45-
pi
46-
4768
let result = Aardvark.OpenCV.PixProcessor.Instance.Scale(src, scaleFactor, filter)
4869
let reference = Aardvark.Base.PixProcessor.Instance.Scale(src, scaleFactor, filter)
4970

5071
let psnr = PixImage.peakSignalToNoiseRatio result reference
51-
psnr |> should be (greaterThan 20.0)
72+
psnr |> should be (greaterThan 20.0)
73+
74+
[<Theory>]
75+
let ``Remapping`` (sub: bool) =
76+
let src = PixImage.generate<float32> Col.Format.RGBA sub
77+
78+
let result =
79+
let mapX = Matrix<float32>(src.Size)
80+
mapX.SetByCoord(fun x _ -> float32 (x * 2L)) |> ignore
81+
82+
let mapY = Matrix<float32>(src.Size)
83+
mapY.SetByCoord(fun _ y -> float32 (src.SizeL.Y - y - 1L)) |> ignore
84+
85+
Aardvark.OpenCV.PixProcessor.Instance.Remap(src, mapX, mapY, ImageInterpolation.Near)
86+
87+
let reference =
88+
let pi = src.Transformed(ImageTrafo.MirrorY)
89+
90+
pi.Volume.SetByCoord(fun x y c ->
91+
if x * 2L < pi.SizeL.X then
92+
pi.Volume.[x * 2L, y, c]
93+
else
94+
0.0f
95+
) |> ignore
96+
97+
pi
98+
99+
let psnr = PixImage.peakSignalToNoiseRatio result reference
100+
psnr |> should be (greaterThan 20.0)
101+
102+
[<Theory>]
103+
let ``Remapping (border type)`` (borderType: ImageBorderType) (sub: bool) =
104+
let src = PixImage.generate<uint16> Col.Format.BGRA sub
105+
let border = [| 1us; 2us; 3us; 4us |]
106+
107+
let result =
108+
let mapX = Matrix<float32>(src.Size)
109+
let mapY = Matrix<float32>(src.Size)
110+
111+
if borderType = ImageBorderType.Const then
112+
mapX.Set -1.0f |> ignore
113+
else
114+
mapX.SetByCoord(fun x _ -> float32 (src.SizeL.X + x)) |> ignore
115+
mapY.SetByCoord(fun _ y -> float32 y) |> ignore
116+
117+
Aardvark.OpenCV.PixProcessor.Instance.Remap(src, mapX, mapY, ImageInterpolation.Near, borderType, border)
118+
119+
let expected =
120+
match borderType with
121+
| ImageBorderType.Const ->
122+
let pi = new PixImage<uint16>(result.Format, result.Size)
123+
pi.GetMatrix<C4us>().Set(C4us border) |> ignore
124+
pi
125+
126+
| ImageBorderType.Repl ->
127+
let pi = new PixImage<uint16>(result.Format, result.Size)
128+
pi.Volume.SetByCoord(fun _ y c -> src.Volume.[src.Size.X - 1, y, c]) |> ignore
129+
pi
130+
131+
| ImageBorderType.Wrap ->
132+
src
133+
134+
| ImageBorderType.Mirror ->
135+
src.Transformed(ImageTrafo.MirrorX)
136+
137+
| _ -> raise <| NotImplementedException()
138+
139+
let psnr = PixImage.peakSignalToNoiseRatio result expected
140+
psnr |> should equal infinity

‎src/Aardvark.OpenCV/ImageProcessing.cs‎

Lines changed: 111 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
namespace Aardvark.OpenCV
99
{
10+
#region Utilities
11+
1012
internal readonly struct GCHandleDiposable : IDisposable
1113
{
1214
public GCHandle Handle { get; }
@@ -40,33 +42,63 @@ public static MatType ToMatType(this Type type, int channels)
4042
else throw new NotSupportedException($"Channel type {type} is not supported.");
4143
}
4244

43-
private static readonly Dictionary<ImageInterpolation, InterpolationFlags> interpolationFlags = new()
45+
public static InterpolationFlags ToInterpolationFlags(this ImageInterpolation interpolation, bool exact)
46+
=> interpolation switch
47+
{
48+
ImageInterpolation.Near when exact => (InterpolationFlags)6,
49+
ImageInterpolation.Near => InterpolationFlags.Nearest,
50+
ImageInterpolation.Linear when exact => InterpolationFlags.LinearExact,
51+
ImageInterpolation.Linear => InterpolationFlags.Linear,
52+
ImageInterpolation.Cubic => InterpolationFlags.Cubic,
53+
ImageInterpolation.Lanczos => InterpolationFlags.Lanczos4,
54+
_ => throw new NotSupportedException($"Filter {interpolation} is not supported.")
55+
};
56+
57+
public static BorderTypes ToBorderTypes(this ImageBorderType borderType)
58+
=> borderType switch
59+
{
60+
ImageBorderType.Const => BorderTypes.Constant,
61+
ImageBorderType.Repl => BorderTypes.Replicate,
62+
ImageBorderType.Wrap => BorderTypes.Wrap,
63+
ImageBorderType.Mirror => BorderTypes.Reflect,
64+
_ => throw new NotSupportedException($"Border type {borderType} is not supported.")
65+
};
66+
67+
private static GCHandleDiposable ToMat(this Array data, V2l size, V2l delta, long firstIndex, MatType matType, int elementSize, out CvMat result)
4468
{
45-
{ ImageInterpolation.Near, (InterpolationFlags)6 }, // INTER_NEAREST_EXACT
46-
{ ImageInterpolation.Linear, InterpolationFlags.Linear },
47-
{ ImageInterpolation.Cubic, InterpolationFlags.Cubic },
48-
{ ImageInterpolation.Lanczos, InterpolationFlags.Lanczos4 },
49-
};
69+
var gc = data.Pin();
5070

51-
public static InterpolationFlags ToInterpolationFlags(this ImageInterpolation interpolation)
52-
{
53-
if (interpolationFlags.TryGetValue(interpolation, out InterpolationFlags flags)) return flags;
54-
else throw new NotSupportedException($"Filter {interpolation} is not supported.");
71+
var ptr = IntPtr.Add(gc.AddrOfPinnedObject(), (int)firstIndex * elementSize);
72+
var sizei = size.ToV2i();
73+
var deltai = delta.YX * elementSize;
74+
75+
result = CvMat.FromPixelData(sizei.YX.ToArray(), matType, ptr, deltai.ToArray());
76+
return gc;
5577
}
78+
79+
public static GCHandleDiposable ToMat<T>(this Matrix<T> matrix, MatType matType, int elementSize, out CvMat result)
80+
=> matrix.Data.ToMat(matrix.Size, matrix.Delta, matrix.FirstIndex, matType, elementSize, out result);
81+
82+
public static GCHandleDiposable ToMat<T>(this Volume<T> volume, MatType matType, int elementSize, out CvMat result)
83+
=> volume.Data.ToMat(volume.Size.XY, volume.Delta.XY, volume.FirstIndex, matType, elementSize, out result);
5684
}
5785

86+
#endregion
87+
5888
public sealed class PixProcessor : IPixProcessor
5989
{
6090
public string Name => "OpenCV";
6191

62-
public PixProcessorCaps Capabilities => PixProcessorCaps.Scale;
92+
public PixProcessorCaps Capabilities => PixProcessorCaps.Scale | PixProcessorCaps.Remap;
6393

6494
[OnAardvarkInit]
6595
public static void Init()
6696
{
6797
PixImage.AddProcessor(Instance);
6898
}
6999

100+
#region Scale
101+
70102
public PixImage<T> Scale<T>(PixImage<T> image, V2d scaleFactor, ImageInterpolation interpolation)
71103
{
72104
var src = image.Volume;
@@ -84,34 +116,88 @@ public PixImage<T> Scale<T>(PixImage<T> image, V2d scaleFactor, ImageInterpolati
84116

85117
var dst = newSize.CreateImageVolume<T>();
86118

87-
var matType = typeof(T).ToMatType((int)src.SZ);
119+
var matType = typeof(T).ToMatType(image.ChannelCount);
88120
var elementSize = typeof(T).GetCLRSize();
89121

90-
using var srcGC = src.Data.Pin();
91-
var srcPtr = IntPtr.Add(srcGC.AddrOfPinnedObject(), (int)src.FirstIndex * elementSize);
92-
var srcSize = src.Size.XY.ToV2i();
93-
var srcDelta = src.Delta.YX * elementSize;
94-
95-
using var dstGC = dst.Data.Pin();
96-
var dstPtr = dstGC.AddrOfPinnedObject();
97-
var dstSize = dst.Size.XY.ToV2i();
122+
using var _src = src.ToMat(matType, elementSize, out var srcMat);
123+
using var _dst = dst.ToMat(matType, elementSize, out var dstMat);
98124

99-
var srcMat = CvMat.FromPixelData(srcSize.YX.ToArray(), matType, srcPtr, srcDelta.ToArray());
100-
var dstMat = CvMat.FromPixelData(dstSize.Y, dstSize.X, matType, dstPtr);
101-
Cv2.Resize(srcMat, dstMat, new Size(dstSize.X, dstSize.Y), interpolation: interpolation.ToInterpolationFlags());
125+
Cv2.Resize(
126+
srcMat, dstMat,
127+
new Size((int)dst.SX, (int)dst.SY),
128+
interpolation: interpolation.ToInterpolationFlags(true)
129+
);
102130

103131
return new (image.Format, dst);
104132
}
105133

134+
#endregion
135+
136+
#region Rotate
137+
106138
public PixImage<T> Rotate<T>(PixImage<T> image, double angleInRadians, bool resize, ImageInterpolation interpolation,
107139
ImageBorderType borderType = ImageBorderType.Const,
108140
T border = default)
109141
=> null;
110142

143+
#endregion
144+
145+
#region Remap
146+
147+
// TODO: Change the interface to accept T[] as border value
148+
/// <inheritdoc cref="Remap{T}(PixImage{T}, Matrix{float}, Matrix{float}, ImageInterpolation, ImageBorderType, T)"/>
111149
public PixImage<T> Remap<T>(PixImage<T> image, Matrix<float> mapX, Matrix<float> mapY, ImageInterpolation interpolation,
112150
ImageBorderType borderType = ImageBorderType.Const,
113-
T border = default)
114-
=> null;
151+
T[] border = default)
152+
{
153+
var src = image.Volume;
154+
155+
if (!src.HasImageWindowLayout())
156+
{
157+
throw new ArgumentException($"Volume must be in image layout (Delta = {src.Delta}).");
158+
}
159+
160+
if (mapX.Size != mapY.Size)
161+
{
162+
throw new ArgumentException($"Size of coordinate maps must match (mapX: {mapX.Size}, mapY: {mapY.Size}).");
163+
}
164+
165+
var matType = typeof(T).ToMatType(image.ChannelCount);
166+
var elementSize = typeof(T).GetCLRSize();
167+
168+
var dstSize = new V3l(mapX.Size, image.ChannelCountL);
169+
var dst = dstSize.CreateImageVolume<T>();
170+
171+
using var _s = src.ToMat(matType, elementSize, out var srcMat);
172+
using var _d = dst.ToMat(matType, elementSize, out var dstMat);
173+
using var _x = mapX.ToMat(MatType.CV_32FC1, 4, out var mapXMat);
174+
using var _y = mapY.ToMat(MatType.CV_32FC1, 4, out var mapYMat);
175+
176+
Scalar borderValue = new();
177+
if (borderType == ImageBorderType.Const && border != null)
178+
{
179+
var order = image.Format.ChannelOrder();
180+
181+
for (var i = 0; i < Fun.Min(border.Length, 4); i++)
182+
borderValue[i] = Convert.ToDouble(border[order[i]]);
183+
}
184+
185+
Cv2.Remap(
186+
srcMat, dstMat, mapXMat, mapYMat,
187+
interpolation.ToInterpolationFlags(false),
188+
borderType.ToBorderTypes(),
189+
borderValue
190+
);
191+
192+
return new (image.Format, dst);
193+
}
194+
195+
public PixImage<T> Remap<T>(PixImage<T> image, Matrix<float> mapX, Matrix<float> mapY, ImageInterpolation interpolation,
196+
ImageBorderType borderType,
197+
T border)
198+
=> Remap(image, mapX, mapY, interpolation, borderType, new T[] { border, border, border, border });
199+
200+
#endregion
115201

116202
private PixProcessor() { }
117203

0 commit comments

Comments
 (0)