Merge pull request #81 from nut-tree/feature/68/scaled_search

s1hofmann · web-flow · commit bdf3452f16c0 · 2019-06-17T01:33:59.000+02:00
Closes #68
diff --git a/lib/provider/opencv/__mocks__/needle.png b/lib/provider/opencv/__mocks__/needle.png
diff --git a/lib/provider/opencv/bound-value.function.spec.ts b/lib/provider/opencv/bound-value.function.spec.ts
@@ -0,0 +1,33 @@
+import { lowerBound, upperBound } from "./bound-value.function";
+
+describe("lowerBound function", () => {
+  it.each([
+    [5, 10, 1, 1],
+    [5, 5, 10, 10],
+    [5, 1, 10, 5],
+    [0, 0, 0, 0]
+  ])("Input: %f, Boundary: %f, minValue: %f, Expected: %f",
+    (input: number, boundary: number, minValue: number, expected: number) => {
+      // WHEN
+      const result = lowerBound(input, boundary, minValue);
+
+      // THEN
+      expect(result).toBe(expected);
+    });
+});
+
+describe("upperBound function", () => {
+  it.each([
+    [5, 10, 1, 5],
+    [5, 5, 10, 10],
+    [5, 1, 10, 10],
+    [5, 5, 5, 5]
+  ])("Input: %f, Boundary: %f, maxValue: %f, Expected: %f",
+    (input: number, boundary: number, maxValue: number, expected: number) => {
+      // WHEN
+      const result = upperBound(input, boundary, maxValue);
+
+      // THEN
+      expect(result).toBe(expected);
+    });
+});
diff --git a/lib/provider/opencv/bound-value.function.ts b/lib/provider/opencv/bound-value.function.ts
@@ -0,0 +1,7 @@
+export function lowerBound(value: number, boundary: number, minValue: number): number {
+  return (value <= boundary) ? minValue : value;
+}
+
+export function upperBound(value: number, boundary: number, maxValue: number): number {
+  return (value >= boundary) ? maxValue : value;
+}
diff --git a/lib/provider/opencv/determine-searchregion.function.spec.ts b/lib/provider/opencv/determine-searchregion.function.spec.ts
@@ -0,0 +1,60 @@
+import { mockPartial } from "sneer";
+import { Image } from "../../image.class";
+import { MatchRequest } from "../../match-request.class";
+import { Region } from "../../region.class";
+import { determineScaledSearchRegion } from "./determine-searchregion.function";
+
+describe("determineSearchRegion", () => {
+  it("should return a search region adopted to pixel density", () => {
+    // GIVEN
+    const imageMock = mockPartial<Image>({
+      pixelDensity: {
+        scaleX: 1.5,
+        scaleY: 2.0
+      }
+    });
+    const needlePath = "/path/to/needle";
+    const inputSearchRegion = new Region(0, 0, 100, 100);
+    const expectedSearchRegion = new Region(0, 0, 150, 200);
+
+    const matchRequest = new MatchRequest(
+      imageMock,
+      needlePath,
+      inputSearchRegion,
+      0.99
+    );
+
+    // WHEN
+    const result = determineScaledSearchRegion(matchRequest);
+
+    // THEN
+    expect(result).toEqual(expectedSearchRegion);
+  });
+
+  it.each([[0, 1], [1, 0]])("should not adjust searchregion for factor 0: scaleX: %i scaleY: %i",
+    (scaleX: number, scaleY: number) => {
+      // GIVEN
+      const imageMock = mockPartial<Image>({
+        pixelDensity: {
+          scaleX,
+          scaleY
+        }
+      });
+      const needlePath = "/path/to/needle";
+      const inputSearchRegion = new Region(0, 0, 100, 100);
+      const expectedSearchRegion = new Region(0, 0, 100, 100);
+
+      const matchRequest = new MatchRequest(
+        imageMock,
+        needlePath,
+        inputSearchRegion,
+        0.99
+      );
+
+      // WHEN
+      const result = determineScaledSearchRegion(matchRequest);
+
+      // THEN
+      expect(result).toEqual(expectedSearchRegion);
+    });
+});
diff --git a/lib/provider/opencv/determine-searchregion.function.ts b/lib/provider/opencv/determine-searchregion.function.ts
@@ -0,0 +1,11 @@
+import { MatchRequest } from "../../match-request.class";
+import { Region } from "../../region.class";
+
+export function determineScaledSearchRegion(matchRequest: MatchRequest): Region {
+  const searchRegion = matchRequest.searchRegion;
+  const scaleX = matchRequest.haystack.pixelDensity.scaleX || 1.0;
+  const scaleY = matchRequest.haystack.pixelDensity.scaleY || 1.0;
+  searchRegion.width *= scaleX;
+  searchRegion.height *= scaleY;
+  return searchRegion;
+}
diff --git a/lib/provider/opencv/find-edges.function.spec.ts b/lib/provider/opencv/find-edges.function.spec.ts
@@ -0,0 +1,22 @@
+import * as cv from "opencv4nodejs-prebuilt";
+import { mockPartial } from "sneer";
+import { findEdges } from "./find-edges.function";
+
+describe("findEdges", () => {
+  it("should convert an image to grayscale and run Canny edge detection", async () => {
+    // GIVEN
+    const grayImageMock = mockPartial<cv.Mat>({
+      cannyAsync: jest.fn()
+    });
+    const inputImageMock = mockPartial<cv.Mat>({
+      cvtColorAsync: jest.fn(() => Promise.resolve(grayImageMock))
+    });
+
+    // WHEN
+    await findEdges(inputImageMock);
+
+    // THEN
+    expect(inputImageMock.cvtColorAsync).toBeCalledWith(cv.COLOR_BGR2GRAY);
+    expect(grayImageMock.cannyAsync).toBeCalledWith(50, 200);
+  });
+});
diff --git a/lib/provider/opencv/find-edges.function.ts b/lib/provider/opencv/find-edges.function.ts
@@ -0,0 +1,6 @@
+import * as cv from "opencv4nodejs-prebuilt";
+
+export async function findEdges(image: cv.Mat): Promise<cv.Mat> {
+  const gray = await image.cvtColorAsync(cv.COLOR_BGR2GRAY);
+  return gray.cannyAsync(50, 200);
+}
diff --git a/lib/provider/opencv/image-processor.class.ts b/lib/provider/opencv/image-processor.class.ts
@@ -2,13 +2,13 @@ import * as cv from "opencv4nodejs-prebuilt";
 import { Image } from "../../image.class";
 import { Region } from "../../region.class";
 
-const determineROI = (img: Image, roi: Region): cv.Rect => {
+function determineROI(img: Image, roi: Region): cv.Rect {
   return new cv.Rect(
     Math.min(Math.max(roi.left, 0), img.width),
     Math.min(Math.max(roi.top, 0), img.height),
     Math.min(roi.width, img.width - roi.left),
     Math.min(roi.height, img.height - roi.top));
-};
+}
 
 export class ImageProcessor {
   /**
diff --git a/lib/provider/opencv/match-image.function.spec.ts b/lib/provider/opencv/match-image.function.spec.ts
@@ -0,0 +1,41 @@
+import * as cv from "opencv4nodejs-prebuilt";
+import { mockPartial } from "sneer";
+import { matchImages } from "./match-image.function";
+
+describe("matchImages", () => {
+  it("should return minLoc position and needle size", async () => {
+    // GIVEN
+    const minLocX = 100;
+    const minLocY = 1000;
+    const matchMock = mockPartial<cv.Mat>({
+      minMaxLocAsync: jest.fn(() => Promise.resolve({
+        maxLoc: new cv.Point2(
+          200,
+          2000
+        ),
+        maxVal: 100,
+        minLoc: new cv.Point2(
+          minLocX,
+          minLocY
+        ),
+        minVal: 0,
+      }))
+    });
+    const haystackMock = mockPartial<cv.Mat>({
+      matchTemplateAsync: jest.fn(() => Promise.resolve(matchMock))
+    });
+    const needleMock = mockPartial<cv.Mat>({
+      cols: 123,
+      rows: 456
+    });
+
+    // WHEN
+    const result = await matchImages(haystackMock, needleMock);
+
+    // THEN
+    expect(result.location.left).toEqual(minLocX);
+    expect(result.location.top).toEqual(minLocY);
+    expect(result.location.width).toEqual(needleMock.cols);
+    expect(result.location.height).toEqual(needleMock.rows);
+  });
+});
diff --git a/lib/provider/opencv/match-image.function.ts b/lib/provider/opencv/match-image.function.ts
@@ -0,0 +1,20 @@
+import * as cv from "opencv4nodejs-prebuilt";
+import { MatchResult } from "../../match-result.class";
+import { Region } from "../../region.class";
+
+export async function matchImages(haystack: cv.Mat, needle: cv.Mat): Promise<MatchResult> {
+  const match = await haystack.matchTemplateAsync(
+    needle,
+    cv.TM_SQDIFF_NORMED,
+  );
+  const minMax = await match.minMaxLocAsync();
+  return new MatchResult(
+    1.0 - minMax.minVal,
+    new Region(
+      minMax.minLoc.x,
+      minMax.minLoc.y,
+      needle.cols,
+      needle.rows,
+    ),
+  );
+}
diff --git a/lib/provider/opencv/scale-image.function.spec.ts b/lib/provider/opencv/scale-image.function.spec.ts
@@ -0,0 +1,40 @@
+import * as path from "path";
+import { ImageProcessor } from "./image-processor.class";
+import { ImageReader } from "./image-reader.class";
+import { scaleImage } from "./scale-image.function";
+
+describe("scaleImage", () => {
+  it.each([[0.5], [1.5]])("should scale an image correctly by factor %f", async (scaleFactor) => {
+    // GIVEN
+    const imageLoader = new ImageReader();
+    const pathToinput = path.resolve(__dirname, "./__mocks__/mouse.png");
+    const inputImage = await imageLoader.load(pathToinput);
+    const inputMat = await ImageProcessor.fromImageWithoutAlphaChannel(inputImage);
+    const expectedWidth = Math.floor(inputMat.cols * scaleFactor);
+    const expectedHeight = Math.floor(inputMat.rows * scaleFactor);
+
+    // WHEN
+    const result = await scaleImage(inputMat, scaleFactor);
+
+    // THEN
+    expect(result.rows).toBe(expectedHeight);
+    expect(result.cols).toBe(expectedWidth);
+  });
+
+  it.each([[0], [-0.25]])("should keep scale if factor <= 0: Scale %f", async (scaleFactor) => {
+    // GIVEN
+    const imageLoader = new ImageReader();
+    const pathToinput = path.resolve(__dirname, "./__mocks__/mouse.png");
+    const inputImage = await imageLoader.load(pathToinput);
+    const inputMat = await ImageProcessor.fromImageWithoutAlphaChannel(inputImage);
+    const expectedWidth = inputMat.cols;
+    const expectedHeight = inputMat.rows;
+
+    // WHEN
+    const result = await scaleImage(inputMat, scaleFactor);
+
+    // THEN
+    expect(result.rows).toBe(expectedHeight);
+    expect(result.cols).toBe(expectedWidth);
+  });
+});
diff --git a/lib/provider/opencv/scale-image.function.ts b/lib/provider/opencv/scale-image.function.ts
@@ -0,0 +1,9 @@
+import * as cv from "opencv4nodejs-prebuilt";
+import { lowerBound } from "./bound-value.function";
+
+export async function scaleImage(image: cv.Mat, scaleFactor: number): Promise<cv.Mat> {
+  const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0);
+  const scaledRows = Math.floor(image.rows * boundScaleFactor);
+  const scaledCols = Math.floor(image.cols * boundScaleFactor);
+  return image.resizeAsync(scaledRows, scaledCols, 0, 0, cv.INTER_AREA);
+}
diff --git a/lib/provider/opencv/scale-location.function.spec.ts b/lib/provider/opencv/scale-location.function.spec.ts
@@ -0,0 +1,30 @@
+import { Region } from "../../region.class";
+import { scaleLocation } from "./scale-location.function";
+
+describe("scaleLocation", () => {
+  it("should scale location of a Region for valid scale factors", () => {
+    // GIVEN
+    const scaleFactor = 0.5;
+    const inputRegion = new Region(100, 100, 10, 10);
+    const expectedRegion = new Region(200, 200, 10, 10);
+
+    // WHEN
+    const result = scaleLocation(inputRegion, scaleFactor);
+
+    // THEN
+    expect(result).toEqual(expectedRegion);
+  });
+
+  it("should not scale location of a Region for invalid scale factors", () => {
+    // GIVEN
+    const scaleFactor = 0.0;
+    const inputRegion = new Region(100, 100, 10, 10);
+    const expectedRegion = new Region(100, 100, 10, 10);
+
+    // WHEN
+    const result = scaleLocation(inputRegion, scaleFactor);
+
+    // THEN
+    expect(result).toEqual(expectedRegion);
+  });
+});
diff --git a/lib/provider/opencv/scale-location.function.ts b/lib/provider/opencv/scale-location.function.ts
@@ -0,0 +1,15 @@
+import { Region } from "../../region.class";
+import { lowerBound } from "./bound-value.function";
+
+export function scaleLocation(
+  result: Region,
+  scaleFactor: number,
+): Region {
+  const boundScaleFactor = lowerBound(scaleFactor, 0.0, 1.0);
+  return new Region(
+    result.left / boundScaleFactor,
+    result.top / boundScaleFactor,
+    result.width,
+    result.height,
+  );
+}
diff --git a/lib/provider/opencv/template-matching-finder.class.spec.ts b/lib/provider/opencv/template-matching-finder.class.spec.ts
@@ -10,38 +10,42 @@ describe("Template-matching finder", () => {
     // GIVEN
     const imageLoader = new ImageReader();
     const SUT = new TemplateMatchingFinder();
-    const imagePath = path.resolve(__dirname, "./__mocks__/mouse.png");
-    const needle = await imageLoader.load(imagePath);
+    const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png");
+    const needlePath = path.resolve(__dirname, "./__mocks__/needle.png");
+    const haystack = await imageLoader.load(haystackPath);
+    const needle = await imageLoader.load(needlePath);
     const minConfidence = 0.99;
-    const searchRegion = new Region(0, 0, needle.width, needle.height);
-    const haystack = new Image(needle.width, needle.height, needle.data, 3);
-    const matchRequest = new MatchRequest(haystack, imagePath, searchRegion, minConfidence);
+    const searchRegion = new Region(0, 0, haystack.width, haystack.height);
+    const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence);
+    const expectedResult = new Region(16, 31, needle.width, needle.height);
 
     // WHEN
     const result = await SUT.findMatch(matchRequest);
 
     // THEN
     expect(result.confidence).toBeGreaterThanOrEqual(minConfidence);
-    expect(result.location).toEqual(searchRegion);
+    expect(result.location).toEqual(expectedResult);
   });
 
   it("findMatch should return a match within a search region when present in image", async () => {
     // GIVEN
     const imageLoader = new ImageReader();
     const SUT = new TemplateMatchingFinder();
-    const imagePath = path.resolve(__dirname, "./__mocks__/mouse.png");
-    const needle = await imageLoader.load(imagePath);
+    const haystackPath = path.resolve(__dirname, "./__mocks__/mouse.png");
+    const needlePath = path.resolve(__dirname, "./__mocks__/needle.png");
+    const haystack = await imageLoader.load(haystackPath);
+    const needle = await imageLoader.load(needlePath);
     const minConfidence = 0.99;
-    const searchRegion = new Region(10, 20, 100, 100);
-    const haystack = new Image(needle.width, needle.height, needle.data, 3);
-    const matchRequest = new MatchRequest(haystack, imagePath, searchRegion, minConfidence);
+    const searchRegion = new Region(10, 20, 140, 100);
+    const matchRequest = new MatchRequest(haystack, needlePath, searchRegion, minConfidence);
+    const expectedResult = new Region(6, 11, needle.width, needle.height);
 
     // WHEN
     const result = await SUT.findMatch(matchRequest);
 
     // THEN
     expect(result.confidence).toBeGreaterThanOrEqual(minConfidence);
-    expect(result.location).toEqual(searchRegion);
+    expect(result.location).toEqual(expectedResult);
   });
 
   it("findMatch should throw on invalid image paths", async () => {
diff --git a/lib/provider/opencv/template-matching-finder.class.ts b/lib/provider/opencv/template-matching-finder.class.ts
diff --git a/lib/scaled-match-result.class.ts b/lib/scaled-match-result.class.ts
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
