Skip to content

Commit 812b01a

Browse files
pm0733464pm0733464
committed
Added a class to work around an image size bug in the face detector.
See b/23709340. Third party developers can use this class to wrap their face detector instance. This avoids the bug by adding padding to images. Change-Id: Iae45e8780b09f8ce38e43003a1f99d71037197b8
1 parent 6d4d017 commit 812b01a

File tree

2 files changed

+192
-3
lines changed

2 files changed

+192
-3
lines changed

‎visionSamples/photo-demo/app/src/main/java/com/google/android/gms/samples/vision/face/patch/SafeFaceDetector.java

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*
2+
* Copyright (C) The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.android.gms.samples.vision.face.patch;
17+
18+
import android.graphics.ImageFormat;
19+
import android.util.Log;
20+
import android.util.SparseArray;
21+
22+
import com.google.android.gms.vision.Detector;
23+
import com.google.android.gms.vision.Frame;
24+
import com.google.android.gms.vision.face.Face;
25+
26+
import java.nio.ByteBuffer;
27+
import java.util.Arrays;
28+
29+
/**
30+
* This is a workaround for a bug in the face detector, in which either very small images (i.e.,
31+
* most images with dimension < 147) and very thin images can cause a crash in the native face
32+
* detection code. This will add padding to such images before face detection in order to avoid
33+
* this issue.<p>
34+
*
35+
* This is not necessary for use with the camera, which doesn't ever create these types of
36+
* images.<p>
37+
*
38+
* This detector should wrap the underlying FaceDetector instance, like this:
39+
*
40+
* Detector<Face> safeDetector = new SafeFaceDetector(faceDetector);
41+
*
42+
* Replace all remaining occurrences of faceDetector with safeDetector.
43+
*/
44+
public class SafeFaceDetector extends Detector<Face> {
45+
private static final String TAG = "SafeFaceDetector";
46+
private Detector<Face> mDelegate;
47+
48+
/**
49+
* Creates a safe face detector to wrap and protect an underlying face detector from images that
50+
* trigger the face detector bug.
51+
*/
52+
public SafeFaceDetector(Detector<Face> delegate) {
53+
mDelegate = delegate;
54+
}
55+
56+
@Override
57+
public void release() {
58+
mDelegate.release();
59+
}
60+
61+
/**
62+
* Determines whether the supplied image may cause a problem with the underlying face detector.
63+
* If it does, padding is added to the image in order to avoid the issue.
64+
*/
65+
@Override
66+
public SparseArray<Face> detect(Frame frame) {
67+
final int kMinDimension = 147;
68+
final int kDimensionLower = 640;
69+
int width = frame.getMetadata().getWidth();
70+
int height = frame.getMetadata().getHeight();
71+
72+
if (height > (2 * kDimensionLower)) {
73+
// The image will be scaled down before detection is run. Check to make sure that this
74+
// won't result in the width going below the minimum
75+
double multiple = (double) height / (double) kDimensionLower;
76+
double lowerWidth = Math.floor((double) width / multiple);
77+
if (lowerWidth < kMinDimension) {
78+
// The width would have gone below the minimum when downsampling, so apply padding
79+
// to the right to keep the width large enough.
80+
int newWidth = (int) Math.ceil(kMinDimension * multiple);
81+
frame = padFrameRight(frame, newWidth);
82+
}
83+
} else if (width > (2 * kDimensionLower)) {
84+
// The image will be scaled down before detection is run. Check to make sure that this
85+
// won't result in the height going below the minimum
86+
double multiple = (double) width / (double) kDimensionLower;
87+
double lowerHeight = Math.floor((double) height / multiple);
88+
if (lowerHeight < kMinDimension) {
89+
int newHeight = (int) Math.ceil(kMinDimension * multiple);
90+
frame = padFrameBottom(frame, newHeight);
91+
}
92+
} else if (width < kMinDimension) {
93+
frame = padFrameRight(frame, kMinDimension);
94+
}
95+
96+
return mDelegate.detect(frame);
97+
}
98+
99+
@Override
100+
public boolean isOperational() {
101+
return mDelegate.isOperational();
102+
}
103+
104+
@Override
105+
public boolean setFocus(int id) {
106+
return mDelegate.setFocus(id);
107+
}
108+
109+
/**
110+
* Creates a new frame based on the original frame, with additional width on the right to
111+
* increase the size to avoid the bug in the underlying face detector.
112+
*/
113+
private Frame padFrameRight(Frame originalFrame, int newWidth) {
114+
Frame.Metadata metadata = originalFrame.getMetadata();
115+
int width = metadata.getWidth();
116+
int height = metadata.getHeight();
117+
118+
Log.i(TAG, "Padded image from: " + width + "x" + height + " to " + newWidth + "x" + height);
119+
120+
ByteBuffer origBuffer = originalFrame.getGrayscaleImageData();
121+
int origOffset = origBuffer.arrayOffset();
122+
byte[] origBytes = origBuffer.array();
123+
124+
// This can be changed to just .allocate in the future, when Frame supports non-direct
125+
// byte buffers.
126+
ByteBuffer paddedBuffer = ByteBuffer.allocateDirect(newWidth * height);
127+
int paddedOffset = paddedBuffer.arrayOffset();
128+
byte[] paddedBytes = paddedBuffer.array();
129+
Arrays.fill(paddedBytes, (byte) 0);
130+
131+
for (int y = 0; y < height; ++y) {
132+
int origStride = origOffset + y * width;
133+
int paddedStride = paddedOffset + y * newWidth;
134+
System.arraycopy(origBytes, origStride, paddedBytes, paddedStride, width);
135+
}
136+
137+
return new Frame.Builder()
138+
.setImageData(paddedBuffer, newWidth, height, ImageFormat.NV21)
139+
.setId(metadata.getId())
140+
.setRotation(metadata.getRotation())
141+
.setTimestampMillis(metadata.getTimestampMillis())
142+
.build();
143+
}
144+
145+
/**
146+
* Creates a new frame based on the original frame, with additional height on the bottom to
147+
* increase the size to avoid the bug in the underlying face detector.
148+
*/
149+
private Frame padFrameBottom(Frame originalFrame, int newHeight) {
150+
Frame.Metadata metadata = originalFrame.getMetadata();
151+
int width = metadata.getWidth();
152+
int height = metadata.getHeight();
153+
154+
Log.i(TAG, "Padded image from: " + width + "x" + height + " to " + width + "x" + newHeight);
155+
156+
ByteBuffer origBuffer = originalFrame.getGrayscaleImageData();
157+
int origOffset = origBuffer.arrayOffset();
158+
byte[] origBytes = origBuffer.array();
159+
160+
// This can be changed to just .allocate in the future, when Frame supports non-direct
161+
// byte buffers.
162+
ByteBuffer paddedBuffer = ByteBuffer.allocateDirect(width * newHeight);
163+
int paddedOffset = paddedBuffer.arrayOffset();
164+
byte[] paddedBytes = paddedBuffer.array();
165+
Arrays.fill(paddedBytes, (byte) 0);
166+
167+
// Copy the image content from the original, without bothering to fill in the padded bottom
168+
// part.
169+
for (int y = 0; y < height; ++y) {
170+
int origStride = origOffset + y * width;
171+
int paddedStride = paddedOffset + y * width;
172+
System.arraycopy(origBytes, origStride, paddedBytes, paddedStride, width);
173+
}
174+
175+
return new Frame.Builder()
176+
.setImageData(paddedBuffer, width, newHeight, ImageFormat.NV21)
177+
.setId(metadata.getId())
178+
.setRotation(metadata.getRotation())
179+
.setTimestampMillis(metadata.getTimestampMillis())
180+
.build();
181+
}
182+
}

‎visionSamples/photo-demo/app/src/main/java/com/google/android/gms/samples/vision/face/photo/PhotoViewerActivity.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import android.util.Log;
2323
import android.util.SparseArray;
2424

25+
import com.google.android.gms.samples.vision.face.patch.SafeFaceDetector;
26+
import com.google.android.gms.vision.Detector;
2527
import com.google.android.gms.vision.Frame;
2628
import com.google.android.gms.vision.face.Face;
2729
import com.google.android.gms.vision.face.FaceDetector;
@@ -58,11 +60,16 @@ protected void onCreate(Bundle savedInstanceState) {
5860
.setLandmarkType(FaceDetector.ALL_LANDMARKS)
5961
.build();
6062

63+
// This is a temporary workaround for a bug in the face detector with respect to operating
64+
// on very small images. This will be fixed in a future release. But in the near term, use
65+
// of the SafeFaceDetector class will patch the issue.
66+
Detector<Face> safeDetector = new SafeFaceDetector(detector);
67+
6168
// Create a frame from the bitmap and run face detection on the frame.
6269
Frame frame = new Frame.Builder().setBitmap(bitmap).build();
63-
SparseArray<Face> faces = detector.detect(frame);
70+
SparseArray<Face> faces = safeDetector.detect(frame);
6471

65-
if (!detector.isOperational()) {
72+
if (!safeDetector.isOperational()) {
6673
// Note: The first time that an app using face API is installed on a device, GMS will
6774
// download a native library to the device in order to do detection. Usually this
6875
// completes before the app is run for the first time. But if that download has not yet
@@ -79,6 +86,6 @@ protected void onCreate(Bundle savedInstanceState) {
7986

8087
// Although detector may be used multiple times for different images, it should be released
8188
// when it is no longer needed in order to free native resources.
82-
detector.release();
89+
safeDetector.release();
8390
}
8491
}

0 commit comments

Comments
 (0)