Skip to content

Commit 58cd257

Browse files
Vladimir KocheryzhkinVladimir Kocheryzhkin
Vladimir Kocheryzhkin
authored and
Vladimir Kocheryzhkin
committed
basic dirty recognition
1 parent 6bc6767 commit 58cd257

File tree

4 files changed

+252
-79
lines changed

4 files changed

+252
-79
lines changed

‎visionSamples/FaceTracker/app/src/main/cpp/native-lib.cpp

Lines changed: 173 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
#include <jni.h>
22
#include <string>
3+
#include <dirent.h>
4+
#include <unordered_map>
35
#include <android/log.h>
46
#include <android/bitmap.h>
7+
#include <dlib/image_io.h>
58

69
#include <dlib/dnn.h>
710
#include <dlib/image_processing/frontal_face_detector.h>
@@ -15,6 +18,47 @@ using namespace dlib;
1518
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)
1619
#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)
1720

21+
22+
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
23+
using residual = dlib::add_prev1<block<N,BN,1,dlib::tag1<SUBNET>>>;
24+
25+
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
26+
using residual_down = dlib::add_prev2<dlib::avg_pool<2,2,2,2,dlib::skip1<dlib::tag2<block<N,BN,2,dlib::tag1<SUBNET>>>>>>;
27+
28+
template <int N, template <typename> class BN, int stride, typename SUBNET>
29+
using block = BN<dlib::con<N,3,3,1,1,dlib::relu<BN<dlib::con<N,3,3,stride,stride,SUBNET>>>>>;
30+
31+
template <int N, typename SUBNET> using ares = dlib::relu<residual<block,N,dlib::affine,SUBNET>>;
32+
template <int N, typename SUBNET> using ares_down = dlib::relu<residual_down<block,N,dlib::affine,SUBNET>>;
33+
34+
template <typename SUBNET> using alevel0 = ares_down<256,SUBNET>;
35+
template <typename SUBNET> using alevel1 = ares<256,ares<256,ares_down<256,SUBNET>>>;
36+
template <typename SUBNET> using alevel2 = ares<128,ares<128,ares_down<128,SUBNET>>>;
37+
template <typename SUBNET> using alevel3 = ares<64,ares<64,ares<64,ares_down<64,SUBNET>>>>;
38+
template <typename SUBNET> using alevel4 = ares<32,ares<32,ares<32,SUBNET>>>;
39+
40+
using anet_type = dlib::loss_metric<dlib::fc_no_bias<128, dlib::avg_pool_everything<
41+
alevel0<
42+
alevel1<
43+
alevel2<
44+
alevel3<
45+
alevel4<
46+
dlib::max_pool<3,3,2,2,dlib::relu<dlib::affine<dlib::con<32,7,7,2,2,
47+
dlib::input_rgb_image_sized<150>
48+
>>>>>>>>>>>>;
49+
50+
dlib::frontal_face_detector detector = dlib::get_frontal_face_detector();
51+
dlib::shape_predictor sp;
52+
anet_type net;
53+
54+
//std::vector<matrix<float, 0, 1>> known_faces;
55+
56+
std::unordered_map<std::string, matrix<float, 0, 1>> known_faces;
57+
58+
typedef matrix<float, 0, 1> sample_type;
59+
typedef linear_kernel<sample_type> lin_kernel;
60+
multiclass_linear_decision_function<lin_kernel, string> df;
61+
1862
typedef struct
1963
{
2064
uint8_t alpha;
@@ -23,51 +67,144 @@ typedef struct
2367
uint8_t blue;
2468
} argb;
2569

70+
71+
2672
extern "C"
27-
JNIEXPORT jstring JNICALL
28-
Java_com_google_android_gms_samples_vision_face_facetracker_CustomDetector_test(JNIEnv *env,
29-
jobject instance,
30-
jobject bmp) {
31-
AndroidBitmapInfo infocolor;
32-
void* pixelscolor;
33-
int y;
34-
int x;
35-
int ret;
36-
if ((ret = AndroidBitmap_getInfo(env, bmp, &infocolor)) < 0) {
37-
LOGE("AndroidBitmap_getInfo() failed ! error=%d", ret);
38-
return env->NewStringUTF("Image broken");
39-
}
40-
LOGI("color image :: width is %d; height is %d; stride is %d; format is %d;flags is %d",
41-
infocolor.width,infocolor.height,infocolor.stride,infocolor.format,infocolor.flags);
73+
{
74+
JNIEXPORT jstring JNICALL
75+
Java_com_google_android_gms_samples_vision_face_facetracker_CustomDetector_test(JNIEnv *env,
76+
jobject instance,
77+
jobject bmp) {
78+
AndroidBitmapInfo infocolor;
79+
void *pixelscolor;
80+
int y;
81+
int x;
82+
int ret;
83+
array2d<rgb_pixel> img;
84+
if ((ret = AndroidBitmap_getInfo(env, bmp, &infocolor)) < 0) {
85+
LOGE("AndroidBitmap_getInfo() failed ! error=%d", ret);
86+
return env->NewStringUTF("Image broken");
87+
}
88+
LOGI("color image :: width is %d; height is %d; stride is %d; format is %d;flags is %d",
89+
infocolor.width, infocolor.height, infocolor.stride, infocolor.format, infocolor.flags);
4290

43-
if (infocolor.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
44-
LOGE("Bitmap format is not RGBA_8888 !");
45-
return env->NewStringUTF("Image broken 2");
46-
}
91+
if (infocolor.format != ANDROID_BITMAP_FORMAT_RGBA_8888) {
92+
LOGE("Bitmap format is not RGBA_8888 !");
93+
return env->NewStringUTF("Image broken 2");
94+
}
4795

48-
if ((ret = AndroidBitmap_lockPixels(env, bmp, &pixelscolor)) < 0) {
49-
LOGE("AndroidBitmap_lockPixels() failed ! error=%d", ret);
50-
}
96+
if ((ret = AndroidBitmap_lockPixels(env, bmp, &pixelscolor)) < 0) {
97+
LOGE("AndroidBitmap_lockPixels() failed ! error=%d", ret);
98+
}
99+
100+
img.set_size(infocolor.height, infocolor.width);
101+
for (y = 0; y < infocolor.height; y++) { //todo: performance
102+
argb *line = (argb *) pixelscolor;
103+
for (x = 0; x < infocolor.width; ++x) {
104+
rgb_pixel p(line[x].red, line[x].green, line[x].blue);
105+
img[y][x] = p;
106+
}
107+
pixelscolor = (char *) pixelscolor + infocolor.stride;
108+
}
109+
110+
//todo: smth wrong with colors
111+
//dlib::save_bmp(img, "/storage/emulated/0/Movies/test2.bmp");
112+
113+
std::vector<dlib::rectangle> dets = detector(img);
114+
LOGI("detected size %d", dets.size());
115+
116+
if(dets.size() > 0 ){
117+
auto face = dets.front();
118+
std::vector<matrix<rgb_pixel>> faces;
119+
int x = face.left();
120+
int y = face.top();
121+
int width = face.width();
122+
int height = face.height();
123+
124+
auto shape = sp(img, face);
125+
matrix<rgb_pixel> face_chip;
126+
extract_image_chip(img, get_face_chip_details(shape, 150, 0.25), face_chip);
127+
faces.push_back(move(face_chip));
51128

52-
for (y=0;y<infocolor.height;y++) {
53-
argb * line = (argb *) pixelscolor;
54-
pixelscolor = (char *)pixelscolor + infocolor.stride;
129+
std::vector<matrix<float, 0, 1>> face_descriptors = net(faces);
130+
131+
if (face_descriptors.size() > 0)
132+
{
133+
matrix<float, 0, 1> face_desc = face_descriptors[0];
134+
LOGI("recognized");
135+
for (auto& i : known_faces) {
136+
if( length(face_desc - i.second ) < 0.5) //todo: extrace thresh
137+
{
138+
return env->NewStringUTF(i.first.c_str());
139+
}
140+
}
141+
}
142+
}
143+
144+
LOGI("unlocking pixels");
145+
AndroidBitmap_unlockPixels(env, bmp);
146+
147+
const char *returnValue = "Unknown";
148+
return env->NewStringUTF(returnValue);
55149
}
150+
}
56151

57-
// array2d<rgb_pixel> img;
58-
// img.set_size(infocolor.height, infocolor.width);
59-
// for (int y = 0; y < infocolor.height; y++){
60-
// for (int x = 0; x < infocolor.width; x++){
61-
// img[y][x] = pixelscolor[y * infocolor.width + x];
62-
// }
63-
// }
152+
extern "C"
153+
JNIEXPORT jint JNICALL
154+
Java_com_google_android_gms_samples_vision_face_facetracker_FaceTrackerActivity_loadResources(
155+
JNIEnv *env, jobject instance)
156+
{
157+
158+
LOGI("load resource");
159+
FILE *file1 = fopen("/storage/emulated/0/Download/shape_predictor_5_face_landmarks.dat", "r+");
160+
FILE *file2 = fopen("/storage/emulated/0/Download/dlib_face_recognition_resnet_model_v1.dat",
161+
"r+");
162+
FILE *file3 = fopen("/storage/emulated/0/Download/faces_linear.svm", "r+");
64163

164+
if (file1 != NULL && file2 != NULL && file3 != NULL) {
165+
fclose(file1);
166+
fclose(file2);
167+
fclose(file3);
168+
dlib::deserialize("/storage/emulated/0/Download/shape_predictor_5_face_landmarks.dat")
169+
>> sp;
170+
dlib::deserialize("/storage/emulated/0/Download/dlib_face_recognition_resnet_model_v1.dat")
171+
>> net;
172+
dlib::deserialize("/storage/emulated/0/Download/faces_linear.svm") >> df;
65173

66-
LOGI("unlocking pixels");
67-
AndroidBitmap_unlockPixels(env, bmp);
174+
DIR *d;
175+
char *p1,*p2;
176+
int ret;
177+
struct dirent *dir;
178+
d = opendir("/storage/emulated/0/Download");
179+
if (d)
180+
{
181+
LOGI("Loading feature vectors using *.vec", p1);
182+
while ((dir = readdir(d)) != NULL)
183+
{
184+
p1=strtok(dir->d_name,".");
185+
p2=strtok(NULL,".");
186+
if(p2!=NULL)
187+
{
188+
ret=strcmp(p2,"vec");
189+
if(ret==0)
190+
{
191+
std::string name = std::string(p1);
192+
std::string file = name + ".vec";
193+
matrix<float, 0, 1> face_vector;
194+
dlib::deserialize("/storage/emulated/0/Download/" + file) >> face_vector;
195+
known_faces.insert({name, face_vector});
68196

197+
//printf("%s\t",p1);
198+
//LOGI("item %s", p1);
199+
}
200+
}
69201

202+
}
203+
closedir(d);
204+
}
205+
} else {
206+
return 1; //failed
207+
}
70208

71-
const char *returnValue = "test";
72-
return env->NewStringUTF(returnValue);
209+
return 0;
73210
}

‎visionSamples/FaceTracker/app/src/main/java/com/google/android/gms/samples/vision/face/facetracker/CustomDetector.java

Lines changed: 45 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,11 @@ public class CustomDetector extends Detector<Face> {
2828
public RecognitionInterface recognitionHandler;
2929
public Frame mFrame;
3030
public Bitmap tmpBitmap;
31-
private volatile boolean IsFirst;
31+
3232

3333
public Face faceToRegognize;
34-
public boolean IsBusy;
34+
public volatile boolean IsRecognitionRequested;
35+
public volatile boolean IsBusy;
3536
private Thread mT;
3637

3738
private int faceid;
@@ -40,17 +41,26 @@ public class CustomDetector extends Detector<Face> {
4041
CustomDetector(Detector<Face> delegate)
4142
{
4243
mDelegate = delegate;
43-
IsFirst = true;
4444
}
4545

46-
void startRecognition(int faceId, int _x, int _y, int _w, int _h)
46+
void startRecognition(int faceId, int _x, int _y, int _w, int _h) {
47+
// if (!IsRunning)
48+
// {
49+
faceid = faceId;
50+
x = _x;
51+
y = _y;
52+
w = _w;
53+
h = _h;
54+
// IsRecognitionRequested = true;
55+
// } else{
56+
// IsRecognitionRequested = false;
57+
// }
58+
}
59+
60+
void resetRecognition()
4761
{
48-
IsBusy = true;
49-
faceid = faceId;
50-
x = _x;
51-
y = _y;
52-
w = _w;
53-
h = _h;
62+
//IsRunning = false; //call it from onRecognized
63+
IsBusy = false;
5464
}
5565

5666
public void setHandlerListener(RecognitionInterface listener)
@@ -61,28 +71,31 @@ public void setHandlerListener(RecognitionInterface listener)
6171
//@Override
6272
public SparseArray<Face> detect(Frame frame) {
6373
mFrame = frame;
64-
if (IsBusy) {
65-
if (recognitionHandler != null) {
66-
YuvImage yuvImage = new YuvImage(frame.getGrayscaleImageData().array(), ImageFormat.NV21,
67-
frame.getMetadata().getWidth(), frame.getMetadata().getHeight(), null);
68-
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
69-
yuvImage.compressToJpeg(new Rect(0, 0, frame.getMetadata().getWidth(),
70-
frame.getMetadata().getHeight()), 100, byteArrayOutputStream);
71-
byte[] jpegArray = byteArrayOutputStream.toByteArray();
72-
Bitmap tmpBitmap = BitmapFactory.decodeByteArray(jpegArray, 0, jpegArray.length);
73-
final Bitmap cropped = Bitmap.createBitmap(tmpBitmap, x, y, w, h);
74-
75-
mT = new Thread(new Runnable() {
76-
@Override
77-
public void run() {
78-
String res = test(cropped);
79-
recognitionHandler.onRecognized(res);
80-
setHandlerListener(null);
81-
IsBusy = false; //todo: stop reset
82-
}
83-
});
84-
mT.start();
85-
}
74+
75+
if (!IsBusy && y > 0 && recognitionHandler != null) {
76+
IsBusy = true;
77+
78+
YuvImage yuvImage = new YuvImage(frame.getGrayscaleImageData().array(), ImageFormat.NV21,
79+
frame.getMetadata().getWidth(), frame.getMetadata().getHeight(), null);
80+
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
81+
yuvImage.compressToJpeg(new Rect(0, 0, frame.getMetadata().getWidth(),
82+
frame.getMetadata().getHeight()), 100, byteArrayOutputStream);
83+
byte[] jpegArray = byteArrayOutputStream.toByteArray();
84+
Bitmap tmpBitmap = BitmapFactory.decodeByteArray(jpegArray, 0, jpegArray.length);
85+
final Bitmap cropped = Bitmap.createBitmap(tmpBitmap, x, y, w, h);
86+
87+
mT = new Thread(new Runnable() {
88+
@Override
89+
public void run() {
90+
String res = test(cropped);
91+
recognitionHandler.onRecognized(res);
92+
}
93+
});
94+
mT.start();
95+
96+
} else{
97+
//it is perform recognition now
98+
//IsRunning = false;
8699
}
87100

88101
SparseArray<Face> faces = mDelegate.detect(frame);

0 commit comments

Comments
 (0)