Skip to content

Commit 489f8df

Browse files
vpisarevvpisarev
committed
Merge pull request #1333 from sovrasov:tld_improvement
2 parents 6651fb0 + 0c5a8e1 commit 489f8df

12 files changed

+244
-176
lines changed

‎modules/tracking/src/multiTracker.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ namespace cv
408408
////To fix: Check the paper, probably this cause wrong learning
409409
//
410410
labPatch.isObject = srValue > tld::THETA_NN;
411-
labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < 0.1;
411+
labPatch.shouldBeIntegrated = abs(srValue - tld::THETA_NN) < tld::CLASSIFIER_MARGIN;
412412
patches[k].push_back(labPatch);
413413
//
414414

‎modules/tracking/src/tldDetector.cpp

Lines changed: 54 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
//M*/
4141

4242
#include "tldDetector.hpp"
43+
#include "tracking_utils.hpp"
4344

4445
#include <opencv2/core/utility.hpp>
4546

@@ -64,27 +65,58 @@ namespace cv
6465
return p;
6566
}
6667

68+
double TLDDetector::computeSminus(const Mat_<uchar>& patch) const
69+
{
70+
double sminus = 0.0;
71+
Mat_<uchar> modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
72+
for (int i = 0; i < *negNum; i++)
73+
{
74+
modelSample.data = &(negExp->data[i * 225]);
75+
sminus = std::max(sminus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0));
76+
}
77+
return sminus;
78+
}
79+
6780
// Calculate Relative similarity of the patch (NN-Model)
6881
double TLDDetector::Sr(const Mat_<uchar>& patch) const
6982
{
7083
double splus = 0.0, sminus = 0.0;
71-
Mat_<uchar> modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
84+
Mat_<uchar> modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
7285
for (int i = 0; i < *posNum; i++)
7386
{
7487
modelSample.data = &(posExp->data[i * 225]);
75-
splus = std::max(splus, 0.5 * (NCC(modelSample, patch) + 1.0));
76-
}
77-
for (int i = 0; i < *negNum; i++)
78-
{
79-
modelSample.data = &(negExp->data[i * 225]);
80-
sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0));
88+
splus = std::max(splus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0));
8189
}
90+
sminus = computeSminus(patch);
8291

8392
if (splus + sminus == 0.0)
8493
return 0.0;
8594
return splus / (sminus + splus);
8695
}
8796

97+
std::pair<double, double> TLDDetector::SrAndSc(const Mat_<uchar>& patch) const
98+
{
99+
double splusC = 0.0, sminus = 0.0, splus = 0.0;
100+
Mat_<uchar> modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
101+
int med = tracking_internal::getMedian((*timeStampsPositive));
102+
for (int i = 0; i < *posNum; i++)
103+
{
104+
modelSample.data = &(posExp->data[i * 225]);
105+
double s = 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0);
106+
107+
if ((int)(*timeStampsPositive)[i] <= med)
108+
splusC = std::max(splusC, s);
109+
110+
splus = std::max(splus, s);
111+
}
112+
sminus = computeSminus(patch);
113+
114+
double sr = (splus + sminus == 0.0) ? 0. : splus / (sminus + splus);
115+
double sc = (splusC + sminus == 0.0) ? 0. : splusC / (sminus + splusC);
116+
117+
return std::pair<double, double>(sr, sc);
118+
}
119+
88120
#ifdef HAVE_OPENCL
89121
double TLDDetector::ocl_Sr(const Mat_<uchar>& patch)
90122
{
@@ -167,7 +199,7 @@ namespace cv
167199
for (int id = 0; id < numOfPatches; id++)
168200
{
169201
double spr = 0.0, smr = 0.0, spc = 0.0, smc = 0;
170-
int med = getMedian((*timeStampsPositive));
202+
int med = tracking_internal::getMedian((*timeStampsPositive));
171203
for (int i = 0; i < *posNum; i++)
172204
{
173205
spr = std::max(spr, 0.5 * (posNCC.at<float>(id * 500 + i) + 1.0));
@@ -195,20 +227,16 @@ namespace cv
195227
{
196228
double splus = 0.0, sminus = 0.0;
197229
Mat_<uchar> modelSample(STANDARD_PATCH_SIZE, STANDARD_PATCH_SIZE);
198-
int med = getMedian((*timeStampsPositive));
230+
int med = tracking_internal::getMedian((*timeStampsPositive));
199231
for (int i = 0; i < *posNum; i++)
200232
{
201233
if ((int)(*timeStampsPositive)[i] <= med)
202234
{
203235
modelSample.data = &(posExp->data[i * 225]);
204-
splus = std::max(splus, 0.5 * (NCC(modelSample, patch) + 1.0));
236+
splus = std::max(splus, 0.5 * (tracking_internal::computeNCC(modelSample, patch) + 1.0));
205237
}
206238
}
207-
for (int i = 0; i < *negNum; i++)
208-
{
209-
modelSample.data = &(negExp->data[i * 225]);
210-
sminus = std::max(sminus, 0.5 * (NCC(modelSample, patch) + 1.0));
211-
}
239+
sminus = computeSminus(patch);
212240

213241
if (splus + sminus == 0.0)
214242
return 0.0;
@@ -249,7 +277,7 @@ namespace cv
249277

250278
Mat resNCC = devNCC.getMat(ACCESS_READ);
251279

252-
int med = getMedian((*timeStampsPositive));
280+
int med = tracking_internal::getMedian((*timeStampsPositive));
253281
for (int i = 0; i < *posNum; i++)
254282
if ((int)(*timeStampsPositive)[i] <= med)
255283
splus = std::max(splus, 0.5 * (resNCC.at<float>(i) +1.0));
@@ -316,9 +344,9 @@ namespace cv
316344
resample(detectorF->resized_imgs[detectorF->ensScaleIDs[ind]],
317345
Rect2d(detectorF->ensBuffer[ind], initSizeF),
318346
detectorF->standardPatches[ind]);
319-
320-
detectorF->scValues[ind] = detectorF->Sc (detectorF->standardPatches[ind]);
321-
detectorF->srValues[ind] = detectorF->Sr (detectorF->standardPatches[ind]);
347+
std::pair<double, double> values = detectorF->SrAndSc(detectorF->standardPatches[ind]);
348+
detectorF->scValues[ind] = values.second;
349+
detectorF->srValues[ind] = values.first;
322350
}
323351
}
324352

@@ -412,15 +440,17 @@ namespace cv
412440
LabeledPatch labPatch;
413441
double curScale = pow(SCALE_STEP, ensScaleIDs[i]);
414442
labPatch.rect = Rect2d(ensBuffer[i].x*curScale, ensBuffer[i].y*curScale, initSize.width * curScale, initSize.height * curScale);
443+
labPatch.Sc = scValues[i];
444+
//printf("max sc %f\n", labPatch.Sc);
415445

416446
const double srValue = srValues[i];
417447
const double scValue = scValues[i];
418448

419449
////To fix: Check the paper, probably this cause wrong learning
420450
//
421-
labPatch.isObject = srValue > THETA_NN;
422-
labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < 0.1;
423-
patches.push_back(labPatch);
451+
labPatch.isObject = srValue > THETA_NN;
452+
labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < CLASSIFIER_MARGIN;
453+
patches.push_back(labPatch);
424454
//
425455

426456
if (!labPatch.isObject)
@@ -440,7 +470,7 @@ namespace cv
440470
}
441471
}
442472

443-
if (maxSc < 0)
473+
if (maxSc < 0)
444474
return false;
445475
else
446476
{
@@ -538,7 +568,7 @@ namespace cv
538568
////To fix: Check the paper, probably this cause wrong learning
539569
//
540570
labPatch.isObject = srValue > THETA_NN;
541-
labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < 0.1;
571+
labPatch.shouldBeIntegrated = abs(srValue - THETA_NN) < CLASSIFIER_MARGIN;
542572
patches.push_back(labPatch);
543573
//
544574

‎modules/tracking/src/tldDetector.hpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ namespace cv
5757
const int MEASURES_PER_CLASSIFIER = 13;
5858
const int GRIDSIZE = 15;
5959
const int DOWNSCALE_MODE = cv::INTER_LINEAR;
60-
const double THETA_NN = 0.50;
60+
const double THETA_NN = 0.5;
6161
const double CORE_THRESHOLD = 0.5;
62+
const double CLASSIFIER_MARGIN = 0.1;
6263
const double SCALE_STEP = 1.2;
6364
const double ENSEMBLE_THRESHOLD = 0.5;
6465
const double VARIANCE_THRESHOLD = 0.5;
@@ -77,6 +78,7 @@ namespace cv
7778
void prepareClassifiers(int rowstep);
7879
double Sr(const Mat_<uchar>& patch) const;
7980
double Sc(const Mat_<uchar>& patch) const;
81+
std::pair<double, double> SrAndSc(const Mat_<uchar>& patch) const;
8082
#ifdef HAVE_OPENCL
8183
double ocl_Sr(const Mat_<uchar>& patch);
8284
double ocl_Sc(const Mat_<uchar>& patch);
@@ -101,13 +103,17 @@ namespace cv
101103
{
102104
Rect2d rect;
103105
bool isObject, shouldBeIntegrated;
106+
double Sc;
104107
};
105108
bool detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector<LabeledPatch>& patches, Size initSize);
106109
bool ocl_detect(const Mat& img, const Mat& imgBlurred, Rect2d& res, std::vector<LabeledPatch>& patches, Size initSize);
107110

108111
friend class MyMouseCallbackDEBUG;
109112
static void computeIntegralImages(const Mat& img, Mat_<double>& intImgP, Mat_<double>& intImgP2){ integral(img, intImgP, intImgP2, CV_64F); }
110113
static inline bool patchVariance(Mat_<double>& intImgP, Mat_<double>& intImgP2, double *originalVariance, Point pt, Size size);
114+
115+
protected:
116+
double computeSminus(const Mat_<uchar>& patch) const;
111117
};
112118

113119

‎modules/tracking/src/tldModel.cpp

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,9 @@ namespace cv
8787
//Generate initial positive samples and put them to the model
8888
positiveExamples.reserve(200);
8989

90-
for (int i = 0; i < (int)closest.size(); i++)
90+
for (size_t i = 0; i < closest.size(); i++)
9191
{
92-
for (int j = 0; j < 20; j++)
92+
for (size_t j = 0; j < 20; j++)
9393
{
9494
Point2f center;
9595
Size2f size;
@@ -102,13 +102,15 @@ namespace cv
102102

103103
resample(scaledImg, RotatedRect(center, size, angle), standardPatch);
104104

105-
for (int y = 0; y < standardPatch.rows; y++)
106-
{
107-
for (int x = 0; x < standardPatch.cols; x++)
108-
{
109-
standardPatch(x, y) += (uchar)rng.gaussian(5.0);
110-
}
111-
}
105+
for( int y = 0; y < standardPatch.rows; y++ )
106+
{
107+
uchar* patchRow = standardPatch.ptr(y);
108+
for( int x = 0; x < standardPatch.cols; x++ )
109+
{
110+
int newValue = patchRow[x] + cvRound(rng.gaussian(5.0));
111+
patchRow[x] = saturate_cast<uchar>(newValue);
112+
}
113+
}
112114

113115
#ifdef BLUR_AS_VADIM
114116
GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0);

‎modules/tracking/src/tldTracker.cpp

Lines changed: 37 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141

4242
#include "tldTracker.hpp"
4343

44-
4544
namespace cv
4645
{
4746

@@ -141,38 +140,38 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox)
141140
std::vector<Rect2d> candidates;
142141
std::vector<double> candidatesRes;
143142
bool trackerNeedsReInit = false;
144-
bool DETECT_FLG = false;
145-
for( int i = 0; i < 2; i++ )
143+
bool DETECT_FLG = false;
144+
145+
//run tracker
146+
Rect2d tmpCandid = boundingBox;
147+
if(!data->failedLastTime && trackerProxy->update(image, tmpCandid))
146148
{
147-
Rect2d tmpCandid = boundingBox;
149+
candidates.push_back(tmpCandid);
150+
resample(image_gray, tmpCandid, standardPatch);
151+
candidatesRes.push_back(tldModel->detector->Sc(standardPatch));
152+
}
153+
else
154+
trackerNeedsReInit = true;
148155

149-
if (i == 1)
150-
{
156+
//run detector
157+
tmpCandid = boundingBox;
151158
#ifdef HAVE_OPENCL
152-
if (ocl::haveOpenCL())
153-
DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
154-
else
159+
if (false)//ocl::useOpenCL())
160+
DETECT_FLG = tldModel->detector->ocl_detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
161+
else
155162
#endif
156-
DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
157-
}
158-
if( ( (i == 0) && !data->failedLastTime && trackerProxy->update(image, tmpCandid) ) || ( DETECT_FLG))
159-
{
160-
candidates.push_back(tmpCandid);
161-
if( i == 0 )
162-
resample(image_gray, tmpCandid, standardPatch);
163-
else
164-
resample(imageForDetector, tmpCandid, standardPatch);
165-
candidatesRes.push_back(tldModel->detector->Sc(standardPatch));
166-
}
167-
else
168-
{
169-
if( i == 0 )
170-
trackerNeedsReInit = true;
171-
}
163+
DETECT_FLG = tldModel->detector->detect(imageForDetector, image_blurred, tmpCandid, detectorResults, tldModel->getMinSize());
164+
165+
if(DETECT_FLG)
166+
{
167+
candidates.push_back(tmpCandid);
168+
resample(imageForDetector, tmpCandid, standardPatch);
169+
candidatesRes.push_back(tldModel->detector->Sc(standardPatch));
172170
}
171+
173172
std::vector<double>::iterator it = std::max_element(candidatesRes.begin(), candidatesRes.end());
174173

175-
if( it == candidatesRes.end() )
174+
if( it == candidatesRes.end() ) //candidates are empty
176175
{
177176
data->confident = false;
178177
data->failedLastTime = true;
@@ -221,7 +220,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox)
221220
tldModel->integrateRelabeled(imageForDetector, image_blurred, detectorResults);
222221
pExpert.additionalExamples(examplesForModel, examplesForEnsemble);
223222
#ifdef HAVE_OPENCL
224-
if (ocl::haveOpenCL())
223+
if (false)//ocl::useOpenCL())
225224
tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, true);
226225
else
227226
#endif
@@ -230,7 +229,7 @@ bool TrackerTLDImpl::updateImpl(const Mat& image, Rect2d& boundingBox)
230229
nExpert.additionalExamples(examplesForModel, examplesForEnsemble);
231230

232231
#ifdef HAVE_OPENCL
233-
if (ocl::haveOpenCL())
232+
if (false)//ocl::useOpenCL())
234233
tldModel->ocl_integrateAdditional(examplesForModel, examplesForEnsemble, false);
235234
else
236235
#endif
@@ -257,12 +256,13 @@ int TrackerTLDImpl::Pexpert::additionalExamples(std::vector<Mat_<uchar> >& examp
257256

258257
double scale = scaleAndBlur(img_, cvRound(log(1.0 * resultBox_.width / (initSize_.width)) / log(SCALE_STEP)),
259258
scaledImg, blurredImg, GaussBlurKernelSize, SCALE_STEP);
259+
260260
TLDDetector::generateScanGrid(img_.rows, img_.cols, initSize_, scanGrid);
261261
getClosestN(scanGrid, Rect2d(resultBox_.x / scale, resultBox_.y / scale, resultBox_.width / scale, resultBox_.height / scale), 10, closest);
262262

263-
for( int i = 0; i < (int)closest.size(); i++ )
263+
for( size_t i = 0; i < closest.size(); i++ )
264264
{
265-
for( int j = 0; j < 10; j++ )
265+
for( size_t j = 0; j < 10; j++ )
266266
{
267267
Point2f center;
268268
Size2f size;
@@ -273,21 +273,24 @@ int TrackerTLDImpl::Pexpert::additionalExamples(std::vector<Mat_<uchar> >& examp
273273
size.height = (float)(closest[i].height * rng.uniform((double)0.99, (double)1.01));
274274
float angle = (float)rng.uniform(-5.0, 5.0);
275275

276+
resample(scaledImg, RotatedRect(center, size, angle), standardPatch);
276277
for( int y = 0; y < standardPatch.rows; y++ )
277278
{
279+
uchar* patchRow = standardPatch.ptr(y);
278280
for( int x = 0; x < standardPatch.cols; x++ )
279281
{
280-
standardPatch(x, y) += (uchar)rng.gaussian(5.0);
282+
int newValue = patchRow[x] + cvRound(rng.gaussian(5.0));
283+
patchRow[x] = saturate_cast<uchar>(newValue);
281284
}
282285
}
283-
#ifdef BLUR_AS_VADIM
286+
examplesForModel.push_back(standardPatch);
287+
288+
#if defined BLUR_AS_VADIM
284289
GaussianBlur(standardPatch, blurredPatch, GaussBlurKernelSize, 0.0);
285290
resize(blurredPatch, blurredPatch, initSize_);
286291
#else
287292
resample(blurredImg, RotatedRect(center, size, angle), blurredPatch);
288293
#endif
289-
resample(scaledImg, RotatedRect(center, size, angle), standardPatch);
290-
examplesForModel.push_back(standardPatch);
291294
examplesForEnsemble.push_back(blurredPatch);
292295
}
293296
}

‎modules/tracking/src/tldTracker.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ class TrackerProxyImpl : public TrackerProxy
112112
};
113113

114114

115-
#define BLUR_AS_VADIM
115+
#undef BLUR_AS_VADIM
116116
#undef CLOSED_LOOP
117117

118118
class TrackerTLDImpl : public TrackerTLD

0 commit comments

Comments
 (0)