diff --git a/.clang_complete b/.clang_complete new file mode 100644 index 0000000..bb9ed4c --- /dev/null +++ b/.clang_complete @@ -0,0 +1,4 @@ +-DBarcode3rdParty_EXPORTS +-DBarcodeLibrary_EXPORTS +-I/home/kurohi/Progs/BarcodeDetectionHough/include +-I/home/kurohi/Progs/BarcodeDetectionHough/utils-3rdparty diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..a41bf40 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required (VERSION 2.9.0) +project (BarcodeDetectionHough) + +find_package(OpenCV 3.4 REQUIRED) + +add_definitions(-std=c++11) + +find_package(Boost COMPONENTS filesystem system chrono program_options REQUIRED) +include_directories (${Boost_INCLUDE_DIRS}) +link_directories(${Boost_LIBRARY_DIRS}) +add_definitions(${Boost_DEFINITIONS}) + +include_directories("${CMAKE_SOURCE_DIR}/include") +include_directories("${CMAKE_SOURCE_DIR}/utils-3rdparty") + +file(GLOB SOURCES "${CMAKE_SOURCE_DIR}/src/*.cpp") +file(GLOB SOURCES_3RD_PARTY "${CMAKE_SOURCE_DIR}/utils-3rdparty/*.cpp") +add_library(BarcodeLibrary SHARED ${SOURCES}) +add_library(Barcode3rdParty SHARED ${SOURCES_3RD_PARTY}) + +add_executable (barcodedetection main.cpp) +add_executable (barcodeonmovie main_movie.cpp) + +target_link_libraries(barcodedetection BarcodeLibrary Barcode3rdParty ${OpenCV_LIBS} ${Boost_LIBRARIES}) +target_link_libraries(barcodeonmovie BarcodeLibrary Barcode3rdParty ${OpenCV_LIBS} ${Boost_LIBRARIES}) diff --git a/README b/README index eb6dd22..416809c 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ Requirements ---- -° Any C++08 compiler (actually used g++ 4.8.1) -° OpenCv 2.4.* with Qt 4 support enabled -° Boost 1.53 or more recent builds +° Any C++08 compiler (actually used g++ 9.1.0) +° OpenCv 3.4.* with Qt 4 support enabled +° Boost 1.65 or more recent builds @@ -24,6 +24,7 @@ In order to facilitate it, the shared library linked by the makefile are: ° opencv_highgui ° opencv_ml ° opencv_imgproc +° opencv_imgcodecs If you encounter any problem check if the required libraries are installed in your system. diff --git a/include/ArtelabDataset.hpp b/include/ArtelabDataset.hpp index 848f5cf..ab47040 100644 --- a/include/ArtelabDataset.hpp +++ b/include/ArtelabDataset.hpp @@ -3,16 +3,17 @@ #define BARCODEDATASET_HPP #include +#include #include "DirectoryInfo.hpp" namespace artelab { - class ArtelabDataset + class ArtelabDataset { public: - typedef struct + typedef struct { FileInfo original; FileInfo detection_gt; @@ -37,4 +38,3 @@ namespace artelab } #endif /* BARCODEDATASET_HPP */ - diff --git a/include/HoughTransform.hpp b/include/HoughTransform.hpp index c726dba..46c4c27 100644 --- a/include/HoughTransform.hpp +++ b/include/HoughTransform.hpp @@ -3,11 +3,12 @@ #define HOUGHTRANSFORM_HPP #include +#include namespace artelab { - class HoughTransform + class HoughTransform { public: @@ -56,4 +57,3 @@ namespace artelab } #endif /* HOUGHTRANSFORM_HPP */ - diff --git a/include/ImageProcessor.hpp b/include/ImageProcessor.hpp index 0a5a4f3..3e6ce43 100644 --- a/include/ImageProcessor.hpp +++ b/include/ImageProcessor.hpp @@ -10,30 +10,33 @@ namespace artelab { - class ImageProcessor - { - public: + class ImageProcessor + { + public: - ImageProcessor(std::string mlp_file, cv::Size win_size, std::string outdir="", bool quiet=false, bool show=false); - virtual ~ImageProcessor(); + ImageProcessor(std::string mlp_file, cv::Size win_size, std::string outdir="", bool quiet=false, bool show=false); + virtual ~ImageProcessor(); - ImageProcessor& show(bool b=true); - ImageProcessor& set_output(std::string outdir); + ImageProcessor& show(bool b=true); + ImageProcessor& set_output(std::string outdir); - results process(std::string imagename, ArtelabDataset::barcode_image barcode); + results process(std::string imagename, ArtelabDataset::barcode_image barcode); + cv::Mat processSimple(cv::Mat img, double& angle); + void drawRectangles(cv::Mat& img_bb, cv::Rect rect, double angle); + std::vector getRectangles(cv::Mat img_hist_projection, double angle); - private: - MLP _mlp; - cv::Size _winsize; - DirectoryInfo _output; - bool _show; - bool _quiet; - - void show_image(std::string name, cv::Mat img); - }; + private: + + MLP _mlp; + cv::Size _winsize; + DirectoryInfo _output; + bool _show; + bool _quiet; + + void show_image(std::string name, cv::Mat img); + }; } - -#endif /* IMAGEPROCESSOR_HPP */ +#endif /* IMAGEPROCESSOR_HPP */ diff --git a/include/MLP.hpp b/include/MLP.hpp index fa162d8..fce14c6 100644 --- a/include/MLP.hpp +++ b/include/MLP.hpp @@ -8,7 +8,7 @@ namespace artelab { - class MLP + class MLP { public: @@ -23,10 +23,9 @@ namespace artelab void predict(const cv::Mat& samples, cv::Mat& outPredictions); private: - cv::NeuralNet_MLP _model; + cv::Ptr _model; }; } #endif /* MLP_H */ - diff --git a/include/accuracy.hpp b/include/accuracy.hpp index ccfd45d..7213bd8 100644 --- a/include/accuracy.hpp +++ b/include/accuracy.hpp @@ -8,9 +8,9 @@ namespace artelab { - typedef struct - { - double jaccard; + typedef struct + { + double jaccard; cv::Mat jaccard_hist; double time; } results; @@ -22,4 +22,3 @@ namespace artelab } #endif /* ACCURACY_HPP */ - diff --git a/include/detection.hpp b/include/detection.hpp index 4aac3cb..25c42d8 100644 --- a/include/detection.hpp +++ b/include/detection.hpp @@ -5,11 +5,10 @@ #include namespace artelab -{ +{ std::vector object_rectangles(cv::Mat feature_image, int thresh=70); } #endif /* DETECTION_HPP */ - diff --git a/include/mlp_threshold.hpp b/include/mlp_threshold.hpp index f253880..deed3cf 100644 --- a/include/mlp_threshold.hpp +++ b/include/mlp_threshold.hpp @@ -9,10 +9,9 @@ namespace artelab { - + cv::Mat threshold_mlp(MLP& nnetwork, cv::Size win_size, HoughTransform& hough); } #endif /* MAKE_PATTERNS_HPP */ - diff --git a/main.cpp b/main.cpp index 0fd5c6b..15fd6d1 100644 --- a/main.cpp +++ b/main.cpp @@ -31,7 +31,7 @@ vector imagenames_to_process(FileInfo file) vector names; std::ifstream infile; infile.open(file.fullName().c_str()); - + while(!infile.eof()) { string line; @@ -43,12 +43,13 @@ vector imagenames_to_process(FileInfo file) return names; } -int main(int argc, char** argv) -{ +int main(int argc, char** argv) +{ po::options_description desc("Usage"); desc.add_options() ("help", "print help") ("dataset,d", po::value()->default_value(datasetdir), "dataset directory") + ("netfile,n", po::value()->default_value(netfile), "trained network file") ("output,o", po::value()->default_value(outdir), "if specified, intermediate images are saved there") ("whitelist,w", po::value()->default_value(whitelist), "list of image names to process") ("show,s", "show intermediate images") @@ -57,55 +58,56 @@ int main(int argc, char** argv) po::variables_map vm; po::store(po::parse_command_line(argc, argv, desc), vm); - po::notify(vm); + po::notify(vm); if (vm.count("help")) { cout << desc << endl; return EXIT_SUCCESS; } - + datasetdir = vm["dataset"].as(); outdir = vm["output"].as(); whitelist = vm["whitelist"].as(); display = vm.count("show") > 0; quiet = vm.count("quiet") > 0; - + netfile = vm["netfile"].as(); + DirectoryInfo dataset_dir(datasetdir); - + ArtelabDataset dataset(dataset_dir); dataset.load_dataset(); if(!quiet) cout << "Dataset Loaded: " << dataset.count() << " Images" << endl; - + cv::Size win_size(61, 3); - + std::map images = dataset.get_barcodes(); std::map::iterator it; - + vector white_list; - if(whitelist != "") + if(whitelist != "") white_list = imagenames_to_process(FileInfo(whitelist)); - + double accuracy = 0, time = 0; int count = 0; cv::Mat jaccard_hist = cv::Mat::zeros(10, 1, CV_32F); - + ImageProcessor pr(netfile, win_size, outdir, quiet, display); - + for(it = images.begin(); it != images.end(); it++) { if(white_list.size() > 0 && std::find(white_list.begin(), white_list.end(), it->first) == white_list.end()) { continue; } - + results res = pr.process(it->first, it->second); - + accuracy += res.jaccard; time += res.time; jaccard_hist += res.jaccard_hist; count++; - + if(display) { char c; @@ -114,15 +116,14 @@ int main(int argc, char** argv) c = cv::waitKey(); } while (c != 32); } - + } - - cout << endl + + cout << endl << "Number of images: " << count << endl << "TOTAL ACCURACY (jaccard): " << accuracy/count << endl << "TOTAL ACCURACY BY THRESHOLD (jaccard): " << jaccard_hist/count << endl << "Average time (sec): " << time/count << endl; - + return EXIT_SUCCESS; } - diff --git a/main_movie.cpp b/main_movie.cpp new file mode 100644 index 0000000..195630d --- /dev/null +++ b/main_movie.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include +#include + +#include "ImageProcessor.hpp" + +#define foreach BOOST_FOREACH + +using namespace artelab; +using std::cout; +using std::endl; +using std::flush; +using std::string; +using std::vector; +namespace po = boost::program_options; + +string netfile = "net61x3.net"; +string outdir = "output"; +string video_stream_file = "0"; + +int main(int argc, char **argv) { + po::options_description desc("Usage"); + desc.add_options()("help", "print help")( + "netfile,n", po::value()->default_value(netfile), + "trained network file")( + "video_stream,v", po::value()->default_value(video_stream_file), + "path to the video file or camera index")( + "output,o", po::value()->default_value(outdir), + "if specified, intermediate images are saved there"); + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); + + if (vm.count("help")) { + cout << desc << endl; + return EXIT_SUCCESS; + } + + netfile = vm["netfile"].as(); + outdir = vm["output"].as(); + video_stream_file = vm["video_stream"].as(); + + cv::Size win_size(61, 3); + + ImageProcessor pr(netfile, win_size, outdir); + cv::VideoCapture video_stream = cv::VideoCapture(video_stream_file); + cv::Mat frame, processed_frame, img_hist_projection; + double angle; + std::vector rects; + + char c = '0'; + cv::namedWindow("detected"); + while (c != 'q') { + if (!video_stream.read(frame)) { + break; + } + img_hist_projection = pr.processSimple(frame, angle); + frame.copyTo(processed_frame); + rects = pr.getRectangles(img_hist_projection, angle); + foreach (cv::Rect r, rects) { + pr.drawRectangles(processed_frame, r, angle); + } + cv::imshow("detected",processed_frame); + c = cv::waitKey(10); + } + + return EXIT_SUCCESS; +} diff --git a/nbproject/Makefile-Debug.mk b/nbproject/Makefile-Debug.mk index f1cdc3d..0e0e4da 100755 --- a/nbproject/Makefile-Debug.mk +++ b/nbproject/Makefile-Debug.mk @@ -38,7 +38,6 @@ OBJECTFILES= \ ${OBJECTDIR}/main.o \ ${OBJECTDIR}/src/ArtelabDataset.o \ ${OBJECTDIR}/src/HoughTransform.o \ - ${OBJECTDIR}/src/ImageProcessor.o \ ${OBJECTDIR}/src/MLP.o \ ${OBJECTDIR}/src/accuracy.o \ ${OBJECTDIR}/src/detection.o \ @@ -48,7 +47,8 @@ OBJECTFILES= \ ${OBJECTDIR}/utils-3rdparty/DirectoryInfo.o \ ${OBJECTDIR}/utils-3rdparty/FileInfo.o \ ${OBJECTDIR}/utils-3rdparty/TimeCounter.o \ - ${OBJECTDIR}/utils-3rdparty/utils.o + ${OBJECTDIR}/utils-3rdparty/utils.o \ + ${OBJECTDIR}/src/ImageProcessor.o # C Compiler Flags @@ -65,7 +65,7 @@ FFLAGS= ASFLAGS= # Link Libraries and Options -LDLIBSOPTIONS=-lboost_filesystem -lboost_system -lboost_chrono -lboost_program_options -lopencv_core -lopencv_highgui -lopencv_ml -lopencv_imgproc +LDLIBSOPTIONS=-lboost_filesystem -lboost_system -lboost_chrono -lboost_program_options -lopencv_core -lopencv_highgui -lopencv_ml -lopencv_imgproc -lopencv_imgcodecs # Build Targets .build-conf: ${BUILD_SUBPROJECTS} diff --git a/nbproject/Makefile-Release.mk b/nbproject/Makefile-Release.mk index e632778..25d3bc6 100755 --- a/nbproject/Makefile-Release.mk +++ b/nbproject/Makefile-Release.mk @@ -38,7 +38,6 @@ OBJECTFILES= \ ${OBJECTDIR}/main.o \ ${OBJECTDIR}/src/ArtelabDataset.o \ ${OBJECTDIR}/src/HoughTransform.o \ - ${OBJECTDIR}/src/ImageProcessor.o \ ${OBJECTDIR}/src/MLP.o \ ${OBJECTDIR}/src/accuracy.o \ ${OBJECTDIR}/src/detection.o \ @@ -48,7 +47,8 @@ OBJECTFILES= \ ${OBJECTDIR}/utils-3rdparty/DirectoryInfo.o \ ${OBJECTDIR}/utils-3rdparty/FileInfo.o \ ${OBJECTDIR}/utils-3rdparty/TimeCounter.o \ - ${OBJECTDIR}/utils-3rdparty/utils.o + ${OBJECTDIR}/utils-3rdparty/utils.o \ + ${OBJECTDIR}/src/ImageProcessor.o # C Compiler Flags diff --git a/src/ArtelabDataset.cpp b/src/ArtelabDataset.cpp index bad1fcb..de5e7ca 100644 --- a/src/ArtelabDataset.cpp +++ b/src/ArtelabDataset.cpp @@ -11,13 +11,13 @@ using std::vector; namespace artelab { - - ArtelabDataset::ArtelabDataset(DirectoryInfo base) + + ArtelabDataset::ArtelabDataset(DirectoryInfo base) { _base_dir = base; } - ArtelabDataset::ArtelabDataset(const ArtelabDataset& orig) + ArtelabDataset::ArtelabDataset(const ArtelabDataset& orig) { *this = orig; } @@ -71,4 +71,4 @@ namespace artelab return _data; } -} \ No newline at end of file +} diff --git a/src/HoughTransform.cpp b/src/HoughTransform.cpp index 922d50a..28c5f4a 100755 --- a/src/HoughTransform.cpp +++ b/src/HoughTransform.cpp @@ -39,7 +39,7 @@ namespace artelab HoughTransform::HoughTransform() { init(); - } + } HoughTransform::HoughTransform(cv::Mat image, bool keep_loc) { @@ -53,7 +53,7 @@ namespace artelab delete [] cos_cache; } - HoughTransform::HoughTransform(const HoughTransform& orig) + HoughTransform::HoughTransform(const HoughTransform& orig) { *this = orig; } @@ -98,7 +98,7 @@ namespace artelab int hough_height = hough_space.cols / 2; r += hough_height; - if (r < 0 || r >= hough_height*2) + if (r < 0 || r >= hough_height*2) continue; hough_space.at(t, r)++; @@ -162,4 +162,4 @@ namespace artelab return out_image; } -} \ No newline at end of file +} diff --git a/src/ImageProcessor.cpp b/src/ImageProcessor.cpp index 600935d..d5e97a2 100644 --- a/src/ImageProcessor.cpp +++ b/src/ImageProcessor.cpp @@ -17,254 +17,271 @@ namespace artelab { - ImageProcessor::ImageProcessor(std::string mlp_file, cv::Size win_size, std::string outdir, bool quiet, bool show) : - _show(show), - _quiet(quiet), - _winsize(win_size) - { - _mlp.load(mlp_file); - _output = DirectoryInfo(outdir); - } - - ImageProcessor::~ImageProcessor() - { } - - ImageProcessor& ImageProcessor::show(bool b) - { - _show = b; - return *this; - } - - ImageProcessor& ImageProcessor::set_output(std::string outdir) - { - _output = DirectoryInfo(outdir); - return *this; - } - - - void ImageProcessor::show_image(std::string name, cv::Mat img) - { - if(_show) - { - cv::namedWindow(name); - cv::imshow(name, img); - } - } - - void draw_lines_at_angle(double angle, std::vector lines, cv::Mat& image, int tolerance=2) - { - cv::Scalar color = image.channels() == 1? cv::Scalar(255) : cv::Scalar(0,0,255); - - foreach(cv::Vec4i l, lines) - { - double delta = double(l[3] - l[1]) / double(l[2] - l[0]); - double act_angle = atan(delta)*180/CV_PI; - act_angle += act_angle < 0? 180 : 0; - double diff = fabs(act_angle - angle); - if(diff < tolerance) - { - int thick = int(diff) > 3? 1 : 4-int(diff); - cv::line(image, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), color, thick, 8); - } - } - } - - void smooth_histogram(cv::Mat& hist, int kernel_size, int iterations) - { - CV_Assert(hist.cols == 1); - - int hist_type = hist.type(); - bool cvt_back = hist_type != CV_32F; - if(cvt_back) hist.convertTo(hist, CV_32F); - - cv::Mat kernel = cv::Mat::ones(kernel_size, 1, CV_32F); - kernel = kernel / float(kernel_size); - - for(int i=0; i < iterations; i++) - { - cv::filter2D(hist, hist, -1, kernel); - } - - if(cvt_back) hist.convertTo(hist, hist_type); - } - - void histograms_from_hough_lines(cv::Mat feature, cv::Mat& row_hist, cv::Mat& col_hist, bool smooth_and_thresh=true) - { - row_hist = get_histogram(feature, HIST_ROW, CV_32F); - col_hist = get_histogram(feature, HIST_COL, CV_32F); - - if(smooth_and_thresh) - { - smooth_histogram(row_hist, 5, 200); - smooth_histogram(col_hist, 5, 200); - - double row_hist_mean = cv::sum(row_hist)[0] / double(row_hist.rows); - double col_hist_mean = cv::sum(col_hist)[0] / double(col_hist.rows); - cv::threshold(row_hist, row_hist, row_hist_mean, 0, cv::THRESH_TOZERO); - cv::threshold(col_hist, col_hist, col_hist_mean, 0, cv::THRESH_TOZERO); - } - } - - cv::Mat project_histograms(cv::Mat row_hist, cv::Mat col_hist) - { - CV_Assert(row_hist.type() == CV_32F); - CV_Assert(col_hist.type() == CV_32F); - - cv::Mat out = row_hist * col_hist.t(); - - double min, max; - cv::minMaxLoc(out, &min, &max); - out = (out - min) / max * 255; - out.convertTo(out, CV_8U); - - return out; - } - - results ImageProcessor::process(std::string name, ArtelabDataset::barcode_image bcimage) - { - cv::Mat img_orig = cv::imread(bcimage.original.fullName(), CV_LOAD_IMAGE_COLOR); - cv::Mat img_truth = cv::imread(bcimage.detection_gt.fullName(), CV_LOAD_IMAGE_GRAYSCALE); - - TimeCounter tc; - tc.start(); - - // Apply canny - cv::Mat img_canny; - cv::GaussianBlur(img_orig, img_canny, cv::Size(17,17), 2); - cv::Canny(img_canny, img_canny, 60, 100, 3); - - // Get hough transform - HoughTransform hough(img_canny); - - // Threshold with MLP - cv::Mat img_neural = threshold_mlp(_mlp, _winsize, hough); - - // find angle - double angle = max_angle_hist(get_histogram(img_neural, HIST_ROW)); - - // lines from canny using probabilistc hough - const int tolerance = 3; - std::vector lines; - cv::HoughLinesP(img_canny, lines, 1, CV_PI/180, 50, 20, 1); - cv::Mat line_image; - if(_show) - { - // impress lines of given angle on the canny image - cv::cvtColor(img_canny, line_image, CV_GRAY2BGR); - draw_lines_at_angle((int(angle+0.5) + 90) % 180, lines, line_image, tolerance); - } - - // obtain a feature image with only the lines. It is rectified. - cv::Mat feature_image; - feature_image = cv::Mat::zeros(img_canny.size(), CV_8U); - draw_lines_at_angle((int(angle+0.5) + 90) % 180, lines, feature_image, tolerance); - feature_image = rotate_image(feature_image, angle); - - //Histograms for detection - cv::Mat row_hist, col_hist,feature_with_hist_smooth; - histograms_from_hough_lines(feature_image, row_hist, col_hist); - feature_with_hist_smooth = draw_histogram_on_image(row_hist, feature_image, cv::Scalar(0,0,255), HIST_ROW); - feature_with_hist_smooth = draw_histogram_on_image(col_hist, feature_with_hist_smooth, cv::Scalar(0,255,0), HIST_COL); - - // project histograms - cv::Mat img_hist_projection = project_histograms(row_hist, col_hist); - - // crop image with projected mask and threshold - cv::Mat bb_mask; - bb_mask = artelab::rotate_image(img_hist_projection, -angle); - double min, max; - cv::minMaxLoc(bb_mask, &min, &max); - cv::threshold(bb_mask, bb_mask, int(0.3*max), 1, cv::THRESH_BINARY); - cv::Mat img_cropped; - if(_show) img_orig.copyTo(img_cropped, bb_mask); - - // draw bounding boxes - std::vector rects = object_rectangles(img_hist_projection, int(0.3*max)); - cv::Mat img_bb; - img_orig.copyTo(img_bb); - cv::Mat img_detection_mask = cv::Mat::zeros(img_orig.size(), CV_8U); - foreach(cv::Rect r, rects) - { - // BB for impression on original image - cv::Mat bb = cv::Mat::zeros(img_bb.size(), CV_8U); - cv::rectangle(bb, r, cv::Scalar(255), 2); - bb = artelab::rotate_image(bb, -angle); - img_bb.setTo(cv::Scalar(0,0,255), bb); - - // BB for measuring accuracy - cv::Mat bb_fill = cv::Mat::zeros(img_orig.size(), CV_8U); - cv::rectangle(bb_fill, r, cv::Scalar(255), -1); - bb_fill = artelab::rotate_image(bb_fill, -angle); - img_detection_mask.setTo(cv::Scalar(255), bb_fill); - } - - tc.stop(); - - // Measuring accuracy - results res = measure_results(img_detection_mask, img_truth, tc); - - // verbose output - if(!_quiet) - { - std::cout << std::setw(15) << std::left << name; - std::cout << std::setw(11) << std::left << ("Angle " + tostring(angle)); - std::cout << std::setw(20) << std::left << ("Accuracy: " + tostring(res.jaccard)); - std::cout << std::setw(6) << std::left << ("Time: " + tostring(res.time)) << std::endl; - } - - // show results - show_image("Prob Hough", line_image); - show_image("histograms smooth", feature_with_hist_smooth); - show_image("histograms projection", img_hist_projection); - show_image("Cropped", img_cropped); - show_image("Boundig Boxes", img_bb); - show_image("Original", img_orig); - show_image("Feature", feature_image); - show_image("Canny", img_canny); - - // saving intermediate images - if(_output.fullPath() != "") - { - std::ostringstream ss; - - // Original - ss << name << "_1_original.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_orig); - - // Hough accumulator - ss.str(""); ss.clear(); ss << name << "_2_accumulator.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), hough.get_hough_image()); - - // Hough MLP threshold - img_neural = draw_histogram_on_image(get_histogram(img_neural, HIST_ROW, CV_32F), img_neural, cv::Scalar(0,0,255), HIST_ROW); - ss.str(""); ss.clear();; ss << name << "_3_thresh_mlp.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_neural); - - // Feature with histograms not processed - histograms_from_hough_lines(feature_image, row_hist, col_hist, false); - cv::Mat feature_with_hist = draw_histogram_on_image(row_hist, feature_image, cv::Scalar(0,0,255), HIST_ROW); - feature_with_hist = draw_histogram_on_image(col_hist, feature_with_hist, cv::Scalar(0,255,0), HIST_COL); - ss.str(""); ss.clear();; ss << name << "_4_hist.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), feature_with_hist); - - // Feature with processed histograms - ss.str(""); ss.clear();; ss << name << "_5_hist_smooth_thresh.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), feature_with_hist_smooth); - - // Histogram projection - ss.str(""); ss.clear();; ss << name << "_6_hist_projection.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_hist_projection); - - // Bounding boxes - ss.str(""); ss.clear();; ss << name << "_7_boxes.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_bb); - - // Canny - ss.str(""); ss.clear(); ss << name << "_canny.png"; - cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_canny); - } - - return res; - } - -} \ No newline at end of file + ImageProcessor::ImageProcessor(std::string mlp_file, cv::Size win_size, std::string outdir, bool quiet, bool show) : + _show(show), + _quiet(quiet), + _winsize(win_size) + { + _mlp.load(mlp_file); + _output = DirectoryInfo(outdir); + } + + ImageProcessor::~ImageProcessor() + { } + + ImageProcessor& ImageProcessor::show(bool b) + { + _show = b; + return *this; + } + + ImageProcessor& ImageProcessor::set_output(std::string outdir) + { + _output = DirectoryInfo(outdir); + return *this; + } + + + void ImageProcessor::show_image(std::string name, cv::Mat img) + { + if(_show) + { + cv::namedWindow(name); + cv::imshow(name, img); + } + } + + void draw_lines_at_angle(double angle, std::vector lines, cv::Mat& image, int tolerance=2) + { + cv::Scalar color = image.channels() == 1? cv::Scalar(255) : cv::Scalar(0,0,255); + + foreach(cv::Vec4i l, lines) + { + double delta = double(l[3] - l[1]) / double(l[2] - l[0]); + double act_angle = atan(delta)*180/CV_PI; + act_angle += act_angle < 0? 180 : 0; + double diff = fabs(act_angle - angle); + if(diff < tolerance) + { + int thick = int(diff) > 3? 1 : 4-int(diff); + cv::line(image, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), color, thick, 8); + } + } + } + + void smooth_histogram(cv::Mat& hist, int kernel_size, int iterations) + { + CV_Assert(hist.cols == 1); + + int hist_type = hist.type(); + bool cvt_back = hist_type != CV_32F; + if(cvt_back) hist.convertTo(hist, CV_32F); + + cv::Mat kernel = cv::Mat::ones(kernel_size, 1, CV_32F); + kernel = kernel / float(kernel_size); + + for(int i=0; i < iterations; i++) + { + cv::filter2D(hist, hist, -1, kernel); + } + + if(cvt_back) hist.convertTo(hist, hist_type); + } + + void histograms_from_hough_lines(cv::Mat feature, cv::Mat& row_hist, cv::Mat& col_hist, bool smooth_and_thresh=true) + { + row_hist = get_histogram(feature, HIST_ROW, CV_32F); + col_hist = get_histogram(feature, HIST_COL, CV_32F); + + if(smooth_and_thresh) + { + smooth_histogram(row_hist, 5, 200); + smooth_histogram(col_hist, 5, 200); + + double row_hist_mean = cv::sum(row_hist)[0] / double(row_hist.rows); + double col_hist_mean = cv::sum(col_hist)[0] / double(col_hist.rows); + cv::threshold(row_hist, row_hist, row_hist_mean, 0, cv::THRESH_TOZERO); + cv::threshold(col_hist, col_hist, col_hist_mean, 0, cv::THRESH_TOZERO); + } + } + + cv::Mat project_histograms(cv::Mat row_hist, cv::Mat col_hist) + { + CV_Assert(row_hist.type() == CV_32F); + CV_Assert(col_hist.type() == CV_32F); + + cv::Mat out = row_hist * col_hist.t(); + + double min, max; + cv::minMaxLoc(out, &min, &max); + out = (out - min) / max * 255; + out.convertTo(out, CV_8U); + + return out; + } + + cv::Mat ImageProcessor::processSimple(cv::Mat img, double& angle){ + + // Apply canny + cv::Mat img_canny; + cv::GaussianBlur(img, img_canny, cv::Size(17,17), 2); + cv::cvtColor(img_canny, img_canny, cv::COLOR_RGB2GRAY); + cv::Canny(img_canny, img_canny, 60, 100, 3); + + // Get hough transform + HoughTransform hough(img_canny); + + // Threshold with MLP + cv::Mat img_neural = threshold_mlp(_mlp, _winsize, hough); + + // find angle + angle = max_angle_hist(get_histogram(img_neural, HIST_ROW)); + + // lines from canny using probabilistc hough + const int tolerance = 3; + std::vector lines; + cv::HoughLinesP(img_canny, lines, 1, CV_PI/180, 50, 20, 1); + cv::Mat line_image; + if(_show){ + // impress lines of given angle on the canny image + cv::cvtColor(img_canny, line_image, CV_GRAY2BGR); + draw_lines_at_angle((int(angle+0.5) + 90) % 180, lines, line_image, tolerance); + } + + // obtain a feature image with only the lines. It is rectified. + cv::Mat feature_image; + feature_image = cv::Mat::zeros(img_canny.size(), CV_8U); + draw_lines_at_angle((int(angle+0.5) + 90) % 180, lines, feature_image, tolerance); + feature_image = rotate_image(feature_image, angle); + + //Histograms for detection + cv::Mat row_hist, col_hist,feature_with_hist_smooth; + histograms_from_hough_lines(feature_image, row_hist, col_hist); + feature_with_hist_smooth = draw_histogram_on_image(row_hist, feature_image, cv::Scalar(0,0,255), HIST_ROW); + feature_with_hist_smooth = draw_histogram_on_image(col_hist, feature_with_hist_smooth, cv::Scalar(0,255,0), HIST_COL); + + // project histograms + cv::Mat img_hist_projection = project_histograms(row_hist, col_hist); + return img_hist_projection; + } + + //TODO: Use cv::RotatedRect instead + void ImageProcessor::drawRectangles(cv::Mat& img_bb, cv::Rect rect, double angle){ + // BB for impression on original image + cv::Mat bb = cv::Mat::zeros(img_bb.size(), CV_8U); + cv::rectangle(bb, rect, cv::Scalar(255), 2); + bb = artelab::rotate_image(bb, -angle); + img_bb.setTo(cv::Scalar(0,0,255), bb); + } + + std::vector ImageProcessor::getRectangles(cv::Mat img_hist_projection, double angle){ + // crop image with projected mask and threshold + cv::Mat bb_mask; + bb_mask = artelab::rotate_image(img_hist_projection, -angle); + double min, max; + cv::minMaxLoc(bb_mask, &min, &max); + cv::threshold(bb_mask, bb_mask, int(0.3*max), 1, cv::THRESH_BINARY); + cv::Mat img_cropped; + + // draw bounding boxes + std::vector rects = object_rectangles(img_hist_projection, int(0.3*max)); + return rects; + } + + results ImageProcessor::process(std::string name, ArtelabDataset::barcode_image bcimage) + { + cv::Mat img_orig = cv::imread(bcimage.original.fullName(), CV_LOAD_IMAGE_COLOR); + cv::Mat img_truth = cv::imread(bcimage.detection_gt.fullName(), CV_LOAD_IMAGE_GRAYSCALE); + + TimeCounter tc; + tc.start(); + + double angle; + cv::Mat img_hist_projection = processSimple(img_orig, angle); + + cv::Mat img_bb; + img_orig.copyTo(img_bb); + cv::Mat img_detection_mask = cv::Mat::zeros(img_orig.size(), CV_8U); + + std::vector rects = getRectangles(img_hist_projection, angle); + foreach(cv::Rect r, rects) + { + drawRectangles(img_bb,r, angle); + + // BB for measuring accuracy + cv::Mat bb_fill = cv::Mat::zeros(img_orig.size(), CV_8U); + cv::rectangle(bb_fill, r, cv::Scalar(255), -1); + bb_fill = artelab::rotate_image(bb_fill, -angle); + img_detection_mask.setTo(cv::Scalar(255), bb_fill); + } + + tc.stop(); + + // Measuring accuracy + results res = measure_results(img_detection_mask, img_truth, tc); + + // verbose output + if(!_quiet) + { + std::cout << std::setw(15) << std::left << name; + std::cout << std::setw(11) << std::left << ("Angle " + tostring(angle)); + std::cout << std::setw(20) << std::left << ("Accuracy: " + tostring(res.jaccard)); + std::cout << std::setw(6) << std::left << ("Time: " + tostring(res.time)) << std::endl; + } + +// // show results +// show_image("Prob Hough", line_image); +// show_image("histograms smooth", feature_with_hist_smooth); +// show_image("histograms projection", img_hist_projection); +// show_image("Cropped", img_cropped); +// show_image("Boundig Boxes", img_bb); +// show_image("Original", img_orig); +// show_image("Feature", feature_image); +// show_image("Canny", img_canny); +// +// // saving intermediate images +// if(_output.fullPath() != "") +// { +// std::ostringstream ss; +// +// // Original +// ss << name << "_1_original.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_orig); +// +// // Hough accumulator +// ss.str(""); ss.clear(); ss << name << "_2_accumulator.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), hough.get_hough_image()); +// +// // Hough MLP threshold +// img_neural = draw_histogram_on_image(get_histogram(img_neural, HIST_ROW, CV_32F), img_neural, cv::Scalar(0,0,255), HIST_ROW); +// ss.str(""); ss.clear();; ss << name << "_3_thresh_mlp.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_neural); +// +// // Feature with histograms not processed +// histograms_from_hough_lines(feature_image, row_hist, col_hist, false); +// cv::Mat feature_with_hist = draw_histogram_on_image(row_hist, feature_image, cv::Scalar(0,0,255), HIST_ROW); +// feature_with_hist = draw_histogram_on_image(col_hist, feature_with_hist, cv::Scalar(0,255,0), HIST_COL); +// ss.str(""); ss.clear();; ss << name << "_4_hist.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), feature_with_hist); +// +// // Feature with processed histograms +// ss.str(""); ss.clear();; ss << name << "_5_hist_smooth_thresh.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), feature_with_hist_smooth); +// +// // Histogram projection +// ss.str(""); ss.clear();; ss << name << "_6_hist_projection.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_hist_projection); +// +// // Bounding boxes +// ss.str(""); ss.clear();; ss << name << "_7_boxes.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_bb); +// +// // Canny +// ss.str(""); ss.clear(); ss << name << "_canny.png"; +// cv::imwrite(_output.fileCombine(ss.str()).fullName(), img_canny); +// } + + return res; + } + +} diff --git a/src/MLP.cpp b/src/MLP.cpp index 212ba38..86acd1f 100644 --- a/src/MLP.cpp +++ b/src/MLP.cpp @@ -8,47 +8,52 @@ namespace artelab MLP::MLP(cv::Mat layers) { - _model.create(layers, CvANN_MLP::SIGMOID_SYM, 1, 1); + std::cout<setLayerSizes(layers); + _model->setActivationFunction(cv::ml::ANN_MLP::ActivationFunctions::SIGMOID_SYM); } MLP::~MLP() {} - void MLP::load(std::string file) + void MLP::load(std::string file) { - _model.load(file.c_str()); + + _model = cv::Algorithm::load(file.c_str()); } - void MLP::save(std::string file) + void MLP::save(std::string file) { - _model.save(file.c_str()); + _model->save(file.c_str()); } - int MLP::train(const cv::Mat& patterns_in, const cv::Mat& targets, const int max_iter) + int MLP::train(const cv::Mat& patterns_in, const cv::Mat& targets, const int max_iter) { CV_Assert(patterns_in.rows == targets.rows); - CvTermCriteria criteria; - criteria.max_iter = max_iter; + cv::TermCriteria criteria; + criteria.maxCount = max_iter; criteria.epsilon = 0.001f; - criteria.type = CV_TERMCRIT_ITER | CV_TERMCRIT_EPS; - - cv::ANN_MLP_TrainParams params; - params.train_method = CvANN_MLP_TrainParams::RPROP; - params.bp_dw_scale = 0.1f; - params.bp_moment_scale = 0.1f; - params.rp_dw0 = 0.1f; - params.rp_dw_plus = 1.2f; - params.rp_dw_minus = 0.5f; - params.rp_dw_min = FLT_EPSILON; - params.rp_dw_max = 50; - params.term_crit = criteria; - - int iterations = _model.train(patterns_in, targets, cv::Mat(), cv::Mat(), params); + criteria.type = cv::TermCriteria::Type::COUNT | cv::TermCriteria::Type::EPS; + + _model->setTrainMethod(cv::ml::ANN_MLP::TrainingMethods::RPROP); + _model->setBackpropMomentumScale(0.1f); + _model->setBackpropWeightScale(0.1f); + _model->setRpropDW0(0.1f); + _model->setRpropDWMax(50); + _model->setRpropDWMin(FLT_EPSILON); + _model->setRpropDWMinus(0.5f); + _model->setRpropDWPlus(1.2f); + _model->setTermCriteria(criteria); + + //TODO: might have to change to COL_SAMPLE + cv::Ptr training_data = cv::ml::TrainData::create(patterns_in, cv::ml::SampleTypes::ROW_SAMPLE, targets); + int iterations = _model->train(training_data); return iterations; } - void MLP::predict(const cv::Mat& samples, cv::Mat& outPredictions) + void MLP::predict(const cv::Mat& samples, cv::Mat& outPredictions) { - _model.predict(samples, outPredictions); + _model->predict(samples, outPredictions); } -} \ No newline at end of file +} diff --git a/src/accuracy.cpp b/src/accuracy.cpp index f3c68c6..e6463a3 100644 --- a/src/accuracy.cpp +++ b/src/accuracy.cpp @@ -4,7 +4,7 @@ namespace artelab { - + double jaccard_overlap(cv::Mat mask, cv::Mat truth) { CV_Assert(mask.type() == CV_8U); @@ -45,6 +45,6 @@ namespace artelab return res; } - - + + } diff --git a/src/draw_hist.cpp b/src/draw_hist.cpp index f0ab67d..c927e1d 100644 --- a/src/draw_hist.cpp +++ b/src/draw_hist.cpp @@ -6,10 +6,10 @@ namespace artelab { - + cv::Mat draw_histogram_on_image(cv::Mat hist, cv::Mat img, cv::Scalar color, int direction, int thick, int hist_height) { - cv::Mat image; + cv::Mat image; img.copyTo(image); if(img.type() == CV_8U) cv::cvtColor(image, image, CV_GRAY2BGR); @@ -38,6 +38,6 @@ namespace artelab } return image; } - - + + } diff --git a/src/hough_histogram.cpp b/src/hough_histogram.cpp index 70e724e..0f6677c 100644 --- a/src/hough_histogram.cpp +++ b/src/hough_histogram.cpp @@ -3,7 +3,7 @@ namespace artelab { - + cv::Mat get_histogram(cv::Mat image, int hist_type, int type) { CV_Assert(image.type() == CV_8U); @@ -33,7 +33,7 @@ namespace artelab return hist; } - + int max_angle_hist(cv::Mat hist) { CV_Assert(hist.cols == 1); @@ -52,5 +52,5 @@ namespace artelab } return angle; } - -} \ No newline at end of file + +} diff --git a/src/mlp_threshold.cpp b/src/mlp_threshold.cpp index 751e902..912e02c 100644 --- a/src/mlp_threshold.cpp +++ b/src/mlp_threshold.cpp @@ -4,21 +4,21 @@ namespace artelab { - void imageToPattern(cv::Mat& roi, cv::Mat& roiToPattern) + void imageToPattern(cv::Mat& roi, cv::Mat& roiToPattern) { roiToPattern.create(1, roi.rows * roi.cols, CV_32F); float* dstRow = roiToPattern.ptr(0); - for (int n = 0, r = 0; r < roi.rows; r++) + for (int n = 0, r = 0; r < roi.rows; r++) { float* srcRow = roi.ptr(r); - for (int c = 0; c < roi.cols; c++) + for (int c = 0; c < roi.cols; c++) { dstRow[n++] = srcRow[c]; } } } - - void read_normalized_patch(const cv::Mat& imageHoughPadded, cv::Size win_size, cv::Mat& roiNorm, cv::Point2i p) + + void read_normalized_patch(const cv::Mat& imageHoughPadded, cv::Size win_size, cv::Mat& roiNorm, cv::Point2i p) { cv::Mat roi(imageHoughPadded, cv::Range(p.y, p.y + win_size.height), @@ -29,25 +29,25 @@ namespace artelab roiNorm = roiNorm * (2.0f / 255.0f) - 1; // [-1,1] } - - void read_patch_as_row_vector(const cv::Mat& imageHoughPadded, cv::Size win_size, cv::Mat& roiAsVector, cv::Point2i p) + + void read_patch_as_row_vector(const cv::Mat& imageHoughPadded, cv::Size win_size, cv::Mat& roiAsVector, cv::Point2i p) { cv::Mat roiNorm; read_normalized_patch(imageHoughPadded, win_size, roiNorm, p); imageToPattern(roiNorm, roiAsVector); } - + cv::Mat zero_padding(const cv::Mat& src, const int x_padding, const int y_padding) { cv::Mat out = cv::Mat::zeros(src.rows + 2*y_padding, src.cols + 2*x_padding, src.type()); - cv::Mat m = cv::Mat(out, + cv::Mat m = cv::Mat(out, cv::Range(y_padding, out.rows-y_padding), cv::Range(x_padding, out.cols-x_padding)); src.copyTo(m); return out; } - - cv::Mat threshold_mlp(MLP& nnetwork, cv::Size win_size, HoughTransform& hough) + + cv::Mat threshold_mlp(MLP& nnetwork, cv::Size win_size, HoughTransform& hough) { int halfWinInH = win_size.height / 2; int halfWinInW = win_size.width / 2; @@ -63,9 +63,9 @@ namespace artelab cv::Mat window(1, win_size.height*win_size.width, CV_32F); - for (int row = 0; row < imageHough.rows; row += win_size.height) + for (int row = 0; row < imageHough.rows; row += win_size.height) { - for (int col = 0; col < imageHough.cols; col += win_size.width) + for (int col = 0; col < imageHough.cols; col += win_size.width) { cv::Point2i p(col, row); @@ -75,12 +75,12 @@ namespace artelab float* netOutput = mlp_out.ptr(0); - for (int r = -halfWinOutH; r <= halfWinOutH; r++) + for (int r = -halfWinOutH; r <= halfWinOutH; r++) { - if ((row + r) < 0 || (row + r) >= outputImage.rows) + if ((row + r) < 0 || (row + r) >= outputImage.rows) continue; uchar* dstRow = outputImage.ptr(row + r); - for (int c = -halfWinOutW; c <= halfWinOutW; c++) + for (int c = -halfWinOutW; c <= halfWinOutW; c++) { if ((col + c) < 0 || (col + c) >= imageHough.cols) continue; @@ -99,4 +99,4 @@ namespace artelab -} \ No newline at end of file +} diff --git a/utils-3rdparty/utils.cpp b/utils-3rdparty/utils.cpp index f674e1f..4eb7dce 100755 --- a/utils-3rdparty/utils.cpp +++ b/utils-3rdparty/utils.cpp @@ -2,16 +2,17 @@ #include #include #include +#include #include #include "utils.hpp" -namespace artelab +namespace artelab { - std::vector &split(const std::string &s, char delim, std::vector &elems) + std::vector &split(const std::string &s, char delim, std::vector &elems) { std::stringstream ss(s); std::string item; - while(std::getline(ss, item, delim)) + while(std::getline(ss, item, delim)) { if (item != "") elems.push_back(item); @@ -20,18 +21,19 @@ namespace artelab } - std::vector split(const std::string &s, char delim) + std::vector split(const std::string &s, char delim) { std::vector elems; return split(s, delim, elems); } - + + //Changed the file existance checking to use stat, as it is at least 2x faster bool file_exists(std::string filename) { - std::ifstream ifile(filename.c_str()); - return ifile; + struct stat buffer; + return (stat(filename.c_str(), &buffer)==0); } - + cv::Mat rotate_image(cv::Mat img, int angle, int size_factor) { cv::Mat out; @@ -42,11 +44,11 @@ namespace artelab cv::warpAffine(img, out, rotation_mat, size); return out; } - + std::string tostring(double d) { std::ostringstream ss; ss << d; return ss.str(); } -} \ No newline at end of file +}