Merge pull request #456 from drnikolaev/caffe-0.16

0.16.5 Release

Author: drnikolaev

3rdparty/half_float/half.hpp

@@ -1058,7 +1058,13 @@ namespace half_float
     CAFFE_UTIL_HD
     half& operator=(detail::expr rhs) { return *this = static_cast<float>(rhs); }
 
-		/// Arithmetic assignment.
+    CAFFE_UTIL_HD
+    half& operator=(const half& rhs) {
+      data_ = rhs.data_;
+      return *this;
+    }
+
+    /// Arithmetic assignment.
 		/// \tparam T type of concrete half expression
 		/// \param rhs half expression to add
 		/// \return reference to this half
@@ -1172,6 +1178,12 @@ namespace half_float
 		detail::uint16 data_;
 	};
 
+  template <>
+  CAFFE_UTIL_IHD half& half::operator= <half>(const half& rhs) {
+    data_ = rhs.data_;
+    return *this;
+  }
+
 #if HALF_ENABLE_CPP11_USER_LITERALS
 	/// Library-defined half-precision literals.
 	/// Import this namespace to enable half-precision floating point literals:
@@ -3086,6 +3098,10 @@ namespace std
 
   inline bool signbit(const half_float::half& arg) { return half_float::detail::signbit(arg); }
 
+  inline half_float::half fabs(half_float::half x) {
+		return abs(x);
+	}
+
 
 #if HALF_ENABLE_CPP11_HASH
 	/// Hash function for half-precision floats.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.7)
+cmake_minimum_required(VERSION 3.4.0)
 if(POLICY CMP0046)
   cmake_policy(SET CMP0046 NEW)
 endif()
@@ -52,7 +52,6 @@ set(python_version "2" CACHE STRING "Specify which Python version to use")
 caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE)
 caffe_option(BUILD_docs   "Build documentation" ON IF UNIX OR APPLE)
 caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON)
-caffe_option(USE_OPENCV "Build with OpenCV support" ON)
 caffe_option(USE_LEVELDB "Build with levelDB" ON)
 caffe_option(USE_LMDB "Build with lmdb" ON)
 caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)

Makefile

@@ -90,6 +90,7 @@ EMPTY_LINT_REPORT := $(BUILD_DIR)/.$(LINT_EXT)
 NONEMPTY_LINT_REPORT := $(BUILD_DIR)/$(LINT_EXT)
 # PY$(PROJECT)_SRC is the python wrapper for $(PROJECT)
 PY$(PROJECT)_SRC := python/$(PROJECT)/_$(PROJECT).cpp
+PY$(PROJECT)_OBJ := $(BUILD_DIR)/src/$(PROJECT)/layers/python_layer.o
 PY$(PROJECT)_SO := python/$(PROJECT)/_$(PROJECT).so
 PY$(PROJECT)_HXX := include/$(PROJECT)/layers/python_layer.hpp
 # MAT$(PROJECT)_SRC is the mex entrance point of matlab package for $(PROJECT)
@@ -192,7 +193,10 @@ ifneq ($(NO_NVML), 1)
 endif
 endif
 
-LIBRARIES += boost_system glog gflags protobuf boost_filesystem m
+# Note: libturbojpeg has a packaging bug. Workaround:
+# $ sudo ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so
+
+LIBRARIES += boost_system glog gflags protobuf boost_filesystem m turbojpeg
 ifeq ($(TEGRA), 1)
     LIBRARIES += hdf5_serial_hl hdf5_serial
 else
@@ -211,14 +215,14 @@ ifeq ($(USE_LMDB), 1)
 	LIBRARIES += lmdb
 endif
 ifeq ($(USE_OPENCV), 1)
-	LIBRARIES += opencv_core opencv_highgui opencv_imgproc 
+	LIBRARIES += opencv_core opencv_highgui opencv_imgproc
 
 	ifeq ($(OPENCV_VERSION), 3)
 		LIBRARIES += opencv_imgcodecs
 	endif
 
 endif
-PYTHON_LIBRARIES ?= boost_python python2.7
+PYTHON_LIBRARIES ?= boost_python python2.7 boost_regex
 WARNINGS := -Wall -Wno-sign-compare
 
 ##############################
@@ -532,10 +536,10 @@ py$(PROJECT): py
 
 py: $(PY$(PROJECT)_SO) $(PROTO_GEN_PY)
 
-$(PY$(PROJECT)_SO): $(PY$(PROJECT)_SRC) $(PY$(PROJECT)_HXX) | $(DYNAMIC_NAME)
+$(PY$(PROJECT)_SO): $(PY$(PROJECT)_OBJ) $(PY$(PROJECT)_SRC) $(PY$(PROJECT)_HXX) | $(DYNAMIC_NAME)
 	@ echo CXX/LD -o $@ $<
-	$(Q)$(CXX) -shared -o $@ $(PY$(PROJECT)_SRC) \
-		-o $@ $(LINKFLAGS) -l$(LIBRARY_NAME) $(PYTHON_LDFLAGS) \
+	$(Q)$(CXX) -shared -o $@ $(PY$(PROJECT)_SRC) $(PY$(PROJECT)_OBJ) \
+		$(LINKFLAGS) -l$(LIBRARY_NAME) $(PYTHON_LDFLAGS) \
 		-Wl,-rpath,$(ORIGIN)/../../build/lib
 
 mat$(PROJECT): mat
@@ -562,7 +566,7 @@ runtest: $(TEST_ALL_BIN)
 	$(TEST_ALL_BIN) $(TEST_GPUID) --gtest_shuffle $(TEST_FILTER)
 
 pytest: py
-	cd python; python -m unittest discover -s caffe/test
+	cd python; python -u -m unittest discover -s caffe/test
 
 mattest: mat
 	cd matlab; $(MATLAB_DIR)/bin/matlab -nodisplay -r 'caffe.run_tests(), exit()'

classification/example.py

@@ -147,7 +147,7 @@ def forward_pass(images, net, transformer, batch_size=None):
             scores = np.copy(output)
         else:
             scores = np.vstack((scores, output))
-        print 'Processed %s/%s images in %f seconds ...' % (len(scores), len(caffe_images), (end - start))
+        print('Processed %s/%s images in %f seconds ...' % (len(scores), len(caffe_images), (end - start)))
 
     return scores
 
@@ -159,7 +159,7 @@ def read_labels(labels_file):
     labels_file -- path to a .txt file
     """
     if not labels_file:
-        print 'WARNING: No labels file provided. Results will be difficult to interpret.'
+        print('WARNING: No labels file provided. Results will be difficult to interpret.')
         return None
 
     labels = []
@@ -252,5 +252,5 @@ def classify(caffemodel, deploy_file, image_files,
     classify(args['caffemodel'], args['deploy_file'], args['image_file'],
             args['mean'], args['labels'], args['batch_size'], not args['nogpu'])
 
-    print 'Script took %f seconds.' % (time.time() - script_start_time,)
+    print('Script took %f seconds.' % (time.time() - script_start_time,))
 

classification/use_archive.py

@@ -33,11 +33,11 @@ def unzip_archive(archive):
         pass
     else:
         if tarfile.is_tarfile(archive):
-            print 'Extracting tarfile ...'
+            print('Extracting tarfile ...')
             with tarfile.open(archive) as tf:
                 tf.extractall(path=tmpdir)
         elif zipfile.is_zipfile(archive):
-            print 'Extracting zipfile ...'
+            print('Extracting zipfile ...')
             with zipfile.ZipFile(archive) as zf:
                 zf.extractall(path=tmpdir)
         else:
@@ -64,7 +64,7 @@ def classify_with_archive(archive, image_files, batch_size=None, use_gpu=True):
         elif filename == 'labels.txt':
             labels_file = full_path
         else:
-            print 'Unknown file:', filename
+            print('Unknown file:', filename)
 
     assert caffemodel is not None, 'Caffe model file not found'
     assert deploy_file is not None, 'Deploy file not found'
@@ -101,5 +101,5 @@ def classify_with_archive(archive, image_files, batch_size=None, use_gpu=True):
                           use_gpu=(not args['nogpu']),
                           )
 
-    print 'Script took %f seconds.' % (time.time() - script_start_time,)
+    print('Script took %f seconds.' % (time.time() - script_start_time,))
 

cmake/ConfigGen.cmake

@@ -56,10 +56,6 @@ function(caffe_generate_export_configs)
     list(APPEND Caffe_DEFINITIONS -DCPU_ONLY)
   endif()
 
-  if(USE_OPENCV)
-    list(APPEND Caffe_DEFINITIONS -DUSE_OPENCV)
-  endif()
-
   if(USE_LMDB)
     list(APPEND Caffe_DEFINITIONS -DUSE_LMDB)
     if (ALLOW_LMDB_NOLOCK)

cmake/Dependencies.cmake

@@ -1,8 +1,11 @@
 # This list is required for static linking and exported to CaffeConfig.cmake
 set(Caffe_LINKER_LIBS "")
 
+find_package(PythonInterp ${python_version})
+
 # ---[ Boost
-find_package(Boost 1.46 REQUIRED COMPONENTS system thread filesystem)
+find_package(Boost 1.54 REQUIRED COMPONENTS system thread filesystem regex python-py${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR})
+set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}_FOUND})
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})
 list(APPEND Caffe_LINKER_LIBS ${Boost_LIBRARIES})
 
@@ -54,6 +57,10 @@ if(USE_LEVELDB)
   list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES})
 endif()
 
+find_package(JPEGTurbo REQUIRED)
+include_directories(SYSTEM ${JPEGTurbo_INCLUDE_DIR})
+list(APPEND Caffe_LINKER_LIBS ${JPEGTurbo_LIBRARIES})
+
 # ---[ CUDA
 include(cmake/Cuda.cmake)
 if(NOT HAVE_CUDA)
@@ -68,23 +75,20 @@ if(NOT HAVE_CUDA)
 endif()
 
 # ---[ OpenCV
-if(USE_OPENCV)
-  find_package(OpenCV QUIET COMPONENTS imgcodecs)
-  if(OPENCV_IMGCODECS_FOUND)
-    find_package(OpenCV REQUIRED COMPONENTS core imgcodecs imgproc)
-    message(STATUS "Found OpenCV 3.x: ${OpenCV_CONFIG_PATH}")
-  else()
-    find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
-    message(STATUS "Found OpenCV 2.x: ${OpenCV_CONFIG_PATH}")
-  endif()
-  include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
-  list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS})
-  add_definitions(-DUSE_OPENCV)
+find_package(OpenCV QUIET COMPONENTS imgcodecs)
+if(OPENCV_IMGCODECS_FOUND)
+  find_package(OpenCV REQUIRED COMPONENTS core imgcodecs imgproc)
+  message(STATUS "Found OpenCV 3.x: ${OpenCV_CONFIG_PATH}")
+else()
+  find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
+  message(STATUS "Found OpenCV 2.x: ${OpenCV_CONFIG_PATH}")
 endif()
+include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
+list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS})
 
 # ---[ BLAS
 if(NOT APPLE)
-  set(BLAS "Atlas" CACHE STRING "Selected BLAS library")
+  set(BLAS "Open" CACHE STRING "Selected BLAS library")
   set_property(CACHE BLAS PROPERTY STRINGS "Atlas;Open;MKL")
 
   if(BLAS STREQUAL "Atlas" OR BLAS STREQUAL "atlas")
@@ -109,46 +113,14 @@ endif()
 
 # ---[ Python
 if(BUILD_python)
-  if(NOT "${python_version}" VERSION_LESS "3.0.0")
-    # use python3
-    find_package(PythonInterp 3.0)
-    find_package(PythonLibs 3.0)
-    find_package(NumPy 1.7.1)
-    # Find the matching boost python implementation
-    set(version ${PYTHONLIBS_VERSION_STRING})
-    
-    STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} )
-    find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}")
-    set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND})
-    
-    while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND)
-      STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} )
-      
-      STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} )
-      find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}")
-      set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND})
-      
-      STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} )
-      if("${has_more_version}" STREQUAL "")
-        break()
-      endif()
-    endwhile()
-    if(NOT Boost_PYTHON_FOUND)
-      find_package(Boost 1.46 COMPONENTS python)
-    endif()
-  else()
-    # disable Python 3 search
-    find_package(PythonInterp 2.7)
-    find_package(PythonLibs 2.7)
-    find_package(NumPy 1.7.1)
-    find_package(Boost 1.46 COMPONENTS python)
-  endif()
+  find_package(PythonLibs ${python_version})
+  find_package(NumPy 1.7.1)
   if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND Boost_PYTHON_FOUND)
     set(HAVE_PYTHON TRUE)
     if(BUILD_python_layer)
       add_definitions(-DWITH_PYTHON_LAYER)
       include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} ${Boost_INCLUDE_DIRS})
-      list(APPEND Caffe_LINKER_LIBS ${PYTHON_LIBRARIES} ${Boost_LIBRARIES})
+      list(APPEND Caffe_LINKER_LIBS ${PYTHON_LIBRARIES})
     endif()
   endif()
 endif()
@@ -186,7 +158,7 @@ else()
 endif()
 if(NCCL_FOUND)
   add_definitions(-DUSE_NCCL)
-  include_directories(SYSTEM ${NCCL_INCLUDE})
+  include_directories(SYSTEM ${NCCL_INCLUDE_DIR})
   list(APPEND Caffe_LINKER_LIBS ${NCCL_LIBRARY})
 endif()
 

cmake/Modules/FindJPEGTurbo.cmake

@@ -0,0 +1,24 @@
+# Find the JPEGTurbo libraries
+#
+# The following variables are optionally searched for defaults
+#  JPEGTurbo_ROOT_DIR:    Base directory where all JPEGTurbo components are found
+#
+# The following are set after configuration is done:
+#  JPEGTurbo_FOUND
+#  JPEGTurbo_INCLUDE_DIR
+#  JPEGTurbo_LIBRARIES
+
+find_path(JPEGTurbo_INCLUDE_DIR NAMES turbojpeg.h
+                             PATHS /usr/include ${JPEGTurbo_ROOT_DIR} ${JPEGTurbo_ROOT_DIR}/include)
+
+find_library(JPEGTurbo_LIBRARIES SHARED NAMES libturbojpeg.so.0
+                              PATHS  ${JPEGTurbo_ROOT_DIR} ${JPEGTurbo_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(JPEGTurbo DEFAULT_MSG JPEGTurbo_INCLUDE_DIR JPEGTurbo_LIBRARIES)
+
+if(JPEGTurbo_FOUND)
+  message(STATUS "Found JPEGTurbo  (include: ${JPEGTurbo_INCLUDE_DIR}, library: ${JPEGTurbo_LIBRARIES})")
+  mark_as_advanced(JPEGTurbo_INCLUDE_DIR JPEGTurbo_LIBRARIES)
+endif()
+

cmake/Modules/FindNCCL.cmake

@@ -21,5 +21,9 @@ find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIR NCCL_LIBRARY
 if(NCCL_FOUND)
   message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIR}, library: ${NCCL_LIBRARY})")
   mark_as_advanced(NCCL_INCLUDE_DIR NCCL_LIBRARY)
+  caffe_parse_header(${NCCL_INCLUDE_DIR}/nccl.h
+          NCCL_VERION_LINES NCCL_MAJOR NCCL_MINOR NCCL_PATCH)
+  set(NCCL_VERSION "${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH}")
+
 endif()
 

cmake/Summary.cmake

@@ -114,7 +114,6 @@ function(caffe_print_configuration_summary)
   caffe_status("  BUILD_matlab      :   ${BUILD_matlab}")
   caffe_status("  BUILD_docs        :   ${BUILD_docs}")
   caffe_status("  CPU_ONLY          :   ${CPU_ONLY}")
-  caffe_status("  USE_OPENCV        :   ${USE_OPENCV}")
   caffe_status("  USE_LEVELDB       :   ${USE_LEVELDB}")
   caffe_status("  USE_LMDB          :   ${USE_LMDB}")
   caffe_status("  ALLOW_LMDB_NOLOCK :   ${ALLOW_LMDB_NOLOCK}")
@@ -133,9 +132,8 @@ function(caffe_print_configuration_summary)
     caffe_status("  LevelDB           : " LEVELDB_FOUND THEN  "Yes (ver. ${LEVELDB_VERSION})" ELSE "No")
     caffe_status("  Snappy            : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" )
   endif()
-  if(USE_OPENCV)
-    caffe_status("  OpenCV            :   Yes (ver. ${OpenCV_VERSION})")
-  endif()
+  caffe_status("  OpenCV            :   Yes (ver. ${OpenCV_VERSION})")
+  caffe_status("  JPEGTurbo         : " JPEGTurbo_FOUND THEN "Yes" ELSE "No" )
   caffe_status("  CUDA              : " HAVE_CUDA THEN "Yes (ver. ${CUDA_VERSION})" ELSE "No" )
   caffe_status("")
   if(HAVE_CUDA)
@@ -148,9 +146,10 @@ function(caffe_print_configuration_summary)
       caffe_status("  cuDNN             :   Disabled")
     endif()
     if(NOT DEFINED USE_NCCL)
-      caffe_status("  NCCL              : " NCCL_FOUND THEN "Yes" ELSE "Not found")
+      caffe_status("  NCCL              : " NCCL_FOUND THEN "Yes (ver. ${NCCL_VERSION})" ELSE "Not
+      found")
     elseif(USE_NCCL)
-      caffe_status("  NCCL              : " NCCL_FOUND THEN "Yes" ELSE "Not found")
+      caffe_status("  NCCL              : " NCCL_FOUND THEN "Yes (ver. ${NCCL_VERSION})" ELSE "Not found")
     else()
       caffe_status("  NCCL              :   Disabled")
     endif()