diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..5ace4600a1f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000..d0a20b64907 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,50 @@ +name: CI + +on: [push, pull_request, workflow_dispatch] + +env: + NUM_THREADS: 4 + +jobs: + build: + name: ${{ matrix.env.BUILD_NAME }} - ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04, ubuntu-20.04] + # envvar defaults: + # WITH_CMAKE: false + # WITH_IO: true + # WITH_CUDA: false + # WITH_CUDNN: false + env: + # - {BUILD_NAME: default-make} + # - {BUILD_NAME: no-io-make, WITH_IO: false} + # - {BUILD_NAME: cuda-make, WITH_CUDA: true} + # - {BUILD_NAME: cudnn-make, WITH_CUDA: true, WITH_CUDNN: true} + - {BUILD_NAME: default-cmake, WITH_CMAKE: true} + - {BUILD_NAME: no-io-cmake, WITH_CMAKE: true, WITH_IO: false} + - {BUILD_NAME: cuda-cmake, WITH_CMAKE: true, WITH_CUDA: true} + - {BUILD_NAME: cudnn-cmake, WITH_CMAKE: true, WITH_CUDA: true, WITH_CUDNN: true} + env: ${{ matrix.env }} + steps: + - uses: actions/checkout@v3 + - name: Install + run: | + sudo -E ./scripts/ci/install-deps.sh + ./scripts/ci/setup-venv.sh ~/venv + source ~/venv/bin/activate + ./scripts/ci/install-python-deps.sh + - name: Configure + run: | + source ~/venv/bin/activate + ./scripts/ci/configure.sh + - name: Build + run: | + source ~/venv/bin/activate + ./scripts/ci/build.sh + - name: Test + run: | + source ~/venv/bin/activate + ./scripts/ci/test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 905cd7d83ef..4608b0a63eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,6 @@ caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ON caffe_option(USE_NCCL "Build Caffe with NCCL library support" OFF) caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON) caffe_option(BUILD_python "Build Python wrapper" ON) -set(python_version "2" CACHE STRING "Specify which Python version to use") caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE) caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON) @@ -105,7 +104,7 @@ add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/l # ---[ pytest target if(BUILD_python) - add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) + add_custom_target(pytest COMMAND python3 -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) add_dependencies(pytest pycaffe) endif() diff --git a/Makefile.config.example b/Makefile.config.example new file mode 100644 index 00000000000..0db168074c0 --- /dev/null +++ b/Makefile.config.example @@ -0,0 +1,119 @@ +## Refer to http://caffe.berkeleyvision.org/installation.html +# Contributions simplifying and improving our build system are welcome! + +# cuDNN acceleration switch (uncomment to build with cuDNN). +# USE_CUDNN := 1 + +# CPU-only switch (uncomment to build without GPU support). +# CPU_ONLY := 1 + +# uncomment to disable IO dependencies and corresponding data layers +# USE_OPENCV := 0 +# USE_LEVELDB := 0 +# USE_LMDB := 0 +# This code is taken from https://github.com/sh1r0/caffe-android-lib +# USE_HDF5 := 0 + +# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary) +# You should not set this flag if you will be reading LMDBs with any +# possibility of simultaneous read and write +# ALLOW_LMDB_NOLOCK := 1 + +# Uncomment if you're using OpenCV 3 +# OPENCV_VERSION := 3 + +# To customize your choice of compiler, uncomment and set the following. +# N.B. the default for Linux is g++ and the default for OSX is clang++ +# CUSTOM_CXX := g++ + +# CUDA directory contains bin/ and lib/ directories that we need. +CUDA_DIR := /usr/local/cuda +# On Ubuntu 14.04, if cuda tools are installed via +# "sudo apt-get install nvidia-cuda-toolkit" then use this instead: +# CUDA_DIR := /usr + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 through *_61 lines for compatibility. +# For CUDA < 8.0, comment the *_60 and *_61 lines for compatibility. +# For CUDA >= 9.0, comment the *_20 and *_21 lines for compatibility. +CUDA_ARCH := -gencode arch=compute_20,code=sm_20 \ + -gencode arch=compute_20,code=sm_21 \ + -gencode arch=compute_30,code=sm_30 \ + -gencode arch=compute_35,code=sm_35 \ + -gencode arch=compute_50,code=sm_50 \ + -gencode arch=compute_52,code=sm_52 \ + -gencode arch=compute_60,code=sm_60 \ + -gencode arch=compute_61,code=sm_61 \ + -gencode arch=compute_61,code=compute_61 + +# BLAS choice: +# atlas for ATLAS (default) +# mkl for MKL +# open for OpenBlas +BLAS := atlas +# Custom (MKL/ATLAS/OpenBLAS) include and lib directories. +# Leave commented to accept the defaults for your choice of BLAS +# (which should work)! +# BLAS_INCLUDE := /path/to/your/blas +# BLAS_LIB := /path/to/your/blas + +# Homebrew puts openblas in a directory that is not on the standard search path +# BLAS_INCLUDE := $(shell brew --prefix openblas)/include +# BLAS_LIB := $(shell brew --prefix openblas)/lib + +# This is required only if you will compile the matlab interface. +# MATLAB directory should contain the mex binary in /bin. +# MATLAB_DIR := /usr/local +# MATLAB_DIR := /Applications/MATLAB_R2012b.app + +# NOTE: this is required only if you will compile the python interface. +# We need to be able to find Python.h and numpy/arrayobject.h. +PYTHON_LIBRARIES := boost_python3 python3.8 +PYTHON_INCLUDE := /usr/include/python3.8 \ + /usr/lib/python3/dist-packages/numpy/core/include +# Anaconda Python distribution is quite popular. Include path: +# Verify anaconda location, sometimes it's in root. +# ANACONDA_HOME := $(HOME)/anaconda +# PYTHON_INCLUDE := $(ANACONDA_HOME)/include \ + # $(ANACONDA_HOME)/include/python3.8 \ + # $(ANACONDA_HOME)/lib/python3.8/site-packages/numpy/core/include + +# We need to be able to find libpythonX.X.so or .dylib. +PYTHON_LIB := /usr/lib /usr/local/lib +# PYTHON_LIB := $(ANACONDA_HOME)/lib + +# Homebrew installs numpy in a non standard path (keg only) +# PYTHON_INCLUDE += $(dir $(shell python3 -c 'import numpy.core; print(numpy.core.__file__)'))/include +# PYTHON_LIB += $(shell brew --prefix numpy)/lib + +# Uncomment to support layers written in Python (will link against Python libs) +# WITH_PYTHON_LAYER := 1 + +# Whatever else you find you need goes here. +INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include +LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib + +# If Homebrew is installed at a non standard location (for example your home directory) and you use it for general dependencies +# INCLUDE_DIRS += $(shell brew --prefix)/include +# LIBRARY_DIRS += $(shell brew --prefix)/lib + +# NCCL acceleration switch (uncomment to build with NCCL) +# https://github.com/NVIDIA/nccl (last tested version: v1.2.3-1+cuda8.0) +# USE_NCCL := 1 + +# Uncomment to use `pkg-config` to specify OpenCV library paths. +# (Usually not necessary -- OpenCV libraries are normally installed in one of the above $LIBRARY_DIRS.) +# USE_PKG_CONFIG := 1 + +# N.B. both build and distribute dirs are cleared on `make clean` +BUILD_DIR := build +DISTRIBUTE_DIR := distribute + +# Uncomment for debugging. Does not work on OSX due to https://github.com/BVLC/caffe/issues/171 +# DEBUG := 1 + +# The ID of the GPU that 'make runtest' will use to run unit tests. +TEST_GPUID := 0 + +# enable pretty build (comment to see full commands) +Q ?= @ diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index ca2e3ad9e5e..537e8d82661 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -43,14 +43,10 @@ list(APPEND Caffe_LINKER_LIBS PUBLIC ${GFLAGS_LIBRARIES}) include(cmake/ProtoBuf.cmake) # ---[ HDF5 -find_package(HDF5 COMPONENTS HL REQUIRED) -list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${HDF5_INCLUDE_DIRS}) -list(APPEND Caffe_LINKER_LIBS PUBLIC ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) - # This code is taken from https://github.com/sh1r0/caffe-android-lib if(USE_HDF5) find_package(HDF5 COMPONENTS HL REQUIRED) - include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR}) + include_directories(SYSTEM ${HDF5_INCLUDE_DIRS}) list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) add_definitions(-DUSE_HDF5) endif() @@ -145,46 +141,37 @@ endif() # ---[ Python if(BUILD_python) - if(NOT "${python_version}" VERSION_LESS "3.0.0") - # use python3 - find_package(PythonInterp 3.0) - find_package(PythonLibs 3.0) - find_package(NumPy 1.7.1) - # Find the matching boost python implementation - set(version ${PYTHONLIBS_VERSION_STRING}) + # use python3 + find_package(Python 3 COMPONENTS Interpreter Development NumPy) + # Find the matching boost python implementation + set(version ${Python_VERSION}) + + STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} ) + find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") + set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) + + while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND) + STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} ) STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} ) find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) - while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND) - STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} ) - - STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} ) - find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") - set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) - - STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) - if("${has_more_version}" STREQUAL "") - break() - endif() - endwhile() - if(NOT Boost_PYTHON_FOUND) - find_package(Boost 1.46 COMPONENTS python) + STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) + if("${has_more_version}" STREQUAL "") + break() endif() - else() - # disable Python 3 search - find_package(PythonInterp 2.7) - find_package(PythonLibs 2.7) - find_package(NumPy 1.7.1) + endwhile() + if(NOT Boost_PYTHON_FOUND) find_package(Boost 1.46 COMPONENTS python) endif() - if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND Boost_PYTHON_FOUND) + + if(Python_Interpreter_FOUND AND Python_NumPy_FOUND AND Boost_PYTHON_FOUND) set(HAVE_PYTHON TRUE) if(BUILD_python_layer) list(APPEND Caffe_DEFINITIONS PRIVATE -DWITH_PYTHON_LAYER) - list(APPEND Caffe_INCLUDE_DIRS PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} PUBLIC ${Boost_INCLUDE_DIRS}) - list(APPEND Caffe_LINKER_LIBS PRIVATE ${PYTHON_LIBRARIES} PUBLIC ${Boost_LIBRARIES}) + list(APPEND Caffe_INCLUDE_DIRS PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} PUBLIC ${Boost_INCLUDE_DIRS}) + list(APPEND Caffe_LINKER_LIBS PRIVATE ${Python_LIBRARIES} PUBLIC ${Boost_LIBRARIES}) endif() endif() endif() diff --git a/cmake/Misc.cmake b/cmake/Misc.cmake index fcb246472f0..56c6a0c4cd4 100644 --- a/cmake/Misc.cmake +++ b/cmake/Misc.cmake @@ -29,7 +29,7 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) endif() # ---[ RPATH settings -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOLEAN "Use link paths for shared library rpath") +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOL "Use link paths for shared library rpath") set(CMAKE_MACOSX_RPATH TRUE) list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES diff --git a/cmake/ProtoBuf.cmake b/cmake/ProtoBuf.cmake index 72ea3230c50..f0a0d2b3cda 100644 --- a/cmake/ProtoBuf.cmake +++ b/cmake/ProtoBuf.cmake @@ -16,7 +16,7 @@ endif() if(PROTOBUF_FOUND) # fetches protobuf version caffe_parse_header(${PROTOBUF_INCLUDE_DIR}/google/protobuf/stubs/common.h VERION_LINE GOOGLE_PROTOBUF_VERSION) - string(REGEX MATCH "([0-9])00([0-9])00([0-9])" PROTOBUF_VERSION ${GOOGLE_PROTOBUF_VERSION}) + string(REGEX MATCH "([0-9])0([0-9]*)00([0-9])" PROTOBUF_VERSION ${GOOGLE_PROTOBUF_VERSION}) set(PROTOBUF_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") unset(GOOGLE_PROTOBUF_VERSION) endif() diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 40b8c2f2966..6c781d2aec6 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -125,8 +125,8 @@ function(caffe_print_configuration_summary) caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") caffe_status(" Boost : Yes (ver. ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION})") - caffe_status(" glog : Yes") - caffe_status(" gflags : Yes") + caffe_status(" glog : " GLOG_FOUND THEN "Yes" ELSE "No" ) + caffe_status(" gflags : " GFLAGS_FOUND THEN "Yes" ELSE "No" ) caffe_status(" protobuf : " PROTOBUF_FOUND THEN "Yes (ver. ${PROTOBUF_VERSION})" ELSE "No" ) if(USE_LMDB) caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No") @@ -153,9 +153,8 @@ function(caffe_print_configuration_summary) endif() if(HAVE_PYTHON) caffe_status("Python:") - caffe_status(" Interpreter :" PYTHON_EXECUTABLE THEN "${PYTHON_EXECUTABLE} (ver. ${PYTHON_VERSION_STRING})" ELSE "No") - caffe_status(" Libraries :" PYTHONLIBS_FOUND THEN "${PYTHON_LIBRARIES} (ver ${PYTHONLIBS_VERSION_STRING})" ELSE "No") - caffe_status(" NumPy :" NUMPY_FOUND THEN "${NUMPY_INCLUDE_DIR} (ver ${NUMPY_VERSION})" ELSE "No") + caffe_status(" Interpreter :" Python_EXECUTABLE THEN "${Python_EXECUTABLE} (ver. ${Python_VERSION})" ELSE "No") + caffe_status(" NumPy :" Python_NumPy_FOUND THEN "${Python_NumPy_INCLUDE_DIRS} (ver ${Python_NumPy_VERSION})" ELSE "No") caffe_status("") endif() if(BUILD_matlab) diff --git a/docker/gpu/Dockerfile b/docker/gpu/Dockerfile index 448cd0d3945..f8ea01445ba 100644 --- a/docker/gpu/Dockerfile +++ b/docker/gpu/Dockerfile @@ -28,7 +28,6 @@ WORKDIR $CAFFE_ROOT RUN cd /opt && \ git clone https://github.com/CMU-Perceptual-Computing-Lab/caffe.git && \ cd $CAFFE_ROOT && \ - sed -i 's/set(python_version "2"/set(python_version "3"/g' CMakeLists.txt && \ # sed -i 's/set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")/set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -std=c++11")/g' CMakeLists.txt && \ # sed -i 's/cudnn.h CUDNN_VERSION_FILE_CONTENTS/cudnn_version.h CUDNN_VERSION_FILE_CONTENTS/g' cmake/Cuda.cmake && \ mkdir build && cd build && \ diff --git a/include/caffe/util/cudnn.hpp b/include/caffe/util/cudnn.hpp index cd3f93f6e28..0051d0c12e3 100644 --- a/include/caffe/util/cudnn.hpp +++ b/include/caffe/util/cudnn.hpp @@ -50,6 +50,10 @@ inline const char* cudnnGetErrorString(cudnnStatus_t status) { return "CUDNN_STATUS_RUNTIME_IN_PROGRESS"; case CUDNN_STATUS_RUNTIME_FP_OVERFLOW: return "CUDNN_STATUS_RUNTIME_FP_OVERFLOW"; +#endif +#if CUDNN_VERSION_MIN(8, 0, 1) + case CUDNN_STATUS_VERSION_MISMATCH: + return "CUDNN_STATUS_VERSION_MISMATCH"; #endif } return "Unknown cudnn status"; diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c53299d265b..c73fcfa7b88 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -8,8 +8,8 @@ file(GLOB_RECURSE python_srcs ${PROJECT_SOURCE_DIR}/python/*.cpp) add_library(pycaffe SHARED ${python_srcs}) caffe_default_properties(pycaffe) set_target_properties(pycaffe PROPERTIES PREFIX "" OUTPUT_NAME "_caffe") -target_include_directories(pycaffe PUBLIC ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) -target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${PYTHON_LIBRARIES}) +target_include_directories(pycaffe PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${Python_LIBRARIES}) if(UNIX OR APPLE) set(__linkname "${PROJECT_SOURCE_DIR}/python/caffe/_caffe.so") diff --git a/python/caffe/draw.py b/python/caffe/draw.py index 0061f490bd9..73ae04abaa8 100644 --- a/python/caffe/draw.py +++ b/python/caffe/draw.py @@ -11,15 +11,7 @@ from caffe.proto import caffe_pb2 -""" -pydot is not supported under python 3 and pydot2 doesn't work properly. -pydotplus works nicely (pip install pydotplus) -""" -try: - # Try to load pydotplus - import pydotplus as pydot -except ImportError: - import pydot +import pydot # Internal layer and blob styles. LAYER_STYLE_DEFAULT = {'shape': 'record', diff --git a/python/caffe/test/test_coord_map.py b/python/caffe/test/test_coord_map.py index 613260e25df..7df1694020d 100644 --- a/python/caffe/test/test_coord_map.py +++ b/python/caffe/test/test_coord_map.py @@ -42,15 +42,15 @@ def test_conv_pool_deconv(self): n = coord_net_spec() # identity for 2x pool, 2x deconv ax, a, b = coord_map_from_to(n.deconv, n.data) - self.assertEquals(ax, 1) - self.assertEquals(a, 1) - self.assertEquals(b, 0) + self.assertEqual(ax, 1) + self.assertEqual(a, 1) + self.assertEqual(b, 0) # shift-by-one for 4x pool, 4x deconv n = coord_net_spec(pool=4, dstride=4) ax, a, b = coord_map_from_to(n.deconv, n.data) - self.assertEquals(ax, 1) - self.assertEquals(a, 1) - self.assertEquals(b, -1) + self.assertEqual(ax, 1) + self.assertEqual(a, 1) + self.assertEqual(b, -1) def test_pass(self): """ @@ -64,9 +64,9 @@ def test_pass(self): n.relu, num_output=10, kernel_size=1, stride=1, pad=0) for top in [n.relu, n.conv1x1]: ax_pass, a_pass, b_pass = coord_map_from_to(top, n.data) - self.assertEquals(ax, ax_pass) - self.assertEquals(a, a_pass) - self.assertEquals(b, b_pass) + self.assertEqual(ax, ax_pass) + self.assertEqual(a, a_pass) + self.assertEqual(b, b_pass) def test_padding(self): """ @@ -78,18 +78,18 @@ def test_padding(self): # conv padding n = coord_net_spec(pad=pad) _, a_pad, b_pad = coord_map_from_to(n.deconv, n.data) - self.assertEquals(a, a_pad) - self.assertEquals(b - pad, b_pad) + self.assertEqual(a, a_pad) + self.assertEqual(b - pad, b_pad) # deconv padding n = coord_net_spec(dpad=pad) _, a_pad, b_pad = coord_map_from_to(n.deconv, n.data) - self.assertEquals(a, a_pad) - self.assertEquals(b + pad, b_pad) + self.assertEqual(a, a_pad) + self.assertEqual(b + pad, b_pad) # pad both to cancel out n = coord_net_spec(pad=pad, dpad=pad) _, a_pad, b_pad = coord_map_from_to(n.deconv, n.data) - self.assertEquals(a, a_pad) - self.assertEquals(b, b_pad) + self.assertEqual(a, a_pad) + self.assertEqual(b, b_pad) def test_multi_conv(self): """ @@ -102,9 +102,9 @@ def test_multi_conv(self): pad=0) ax1, a1, b1 = coord_map_from_to(n.conv_data, n.data) ax2, a2, b2 = coord_map_from_to(n.conv_aux, n.aux) - self.assertEquals(ax1, ax2) - self.assertEquals(a1, a2) - self.assertEquals(b1, b2) + self.assertEqual(ax1, ax2) + self.assertEqual(a1, a2) + self.assertEqual(b1, b2) def test_rect(self): """ @@ -117,10 +117,10 @@ def test_rect(self): ax_5x5, a_5x5, b_5x5 = coord_map_from_to(n5x5.deconv, n5x5.data) ax_3x5, a_3x5, b_3x5 = coord_map_from_to(n3x5.deconv, n3x5.data) self.assertTrue(ax_3x3 == ax_5x5 == ax_3x5) - self.assertEquals(a_3x3, a_3x5[0]) - self.assertEquals(b_3x3, b_3x5[0]) - self.assertEquals(a_5x5, a_3x5[1]) - self.assertEquals(b_5x5, b_3x5[1]) + self.assertEqual(a_3x3, a_3x5[0]) + self.assertEqual(b_3x3, b_3x5[0]) + self.assertEqual(a_5x5, a_3x5[1]) + self.assertEqual(b_5x5, b_3x5[1]) def test_nd_conv(self): """ @@ -137,11 +137,11 @@ def test_nd_conv(self): n.deconv = L.Deconvolution( n.pool, num_output=10, kernel_size=4, stride=2, pad=0) ax, a, b = coord_map_from_to(n.deconv, n.data) - self.assertEquals(ax, 1) + self.assertEqual(ax, 1) self.assertTrue(len(a) == len(b)) self.assertTrue(np.all(a == 1)) - self.assertEquals(b[0] - 1, b[1]) - self.assertEquals(b[1] - 1, b[2]) + self.assertEqual(b[0] - 1, b[1]) + self.assertEqual(b[1] - 1, b[2]) def test_crop_of_crop(self): """ @@ -153,9 +153,9 @@ def test_crop_of_crop(self): ax, a, b = coord_map_from_to(n.deconv, n.data) n.crop = L.Crop(n.deconv, n.data, axis=2, offset=offset) ax_crop, a_crop, b_crop = coord_map_from_to(n.crop, n.data) - self.assertEquals(ax, ax_crop) - self.assertEquals(a, a_crop) - self.assertEquals(b + offset, b_crop) + self.assertEqual(ax, ax_crop) + self.assertEqual(a, a_crop) + self.assertEqual(b + offset, b_crop) def test_crop_helper(self): """ diff --git a/python/caffe/test/test_net_spec.py b/python/caffe/test/test_net_spec.py index ffe71bacb08..20871985b3e 100644 --- a/python/caffe/test/test_net_spec.py +++ b/python/caffe/test/test_net_spec.py @@ -84,6 +84,6 @@ def test_type_error(self): """Test that a TypeError is raised when a Function input isn't a Top.""" data = L.DummyData(ntop=2) # data is a 2-tuple of Tops r = r"^Silence input 0 is not a Top \(type is <(type|class) 'tuple'>\)$" - with self.assertRaisesRegexp(TypeError, r): + with self.assertRaisesRegex(TypeError, r): L.Silence(data, ntop=0) # should raise: data is a tuple, not a Top L.Silence(*data, ntop=0) # shouldn't raise: each elt of data is a Top diff --git a/scripts/travis/build.sh b/scripts/ci/build.sh similarity index 54% rename from scripts/travis/build.sh rename to scripts/ci/build.sh index bb9406f046c..4748ee12648 100755 --- a/scripts/travis/build.sh +++ b/scripts/ci/build.sh @@ -5,9 +5,12 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh if ! $WITH_CMAKE ; then + echo -e "\e[35m\e[1mmake --jobs $NUM_THREADS all test pycaffe warn\e[0m" make --jobs $NUM_THREADS all test pycaffe warn else + echo -e "\e[35m\e[1mcd build; make --jobs $NUM_THREADS all test.testbin\e[0m" cd build make --jobs $NUM_THREADS all test.testbin fi +echo -e "\e[35m\e[1mmake lint\e[0m" make lint diff --git a/scripts/travis/configure-cmake.sh b/scripts/ci/configure-cmake.sh similarity index 85% rename from scripts/travis/configure-cmake.sh rename to scripts/ci/configure-cmake.sh index 772f1e2ce8d..cd34bdd2b33 100644 --- a/scripts/travis/configure-cmake.sh +++ b/scripts/ci/configure-cmake.sh @@ -1,14 +1,11 @@ # CMake configuration +echo -e "\e[35m\e[1mmkdir -p build; cd build\e[0m" mkdir -p build cd build ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open" -if $WITH_PYTHON3 ; then - ARGS="$ARGS -Dpython_version=3" -fi - if $WITH_IO ; then ARGS="$ARGS -DUSE_OPENCV=On -DUSE_LMDB=On -DUSE_LEVELDB=On" else @@ -28,5 +25,5 @@ else ARGS="$ARGS -DUSE_CUDNN=Off" fi +echo -e "\e[35m\e[1mcmake .. ${ARGS}\e[0m" cmake .. $ARGS - diff --git a/scripts/travis/configure-make.sh b/scripts/ci/configure-make.sh similarity index 52% rename from scripts/travis/configure-make.sh rename to scripts/ci/configure-make.sh index ddc40fffa9d..d91ccf459e4 100644 --- a/scripts/travis/configure-make.sh +++ b/scripts/ci/configure-make.sh @@ -1,21 +1,21 @@ # raw Makefile configuration LINE () { + echo -e "\e[35m\e[1m$@\e[0m" echo "$@" >> Makefile.config } +echo -e "\e[35m\e[1mcp Makefile.config.example Makefile.config\e[0m" cp Makefile.config.example Makefile.config LINE "BLAS := open" LINE "WITH_PYTHON_LAYER := 1" -if $WITH_PYTHON3 ; then - # TODO(lukeyeager) this path is currently disabled because of test errors like: - # ImportError: dynamic module does not define init function (PyInit__caffe) - LINE "PYTHON_LIBRARIES := python3.4m boost_python-py34" - LINE "PYTHON_INCLUDE := /usr/include/python3.4 /usr/lib/python3/dist-packages/numpy/core/include" - LINE "INCLUDE_DIRS := \$(INCLUDE_DIRS) \$(PYTHON_INCLUDE)" -fi +# TODO(lukeyeager) this path is currently disabled because of test errors like: +# ImportError: dynamic module does not define init function (PyInit__caffe) +# LINE "PYTHON_LIBRARIES := python3.8m boost_python-py38" +# LINE "PYTHON_INCLUDE := /usr/include/python3.8 /usr/lib/python3/dist-packages/numpy/core/include" +# LINE "INCLUDE_DIRS := \$(INCLUDE_DIRS) \$(PYTHON_INCLUDE)" if ! $WITH_IO ; then LINE "USE_OPENCV := 0" diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh new file mode 100755 index 00000000000..6990a48e32e --- /dev/null +++ b/scripts/ci/configure.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# configure the project + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +echo -e "\e[35m\e[1mWITH_CMAKE: ${WITH_CMAKE}\e[0m" +echo -e "\e[35m\e[1mWITH_IO: ${WITH_IO}\e[0m" +echo -e "\e[35m\e[1mWITH_CUDA: ${WITH_CUDA}\e[0m" +echo -e "\e[35m\e[1mWITH_CUDNN: ${WITH_CUDNN}\e[0m" + +if ! $WITH_CMAKE ; then + source $BASEDIR/configure-make.sh +else + source $BASEDIR/configure-cmake.sh +fi diff --git a/scripts/travis/defaults.sh b/scripts/ci/defaults.sh similarity index 83% rename from scripts/travis/defaults.sh rename to scripts/ci/defaults.sh index d69c0a7d964..f7812d004c9 100755 --- a/scripts/travis/defaults.sh +++ b/scripts/ci/defaults.sh @@ -4,7 +4,6 @@ set -e WITH_CMAKE=${WITH_CMAKE:-false} -WITH_PYTHON3=${WITH_PYTHON3:-false} WITH_IO=${WITH_IO:-true} WITH_CUDA=${WITH_CUDA:-false} WITH_CUDNN=${WITH_CUDNN:-false} diff --git a/scripts/ci/install-deps.sh b/scripts/ci/install-deps.sh new file mode 100755 index 00000000000..db9a24276fa --- /dev/null +++ b/scripts/ci/install-deps.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# install dependencies +# (this script must be run as root) + +BASEDIR=$(dirname $0) +source $BASEDIR/defaults.sh + +apt-get -yq update +apt-get install -yq --no-install-recommends \ + build-essential \ + graphviz \ + libboost-filesystem-dev \ + libboost-python-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libhdf5-serial-dev \ + libopenblas-dev \ + libprotobuf-dev \ + protobuf-compiler \ + python3-virtualenv \ + wget + +if $WITH_CMAKE ; then + apt-get install -yq --no-install-recommends cmake +fi + +# Python3 +apt-get install -yq --no-install-recommends \ + python3-dev \ + python3-numpy \ + python3-skimage + +if $WITH_IO ; then + apt-get install -yq --no-install-recommends \ + libleveldb-dev \ + liblmdb-dev \ + libopencv-dev \ + libsnappy-dev +fi + +if $WITH_CUDA ; then + # install repo packages + CUDA_REPO_PKG=cuda-repo-ubuntu1404_7.5-18_amd64.deb + wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/$CUDA_REPO_PKG + dpkg -i $CUDA_REPO_PKG + rm $CUDA_REPO_PKG + + if $WITH_CUDNN ; then + ML_REPO_PKG=nvidia-machine-learning-repo-ubuntu1404_4.0-2_amd64.deb + wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/$ML_REPO_PKG + dpkg -i $ML_REPO_PKG + fi + + # update package lists + apt-get -yq update + + # install packages + CUDA_PKG_VERSION="7-5" + CUDA_VERSION="7.5" + apt-get install -y --no-install-recommends \ + cuda-core-$CUDA_PKG_VERSION \ + cuda-cudart-dev-$CUDA_PKG_VERSION \ + cuda-cublas-dev-$CUDA_PKG_VERSION \ + cuda-curand-dev-$CUDA_PKG_VERSION + # manually create CUDA symlink + ln -s /usr/local/cuda-$CUDA_VERSION /usr/local/cuda + + if $WITH_CUDNN ; then + apt-get install -yq --no-install-recommends libcudnn7-dev + fi +fi + diff --git a/scripts/travis/install-python-deps.sh b/scripts/ci/install-python-deps.sh similarity index 52% rename from scripts/travis/install-python-deps.sh rename to scripts/ci/install-python-deps.sh index 910d35a93be..2d450d11a79 100755 --- a/scripts/travis/install-python-deps.sh +++ b/scripts/ci/install-python-deps.sh @@ -5,11 +5,6 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh -if ! $WITH_PYTHON3 ; then - # Python2 - : -else - # Python3 - pip install --pre protobuf==3.0.0b3 - pip install pydot -fi +# Python3 +pip install \ + pydot diff --git a/scripts/travis/setup-venv.sh b/scripts/ci/setup-venv.sh similarity index 60% rename from scripts/travis/setup-venv.sh rename to scripts/ci/setup-venv.sh index 81245f146da..1d757cf1de0 100755 --- a/scripts/travis/setup-venv.sh +++ b/scripts/ci/setup-venv.sh @@ -8,11 +8,7 @@ source $BASEDIR/defaults.sh VENV_DIR=${1:-~/venv} # setup our own virtualenv -if $WITH_PYTHON3; then - PYTHON_EXE='/usr/bin/python3' -else - PYTHON_EXE='/usr/bin/python2' -fi +PYTHON_EXE='/usr/bin/python3' # use --system-site-packages so that Python will use deb packages -virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages +python3 -m virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages diff --git a/scripts/travis/test.sh b/scripts/ci/test.sh similarity index 100% rename from scripts/travis/test.sh rename to scripts/ci/test.sh diff --git a/scripts/travis/configure.sh b/scripts/travis/configure.sh deleted file mode 100755 index ef740c8982e..00000000000 --- a/scripts/travis/configure.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# configure the project - -BASEDIR=$(dirname $0) -source $BASEDIR/defaults.sh - -if ! $WITH_CMAKE ; then - source $BASEDIR/configure-make.sh -else - source $BASEDIR/configure-cmake.sh -fi diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh deleted file mode 100755 index abf9cf1ca70..00000000000 --- a/scripts/travis/install-deps.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# install dependencies -# (this script must be run as root) - -BASEDIR=$(dirname $0) -source $BASEDIR/defaults.sh - -apt-get -y update -apt-get install -y --no-install-recommends \ - build-essential \ - graphviz \ - libboost-filesystem-dev \ - libboost-python-dev \ - libboost-system-dev \ - libboost-thread-dev \ - libgflags-dev \ - libgoogle-glog-dev \ - libhdf5-serial-dev \ - libopenblas-dev \ - python-virtualenv \ - wget - -if $WITH_CMAKE ; then - apt-get install -y --no-install-recommends cmake -fi - -if ! $WITH_PYTHON3 ; then - # Python2 - apt-get install -y --no-install-recommends \ - libprotobuf-dev \ - protobuf-compiler \ - python-dev \ - python-numpy \ - python-protobuf \ - python-pydot \ - python-skimage -else - # Python3 - apt-get install -y --no-install-recommends \ - python3-dev \ - python3-numpy \ - python3-skimage - - # build Protobuf3 since it's needed for Python3 - PROTOBUF3_DIR=~/protobuf3 - pushd . - if [ -d "$PROTOBUF3_DIR" ] && [ -e "$PROTOBUF3_DIR/src/protoc" ]; then - echo "Using cached protobuf3 build ..." - cd $PROTOBUF3_DIR - else - echo "Building protobuf3 from source ..." - rm -rf $PROTOBUF3_DIR - mkdir $PROTOBUF3_DIR - - # install some more dependencies required to build protobuf3 - apt-get install -y --no-install-recommends \ - curl \ - dh-autoreconf \ - unzip - - wget https://github.com/google/protobuf/archive/3.0.x.tar.gz -O protobuf3.tar.gz - tar -xzf protobuf3.tar.gz -C $PROTOBUF3_DIR --strip 1 - rm protobuf3.tar.gz - cd $PROTOBUF3_DIR - ./autogen.sh - ./configure --prefix=/usr - make --jobs=$NUM_THREADS - fi - make install - popd -fi - -if $WITH_IO ; then - apt-get install -y --no-install-recommends \ - libleveldb-dev \ - liblmdb-dev \ - libopencv-dev \ - libsnappy-dev -fi - -if $WITH_CUDA ; then - # install repo packages - CUDA_REPO_PKG=cuda-repo-ubuntu1404_7.5-18_amd64.deb - wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/$CUDA_REPO_PKG - dpkg -i $CUDA_REPO_PKG - rm $CUDA_REPO_PKG - - if $WITH_CUDNN ; then - ML_REPO_PKG=nvidia-machine-learning-repo-ubuntu1404_4.0-2_amd64.deb - wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/$ML_REPO_PKG - dpkg -i $ML_REPO_PKG - fi - - # update package lists - apt-get -y update - - # install packages - CUDA_PKG_VERSION="7-5" - CUDA_VERSION="7.5" - apt-get install -y --no-install-recommends \ - cuda-core-$CUDA_PKG_VERSION \ - cuda-cudart-dev-$CUDA_PKG_VERSION \ - cuda-cublas-dev-$CUDA_PKG_VERSION \ - cuda-curand-dev-$CUDA_PKG_VERSION - # manually create CUDA symlink - ln -s /usr/local/cuda-$CUDA_VERSION /usr/local/cuda - - if $WITH_CUDNN ; then - apt-get install -y --no-install-recommends libcudnn7-dev - fi -fi - diff --git a/src/caffe/layers/cudnn_conv_layer.cpp b/src/caffe/layers/cudnn_conv_layer.cpp index a4a69b3ac6a..a1812c17569 100644 --- a/src/caffe/layers/cudnn_conv_layer.cpp +++ b/src/caffe/layers/cudnn_conv_layer.cpp @@ -116,7 +116,7 @@ void CuDNNConvolutionLayer::Reshape( cudnnConvolutionFwdAlgoPerf_t fwd_algo_pref_[4]; cudnnConvolutionBwdDataAlgoPerf_t bwd_data_algo_pref_[4]; - //get memory sizes + // get memory sizes cudaMemGetInfo(&free_memory, &total_memory); #else // Specify workspace limit for kernels directly until we have a @@ -142,46 +142,63 @@ void CuDNNConvolutionLayer::Reshape( // Note: Copied from https://github.com/Qengineering/caffe/tree/ssd/src/caffe/layers #if CUDNN_VERSION_MIN(8, 0, 0) // choose forward algorithm for filter - // in forward filter the CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED is not implemented in cuDNN 8 - CUDNN_CHECK(cudnnGetConvolutionForwardAlgorithm_v7(handle_[0], bottom_descs_[i], filter_desc_, conv_descs_[i], top_descs_[i], 4, &RetCnt, fwd_algo_pref_)); + // in forward filter the CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED is not + // implemented in cuDNN 8 + CUDNN_CHECK(cudnnGetConvolutionForwardAlgorithm_v7(handle_[0], + bottom_descs_[i], + filter_desc_, + conv_descs_[i], + top_descs_[i], + 4, + &RetCnt, + fwd_algo_pref_)); found_conv_algorithm = false; - for(int n=0;n::Reshape( // // We have found that CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM is // // buggy. Thus, if this algo was chosen, choose winograd instead. If - // // winograd is not supported or workspace is larger than threshold, choose + // // winograd is not supported or workspace is larger than threshold, choose // NOLINT(whitespace/line_length) // // implicit_gemm instead. // if (fwd_algo_[i] == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM) { // size_t winograd_workspace_size; @@ -226,7 +226,7 @@ void CuDNNDeconvolutionLayer::Reshape( // &workspace_bwd_data_sizes_[i])); // } - // // reduce over all workspace sizes to get a maximum to allocate / reallocate + // // reduce over all workspace sizes to get a maximum to allocate / reallocate // NOLINT(whitespace/line_length) // size_t total_workspace_fwd = 0; // size_t total_workspace_bwd_data = 0; // size_t total_workspace_bwd_filter = 0; @@ -255,7 +255,7 @@ void CuDNNDeconvolutionLayer::Reshape( // // free the existing workspace and allocate a new (larger) one // cudaFree(this->workspaceData); - // cudaError_t err = cudaMalloc(&(this->workspaceData), workspaceSizeInBytes); + // cudaError_t err = cudaMalloc(&(this->workspaceData), workspaceSizeInBytes); // NOLINT(whitespace/line_length) // if (err != cudaSuccess) { // // force zero memory path // for (int i = 0; i < bottom.size(); i++) { @@ -278,7 +278,7 @@ void CuDNNDeconvolutionLayer::Reshape( // // if we succeed in the allocation, set pointer aliases for workspaces // for (int g = 0; g < (this->group_ * CUDNN_STREAMS_PER_GROUP); g++) { - // workspace[g] = reinterpret_cast(workspaceData) + g*max_workspace; + // workspace[g] = reinterpret_cast(workspaceData) + g*max_workspace; // NOLINT(whitespace/line_length) // } // } diff --git a/src/caffe/layers/window_data_layer.cpp b/src/caffe/layers/window_data_layer.cpp index 1bf3760e9fd..f41169debe4 100644 --- a/src/caffe/layers/window_data_layer.cpp +++ b/src/caffe/layers/window_data_layer.cpp @@ -290,7 +290,7 @@ void WindowDataLayer::load_batch(Batch* batch) { image_database_cache_[window[WindowDataLayer::IMAGE_INDEX]]; cv_img = DecodeDatumToCVMat(image_cached.second, true); } else { - cv_img = cv::imread(image.first, CV_LOAD_IMAGE_COLOR); + cv_img = cv::imread(image.first, cv::IMREAD_COLOR); if (!cv_img.data) { LOG(ERROR) << "Could not open or find file " << image.first; return; diff --git a/src/caffe/test/test_io.cpp b/src/caffe/test/test_io.cpp index c2c919e90dc..b80df287fba 100644 --- a/src/caffe/test/test_io.cpp +++ b/src/caffe/test/test_io.cpp @@ -20,8 +20,8 @@ class IOTest : public ::testing::Test {}; bool ReadImageToDatumReference(const string& filename, const int label, const int height, const int width, const bool is_color, Datum* datum) { cv::Mat cv_img; - int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : - CV_LOAD_IMAGE_GRAYSCALE); + int cv_read_flag = (is_color ? cv::IMREAD_COLOR : + cv::IMREAD_GRAYSCALE); cv::Mat cv_img_origin = cv::imread(filename, cv_read_flag); if (!cv_img_origin.data) { diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 5295d9dddb9..8517396b76c 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -54,7 +54,13 @@ bool ReadProtoFromBinaryFile(const char* filename, Message* proto) { CHECK_NE(fd, -1) << "File not found: " << filename; ZeroCopyInputStream* raw_input = new FileInputStream(fd); CodedInputStream* coded_input = new CodedInputStream(raw_input); +#if GOOGLE_PROTOBUF_VERSION >= 3011000 + // Only take one parameter since protobuf 3.11 + coded_input->SetTotalBytesLimit(kProtoReadBytesLimit); +#else + // Total bytes hard limit / warning limit are set coded_input->SetTotalBytesLimit(kProtoReadBytesLimit, 536870912); +#endif bool success = proto->ParseFromCodedStream(coded_input); @@ -73,8 +79,8 @@ void WriteProtoToBinaryFile(const Message& proto, const char* filename) { cv::Mat ReadImageToCVMat(const string& filename, const int height, const int width, const bool is_color) { cv::Mat cv_img; - int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : - CV_LOAD_IMAGE_GRAYSCALE); + int cv_read_flag = (is_color ? cv::IMREAD_COLOR : + cv::IMREAD_GRAYSCALE); cv::Mat cv_img_origin = cv::imread(filename, cv_read_flag); if (!cv_img_origin.data) { LOG(ERROR) << "Could not open or find file " << filename; @@ -179,8 +185,8 @@ cv::Mat DecodeDatumToCVMat(const Datum& datum, bool is_color) { CHECK(datum.encoded()) << "Datum not encoded"; const string& data = datum.data(); std::vector vec_data(data.c_str(), data.c_str() + data.size()); - int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : - CV_LOAD_IMAGE_GRAYSCALE); + int cv_read_flag = (is_color ? cv::IMREAD_COLOR : + cv::IMREAD_GRAYSCALE); cv_img = cv::imdecode(vec_data, cv_read_flag); if (!cv_img.data) { LOG(ERROR) << "Could not decode datum ";