From 73ab80b50759f8631f4f4c198782fbc1233cd362 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 10:53:25 +0100 Subject: [PATCH 01/46] Add compatibility for protobuf >= 3.11 --- src/caffe/util/io.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 5295d9dddb9..485ff962134 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -53,8 +53,14 @@ bool ReadProtoFromBinaryFile(const char* filename, Message* proto) { int fd = open(filename, O_RDONLY); CHECK_NE(fd, -1) << "File not found: " << filename; ZeroCopyInputStream* raw_input = new FileInputStream(fd); - CodedInputStream* coded_input = new CodedInputStream(raw_input); + CodedInputStream* coded_input = new CodedInputStream(raw_input); +#if GOOGLE_PROTOBUF_VERSION >= 3011000 + // Only take one parameter since protobuf 3.11 + coded_input->SetTotalBytesLimit(kProtoReadBytesLimit); +#else + // Total bytes hard limit / warning limit are set coded_input->SetTotalBytesLimit(kProtoReadBytesLimit, 536870912); +#endif bool success = proto->ParseFromCodedStream(coded_input); From a47bf0711d28e2be157b8ec94bd2f7cb713e0bc2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:02:46 +0100 Subject: [PATCH 02/46] (actions) add GH actions --- .github/workflows/main.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000..afe2910fc1f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +name: CI + +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + name: ${{ matrix.build_name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # Use a build matrix to test many builds in parallel + # envvar defaults: + # WITH_CMAKE: false + # WITH_PYTHON3: false + # WITH_IO: true + # WITH_CUDA: false + # WITH_CUDNN: false + - BUILD_NAME="default-make" + # - BUILD_NAME="python3-make" + # WITH_PYTHON3=true + - BUILD_NAME="no-io-make" + WITH_IO=false + - BUILD_NAME="cuda-make" + WITH_CUDA=true + - BUILD_NAME="cudnn-make" + WITH_CUDA=true + WITH_CUDNN=true + - BUILD_NAME="default-cmake" + WITH_CMAKE=true + - BUILD_NAME="python3-cmake" + WITH_CMAKE=true WITH_PYTHON3=true + - BUILD_NAME="no-io-cmake" + WITH_CMAKE=true + WITH_IO=false + - BUILD_NAME="cuda-cmake" + WITH_CMAKE=true + WITH_CUDA=true + - BUILD_NAME="cudnn-cmake" + WITH_CMAKE=true + WITH_CUDA=true + WITH_CUDNN=true + env: + NUM_THREADS: 4 + steps: + - name: Defaults + run: | + source ./scripts/travis/defaults.sh + - name: Install + run: | + sudo -E ./scripts/travis/install-deps.sh + ./scripts/travis/setup-venv.sh ~/venv + source ~/venv/bin/activate + ./scripts/travis/install-python-deps.sh + - name: Configure + run: | + ./scripts/travis/configure.sh + - name: Build + run: | + ./scripts/travis/build.sh + - name: Test + run: | + ./scripts/travis/test.sh From 8ea31b66c82c49b171cf4398929e46ddd0606dff Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:05:53 +0100 Subject: [PATCH 03/46] (actions) fix indent --- .github/workflows/main.yml | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index afe2910fc1f..9c12e14fbeb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,24 +41,24 @@ jobs: WITH_CMAKE=true WITH_CUDA=true WITH_CUDNN=true - env: - NUM_THREADS: 4 - steps: - - name: Defaults - run: | - source ./scripts/travis/defaults.sh - - name: Install - run: | - sudo -E ./scripts/travis/install-deps.sh - ./scripts/travis/setup-venv.sh ~/venv - source ~/venv/bin/activate - ./scripts/travis/install-python-deps.sh - - name: Configure - run: | - ./scripts/travis/configure.sh - - name: Build - run: | - ./scripts/travis/build.sh - - name: Test - run: | - ./scripts/travis/test.sh + env: + NUM_THREADS: 4 + steps: + - name: Defaults + run: | + source ./scripts/travis/defaults.sh + - name: Install + run: | + sudo -E ./scripts/travis/install-deps.sh + ./scripts/travis/setup-venv.sh ~/venv + source ~/venv/bin/activate + ./scripts/travis/install-python-deps.sh + - name: Configure + run: | + ./scripts/travis/configure.sh + - name: Build + run: | + ./scripts/travis/build.sh + - name: Test + run: | + ./scripts/travis/test.sh From bc03fdd2b41b24428eb1610b40015315047a9edc Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:11:27 +0100 Subject: [PATCH 04/46] (actions) fix syntax --- .github/workflows/main.yml | 54 ++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c12e14fbeb..dfd27adace4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,37 +10,29 @@ jobs: fail-fast: false matrix: include: - # Use a build matrix to test many builds in parallel - # envvar defaults: - # WITH_CMAKE: false - # WITH_PYTHON3: false - # WITH_IO: true - # WITH_CUDA: false - # WITH_CUDNN: false - - BUILD_NAME="default-make" - # - BUILD_NAME="python3-make" - # WITH_PYTHON3=true - - BUILD_NAME="no-io-make" - WITH_IO=false - - BUILD_NAME="cuda-make" - WITH_CUDA=true - - BUILD_NAME="cudnn-make" - WITH_CUDA=true - WITH_CUDNN=true - - BUILD_NAME="default-cmake" - WITH_CMAKE=true - - BUILD_NAME="python3-cmake" - WITH_CMAKE=true WITH_PYTHON3=true - - BUILD_NAME="no-io-cmake" - WITH_CMAKE=true - WITH_IO=false - - BUILD_NAME="cuda-cmake" - WITH_CMAKE=true - WITH_CUDA=true - - BUILD_NAME="cudnn-cmake" - WITH_CMAKE=true - WITH_CUDA=true - WITH_CUDNN=true + - BUILD_NAME: default-make + - BUILD_NAME: no-io-make + WITH_IO: false + - BUILD_NAME: cuda-make + WITH_CUDA: true + - BUILD_NAME: cudnn-make + WITH_CUDA: true + WITH_CUDNN: true + - BUILD_NAME: default-cmake + WITH_CMAKE: true + - BUILD_NAME: python3-cmake + WITH_CMAKE: true + WITH_PYTHON3: true + - BUILD_NAME: no-io-cmake + WITH_CMAKE: true + WITH_IO: false + - BUILD_NAME: cuda-cmake + WITH_CMAKE: true + WITH_CUDA: true + - BUILD_NAME: cudnn-cmake + WITH_CMAKE: true + WITH_CUDA: true + WITH_CUDNN: true env: NUM_THREADS: 4 steps: From d7ac6e482edf5b6344a5fe4a25f6243593952d7e Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:12:21 +0100 Subject: [PATCH 05/46] (actions) restore commented include --- .github/workflows/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfd27adace4..143974916ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,7 +10,16 @@ jobs: fail-fast: false matrix: include: + # Use a build matrix to test many builds in parallel + # envvar defaults: + # WITH_CMAKE: false + # WITH_PYTHON3: false + # WITH_IO: true + # WITH_CUDA: false + # WITH_CUDNN: false - BUILD_NAME: default-make + # - BUILD_NAME: python3-make + # WITH_PYTHON3: true - BUILD_NAME: no-io-make WITH_IO: false - BUILD_NAME: cuda-make From e8910026890786047094fa4c473a5bba47ace9d6 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:17:13 +0100 Subject: [PATCH 06/46] (actions) need to checkout the repo --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 143974916ca..9f7fb6cf104 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,6 +45,7 @@ jobs: env: NUM_THREADS: 4 steps: + - uses: actions/checkout@v3 - name: Defaults run: | source ./scripts/travis/defaults.sh From e792776696b2e9949ed63c1e0ee5174488ea77f3 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:18:17 +0100 Subject: [PATCH 07/46] (depedabot) update GH actions --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..5ace4600a1f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From 8aff18dabd39f571dd8a905caf76a02929af5db8 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:20:59 +0100 Subject: [PATCH 08/46] (CI) Drop python3 support --- scripts/travis/install-deps.sh | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh index abf9cf1ca70..a5f85ad1098 100755 --- a/scripts/travis/install-deps.sh +++ b/scripts/travis/install-deps.sh @@ -17,29 +17,18 @@ apt-get install -y --no-install-recommends \ libgoogle-glog-dev \ libhdf5-serial-dev \ libopenblas-dev \ - python-virtualenv \ + python3-virtualenv \ wget if $WITH_CMAKE ; then apt-get install -y --no-install-recommends cmake fi -if ! $WITH_PYTHON3 ; then - # Python2 - apt-get install -y --no-install-recommends \ - libprotobuf-dev \ - protobuf-compiler \ - python-dev \ - python-numpy \ - python-protobuf \ - python-pydot \ - python-skimage -else - # Python3 - apt-get install -y --no-install-recommends \ - python3-dev \ - python3-numpy \ - python3-skimage +# Python3 +apt-get install -y --no-install-recommends \ + python3-dev \ + python3-numpy \ + python3-skimage # build Protobuf3 since it's needed for Python3 PROTOBUF3_DIR=~/protobuf3 From dd5a41b9de8d788fe8d56440b9c6e6b010790402 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:21:54 +0100 Subject: [PATCH 09/46] (actions) drop python3 support --- .github/workflows/main.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f7fb6cf104..fbbb5e92fbe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,13 +13,10 @@ jobs: # Use a build matrix to test many builds in parallel # envvar defaults: # WITH_CMAKE: false - # WITH_PYTHON3: false # WITH_IO: true # WITH_CUDA: false # WITH_CUDNN: false - BUILD_NAME: default-make - # - BUILD_NAME: python3-make - # WITH_PYTHON3: true - BUILD_NAME: no-io-make WITH_IO: false - BUILD_NAME: cuda-make @@ -29,9 +26,6 @@ jobs: WITH_CUDNN: true - BUILD_NAME: default-cmake WITH_CMAKE: true - - BUILD_NAME: python3-cmake - WITH_CMAKE: true - WITH_PYTHON3: true - BUILD_NAME: no-io-cmake WITH_CMAKE: true WITH_IO: false From a7a85d4dfdf95233e6fc0df5547066efc07a9499 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:23:03 +0100 Subject: [PATCH 10/46] (CI) drop python2 support --- scripts/travis/configure-cmake.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/travis/configure-cmake.sh b/scripts/travis/configure-cmake.sh index 772f1e2ce8d..d21c8b3af69 100644 --- a/scripts/travis/configure-cmake.sh +++ b/scripts/travis/configure-cmake.sh @@ -3,11 +3,7 @@ mkdir -p build cd build -ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open" - -if $WITH_PYTHON3 ; then - ARGS="$ARGS -Dpython_version=3" -fi +ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open -Dpython_version=3" if $WITH_IO ; then ARGS="$ARGS -DUSE_OPENCV=On -DUSE_LMDB=On -DUSE_LEVELDB=On" @@ -29,4 +25,3 @@ else fi cmake .. $ARGS - From 43d007ef71da7f8807d4ff51f74179199a164103 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:24:10 +0100 Subject: [PATCH 11/46] (CI) drop python2 support --- scripts/travis/configure-make.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/travis/configure-make.sh b/scripts/travis/configure-make.sh index ddc40fffa9d..ae2aa717aba 100644 --- a/scripts/travis/configure-make.sh +++ b/scripts/travis/configure-make.sh @@ -9,13 +9,11 @@ cp Makefile.config.example Makefile.config LINE "BLAS := open" LINE "WITH_PYTHON_LAYER := 1" -if $WITH_PYTHON3 ; then - # TODO(lukeyeager) this path is currently disabled because of test errors like: - # ImportError: dynamic module does not define init function (PyInit__caffe) - LINE "PYTHON_LIBRARIES := python3.4m boost_python-py34" - LINE "PYTHON_INCLUDE := /usr/include/python3.4 /usr/lib/python3/dist-packages/numpy/core/include" - LINE "INCLUDE_DIRS := \$(INCLUDE_DIRS) \$(PYTHON_INCLUDE)" -fi +# TODO(lukeyeager) this path is currently disabled because of test errors like: +# ImportError: dynamic module does not define init function (PyInit__caffe) +# LINE "PYTHON_LIBRARIES := python3.8m boost_python-py38" +# LINE "PYTHON_INCLUDE := /usr/include/python3.8 /usr/lib/python3/dist-packages/numpy/core/include" +# LINE "INCLUDE_DIRS := \$(INCLUDE_DIRS) \$(PYTHON_INCLUDE)" if ! $WITH_IO ; then LINE "USE_OPENCV := 0" From 6834715cd629aba0930e0c1ba40b1398cc8b8a30 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:24:38 +0100 Subject: [PATCH 12/46] (CI)(defaults) drop python2 support --- scripts/travis/defaults.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/travis/defaults.sh b/scripts/travis/defaults.sh index d69c0a7d964..f7812d004c9 100755 --- a/scripts/travis/defaults.sh +++ b/scripts/travis/defaults.sh @@ -4,7 +4,6 @@ set -e WITH_CMAKE=${WITH_CMAKE:-false} -WITH_PYTHON3=${WITH_PYTHON3:-false} WITH_IO=${WITH_IO:-true} WITH_CUDA=${WITH_CUDA:-false} WITH_CUDNN=${WITH_CUDNN:-false} From 4740e07fc464d330d3f9cb6023bb58a60006419b Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:25:24 +0100 Subject: [PATCH 13/46] (CI)(python) drop python2 support --- scripts/travis/install-python-deps.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/travis/install-python-deps.sh b/scripts/travis/install-python-deps.sh index 910d35a93be..54ba2a17223 100755 --- a/scripts/travis/install-python-deps.sh +++ b/scripts/travis/install-python-deps.sh @@ -5,11 +5,6 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh -if ! $WITH_PYTHON3 ; then - # Python2 - : -else - # Python3 - pip install --pre protobuf==3.0.0b3 - pip install pydot -fi +# Python3 +pip install --pre protobuf==3.0.0b3 +pip install pydot From 1dcb93a2c69276d80426d15fa71caa43e071d693 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:26:16 +0100 Subject: [PATCH 14/46] (CI)(venv) drop python2 support --- scripts/travis/setup-venv.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/travis/setup-venv.sh b/scripts/travis/setup-venv.sh index 81245f146da..8f737a0165e 100755 --- a/scripts/travis/setup-venv.sh +++ b/scripts/travis/setup-venv.sh @@ -8,11 +8,7 @@ source $BASEDIR/defaults.sh VENV_DIR=${1:-~/venv} # setup our own virtualenv -if $WITH_PYTHON3; then - PYTHON_EXE='/usr/bin/python3' -else - PYTHON_EXE='/usr/bin/python2' -fi +PYTHON_EXE='/usr/bin/python3' # use --system-site-packages so that Python will use deb packages virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages From a122e57da4a988e03188e3b7104eaae8993c891a Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:31:12 +0100 Subject: [PATCH 15/46] (CI) don't build protobuf from source --- scripts/travis/install-deps.sh | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh index a5f85ad1098..d785513ca43 100755 --- a/scripts/travis/install-deps.sh +++ b/scripts/travis/install-deps.sh @@ -30,35 +30,6 @@ apt-get install -y --no-install-recommends \ python3-numpy \ python3-skimage - # build Protobuf3 since it's needed for Python3 - PROTOBUF3_DIR=~/protobuf3 - pushd . - if [ -d "$PROTOBUF3_DIR" ] && [ -e "$PROTOBUF3_DIR/src/protoc" ]; then - echo "Using cached protobuf3 build ..." - cd $PROTOBUF3_DIR - else - echo "Building protobuf3 from source ..." - rm -rf $PROTOBUF3_DIR - mkdir $PROTOBUF3_DIR - - # install some more dependencies required to build protobuf3 - apt-get install -y --no-install-recommends \ - curl \ - dh-autoreconf \ - unzip - - wget https://github.com/google/protobuf/archive/3.0.x.tar.gz -O protobuf3.tar.gz - tar -xzf protobuf3.tar.gz -C $PROTOBUF3_DIR --strip 1 - rm protobuf3.tar.gz - cd $PROTOBUF3_DIR - ./autogen.sh - ./configure --prefix=/usr - make --jobs=$NUM_THREADS - fi - make install - popd -fi - if $WITH_IO ; then apt-get install -y --no-install-recommends \ libleveldb-dev \ From ad51b8b72417f1ccd1f1eaa49768a4d307813e6a Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 4 Nov 2022 12:32:35 +0100 Subject: [PATCH 16/46] (CI) install protobuf from pip --- scripts/travis/install-python-deps.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/travis/install-python-deps.sh b/scripts/travis/install-python-deps.sh index 54ba2a17223..26a021ca394 100755 --- a/scripts/travis/install-python-deps.sh +++ b/scripts/travis/install-python-deps.sh @@ -6,5 +6,6 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh # Python3 -pip install --pre protobuf==3.0.0b3 -pip install pydot +pip install \ + protobuf + pydot From 182475c3b9dade69d5c830e41fda366ef672faf8 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 7 Nov 2022 12:07:48 +0100 Subject: [PATCH 17/46] (travis) fix typo --- scripts/travis/install-python-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/travis/install-python-deps.sh b/scripts/travis/install-python-deps.sh index 26a021ca394..102fb90526b 100755 --- a/scripts/travis/install-python-deps.sh +++ b/scripts/travis/install-python-deps.sh @@ -7,5 +7,5 @@ source $BASEDIR/defaults.sh # Python3 pip install \ - protobuf + protobuf \ pydot From 685a7617f42ff4be7040c3e00e8eca3de70fe33a Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 9 Dec 2022 17:59:27 +0100 Subject: [PATCH 18/46] Restore Makefile.config.example --- Makefile.config.example | 123 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 Makefile.config.example diff --git a/Makefile.config.example b/Makefile.config.example new file mode 100644 index 00000000000..a9776b73df2 --- /dev/null +++ b/Makefile.config.example @@ -0,0 +1,123 @@ +## Refer to http://caffe.berkeleyvision.org/installation.html +# Contributions simplifying and improving our build system are welcome! + +# cuDNN acceleration switch (uncomment to build with cuDNN). +# USE_CUDNN := 1 + +# CPU-only switch (uncomment to build without GPU support). +# CPU_ONLY := 1 + +# uncomment to disable IO dependencies and corresponding data layers +# USE_OPENCV := 0 +# USE_LEVELDB := 0 +# USE_LMDB := 0 +# This code is taken from https://github.com/sh1r0/caffe-android-lib +# USE_HDF5 := 0 + +# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary) +# You should not set this flag if you will be reading LMDBs with any +# possibility of simultaneous read and write +# ALLOW_LMDB_NOLOCK := 1 + +# Uncomment if you're using OpenCV 3 +# OPENCV_VERSION := 3 + +# To customize your choice of compiler, uncomment and set the following. +# N.B. the default for Linux is g++ and the default for OSX is clang++ +# CUSTOM_CXX := g++ + +# CUDA directory contains bin/ and lib/ directories that we need. +CUDA_DIR := /usr/local/cuda +# On Ubuntu 14.04, if cuda tools are installed via +# "sudo apt-get install nvidia-cuda-toolkit" then use this instead: +# CUDA_DIR := /usr + +# CUDA architecture setting: going with all of them. +# For CUDA < 6.0, comment the *_50 through *_61 lines for compatibility. +# For CUDA < 8.0, comment the *_60 and *_61 lines for compatibility. +# For CUDA >= 9.0, comment the *_20 and *_21 lines for compatibility. +CUDA_ARCH := -gencode arch=compute_20,code=sm_20 \ + -gencode arch=compute_20,code=sm_21 \ + -gencode arch=compute_30,code=sm_30 \ + -gencode arch=compute_35,code=sm_35 \ + -gencode arch=compute_50,code=sm_50 \ + -gencode arch=compute_52,code=sm_52 \ + -gencode arch=compute_60,code=sm_60 \ + -gencode arch=compute_61,code=sm_61 \ + -gencode arch=compute_61,code=compute_61 + +# BLAS choice: +# atlas for ATLAS (default) +# mkl for MKL +# open for OpenBlas +BLAS := atlas +# Custom (MKL/ATLAS/OpenBLAS) include and lib directories. +# Leave commented to accept the defaults for your choice of BLAS +# (which should work)! +# BLAS_INCLUDE := /path/to/your/blas +# BLAS_LIB := /path/to/your/blas + +# Homebrew puts openblas in a directory that is not on the standard search path +# BLAS_INCLUDE := $(shell brew --prefix openblas)/include +# BLAS_LIB := $(shell brew --prefix openblas)/lib + +# This is required only if you will compile the matlab interface. +# MATLAB directory should contain the mex binary in /bin. +# MATLAB_DIR := /usr/local +# MATLAB_DIR := /Applications/MATLAB_R2012b.app + +# NOTE: this is required only if you will compile the python interface. +# We need to be able to find Python.h and numpy/arrayobject.h. +PYTHON_INCLUDE := /usr/include/python2.7 \ + /usr/lib/python2.7/dist-packages/numpy/core/include +# Anaconda Python distribution is quite popular. Include path: +# Verify anaconda location, sometimes it's in root. +# ANACONDA_HOME := $(HOME)/anaconda +# PYTHON_INCLUDE := $(ANACONDA_HOME)/include \ + # $(ANACONDA_HOME)/include/python2.7 \ + # $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include + +# Uncomment to use Python 3 (default is Python 2) +# PYTHON_LIBRARIES := boost_python3 python3.5m +# PYTHON_INCLUDE := /usr/include/python3.5m \ +# /usr/lib/python3.5/dist-packages/numpy/core/include + +# We need to be able to find libpythonX.X.so or .dylib. +PYTHON_LIB := /usr/lib /usr/local/lib +# PYTHON_LIB := $(ANACONDA_HOME)/lib + +# Homebrew installs numpy in a non standard path (keg only) +# PYTHON_INCLUDE += $(dir $(shell python -c 'import numpy.core; print(numpy.core.__file__)'))/include +# PYTHON_LIB += $(shell brew --prefix numpy)/lib + +# Uncomment to support layers written in Python (will link against Python libs) +# WITH_PYTHON_LAYER := 1 + +# Whatever else you find you need goes here. +INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include +LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib + +# If Homebrew is installed at a non standard location (for example your home directory) and you use it for general dependencies +# INCLUDE_DIRS += $(shell brew --prefix)/include +# LIBRARY_DIRS += $(shell brew --prefix)/lib + +# NCCL acceleration switch (uncomment to build with NCCL) +# https://github.com/NVIDIA/nccl (last tested version: v1.2.3-1+cuda8.0) +# USE_NCCL := 1 + +# Uncomment to use `pkg-config` to specify OpenCV library paths. +# (Usually not necessary -- OpenCV libraries are normally installed in one of the above $LIBRARY_DIRS.) +# USE_PKG_CONFIG := 1 + +# N.B. both build and distribute dirs are cleared on `make clean` +BUILD_DIR := build +DISTRIBUTE_DIR := distribute + +# Uncomment for debugging. Does not work on OSX due to https://github.com/BVLC/caffe/issues/171 +# DEBUG := 1 + +# The ID of the GPU that 'make runtest' will use to run unit tests. +TEST_GPUID := 0 + +# enable pretty build (comment to see full commands) +Q ?= @ From 9fdfcc176e3b153acfa25ce232acdd7c74d968c7 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 9 Dec 2022 18:05:27 +0100 Subject: [PATCH 19/46] (actions) run on 20.04 for now --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fbbb5e92fbe..a6a5b6dca9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,7 +5,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: name: ${{ matrix.build_name }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: From c8af02bf320182657ee3ccf4cdfe3719cb95f6c8 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 9 Dec 2022 18:14:34 +0100 Subject: [PATCH 20/46] (travis) don't install protobuf with pip --- scripts/travis/install-python-deps.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/travis/install-python-deps.sh b/scripts/travis/install-python-deps.sh index 102fb90526b..2d450d11a79 100755 --- a/scripts/travis/install-python-deps.sh +++ b/scripts/travis/install-python-deps.sh @@ -7,5 +7,4 @@ source $BASEDIR/defaults.sh # Python3 pip install \ - protobuf \ pydot From 238e268dee35195ae63a25e69614a079f04a4e4c Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 9 Dec 2022 18:15:57 +0100 Subject: [PATCH 21/46] (travis) install protobuf with apt-get --- scripts/travis/install-deps.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh index d785513ca43..0bfd5c07326 100755 --- a/scripts/travis/install-deps.sh +++ b/scripts/travis/install-deps.sh @@ -17,6 +17,8 @@ apt-get install -y --no-install-recommends \ libgoogle-glog-dev \ libhdf5-serial-dev \ libopenblas-dev \ + libprotobuf-dev \ + protobuf-compiler \ python3-virtualenv \ wget From adb68d1ba7b52eb9ec816a1222ad7ab98a56d3ed Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 10:53:25 +0100 Subject: [PATCH 22/46] (cmake) only keep conditional HDF5 find --- cmake/Dependencies.cmake | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index ca2e3ad9e5e..f23dd941499 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -43,10 +43,6 @@ list(APPEND Caffe_LINKER_LIBS PUBLIC ${GFLAGS_LIBRARIES}) include(cmake/ProtoBuf.cmake) # ---[ HDF5 -find_package(HDF5 COMPONENTS HL REQUIRED) -list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${HDF5_INCLUDE_DIRS}) -list(APPEND Caffe_LINKER_LIBS PUBLIC ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) - # This code is taken from https://github.com/sh1r0/caffe-android-lib if(USE_HDF5) find_package(HDF5 COMPONENTS HL REQUIRED) From d157137c862d58240f2de76f830db32778ef07a4 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 16:35:43 +0100 Subject: [PATCH 23/46] (cmake) cleanup --- CMakeLists.txt | 3 +- cmake/Dependencies.cmake | 49 +++++++++++++------------------ cmake/Misc.cmake | 2 +- cmake/ProtoBuf.cmake | 2 +- cmake/Summary.cmake | 9 +++--- docker/gpu/Dockerfile | 1 - scripts/travis/configure-cmake.sh | 2 +- 7 files changed, 28 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 905cd7d83ef..4608b0a63eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,6 @@ caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ON caffe_option(USE_NCCL "Build Caffe with NCCL library support" OFF) caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON) caffe_option(BUILD_python "Build Python wrapper" ON) -set(python_version "2" CACHE STRING "Specify which Python version to use") caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE) caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON) @@ -105,7 +104,7 @@ add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/l # ---[ pytest target if(BUILD_python) - add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) + add_custom_target(pytest COMMAND python3 -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python ) add_dependencies(pytest pycaffe) endif() diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index f23dd941499..7a451aacd04 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -141,46 +141,37 @@ endif() # ---[ Python if(BUILD_python) - if(NOT "${python_version}" VERSION_LESS "3.0.0") - # use python3 - find_package(PythonInterp 3.0) - find_package(PythonLibs 3.0) - find_package(NumPy 1.7.1) - # Find the matching boost python implementation - set(version ${PYTHONLIBS_VERSION_STRING}) + # use python3 + find_package(Python 3 COMPONENTS Interpreter Development NumPy) + # Find the matching boost python implementation + set(version ${Python_VERSION}) + + STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} ) + find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") + set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) + + while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND) + STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} ) STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} ) find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) - while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND) - STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} ) - - STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} ) - find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}") - set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND}) - - STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) - if("${has_more_version}" STREQUAL "") - break() - endif() - endwhile() - if(NOT Boost_PYTHON_FOUND) - find_package(Boost 1.46 COMPONENTS python) + STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} ) + if("${has_more_version}" STREQUAL "") + break() endif() - else() - # disable Python 3 search - find_package(PythonInterp 2.7) - find_package(PythonLibs 2.7) - find_package(NumPy 1.7.1) + endwhile() + if(NOT Boost_PYTHON_FOUND) find_package(Boost 1.46 COMPONENTS python) endif() - if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND Boost_PYTHON_FOUND) + + if(Python_Interpreter_FOUND AND Python_NumPy_FOUND AND Boost_PYTHON_FOUND) set(HAVE_PYTHON TRUE) if(BUILD_python_layer) list(APPEND Caffe_DEFINITIONS PRIVATE -DWITH_PYTHON_LAYER) - list(APPEND Caffe_INCLUDE_DIRS PRIVATE ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} PUBLIC ${Boost_INCLUDE_DIRS}) - list(APPEND Caffe_LINKER_LIBS PRIVATE ${PYTHON_LIBRARIES} PUBLIC ${Boost_LIBRARIES}) + list(APPEND Caffe_INCLUDE_DIRS PRIVATE ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS} PUBLIC ${Boost_INCLUDE_DIRS}) + list(APPEND Caffe_LINKER_LIBS PRIVATE ${Python_LIBRARIES} PUBLIC ${Boost_LIBRARIES}) endif() endif() endif() diff --git a/cmake/Misc.cmake b/cmake/Misc.cmake index fcb246472f0..56c6a0c4cd4 100644 --- a/cmake/Misc.cmake +++ b/cmake/Misc.cmake @@ -29,7 +29,7 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) endif() # ---[ RPATH settings -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOLEAN "Use link paths for shared library rpath") +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOL "Use link paths for shared library rpath") set(CMAKE_MACOSX_RPATH TRUE) list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES diff --git a/cmake/ProtoBuf.cmake b/cmake/ProtoBuf.cmake index 72ea3230c50..f0a0d2b3cda 100644 --- a/cmake/ProtoBuf.cmake +++ b/cmake/ProtoBuf.cmake @@ -16,7 +16,7 @@ endif() if(PROTOBUF_FOUND) # fetches protobuf version caffe_parse_header(${PROTOBUF_INCLUDE_DIR}/google/protobuf/stubs/common.h VERION_LINE GOOGLE_PROTOBUF_VERSION) - string(REGEX MATCH "([0-9])00([0-9])00([0-9])" PROTOBUF_VERSION ${GOOGLE_PROTOBUF_VERSION}) + string(REGEX MATCH "([0-9])0([0-9]*)00([0-9])" PROTOBUF_VERSION ${GOOGLE_PROTOBUF_VERSION}) set(PROTOBUF_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") unset(GOOGLE_PROTOBUF_VERSION) endif() diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 40b8c2f2966..6c781d2aec6 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -125,8 +125,8 @@ function(caffe_print_configuration_summary) caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") caffe_status(" Boost : Yes (ver. ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION})") - caffe_status(" glog : Yes") - caffe_status(" gflags : Yes") + caffe_status(" glog : " GLOG_FOUND THEN "Yes" ELSE "No" ) + caffe_status(" gflags : " GFLAGS_FOUND THEN "Yes" ELSE "No" ) caffe_status(" protobuf : " PROTOBUF_FOUND THEN "Yes (ver. ${PROTOBUF_VERSION})" ELSE "No" ) if(USE_LMDB) caffe_status(" lmdb : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No") @@ -153,9 +153,8 @@ function(caffe_print_configuration_summary) endif() if(HAVE_PYTHON) caffe_status("Python:") - caffe_status(" Interpreter :" PYTHON_EXECUTABLE THEN "${PYTHON_EXECUTABLE} (ver. ${PYTHON_VERSION_STRING})" ELSE "No") - caffe_status(" Libraries :" PYTHONLIBS_FOUND THEN "${PYTHON_LIBRARIES} (ver ${PYTHONLIBS_VERSION_STRING})" ELSE "No") - caffe_status(" NumPy :" NUMPY_FOUND THEN "${NUMPY_INCLUDE_DIR} (ver ${NUMPY_VERSION})" ELSE "No") + caffe_status(" Interpreter :" Python_EXECUTABLE THEN "${Python_EXECUTABLE} (ver. ${Python_VERSION})" ELSE "No") + caffe_status(" NumPy :" Python_NumPy_FOUND THEN "${Python_NumPy_INCLUDE_DIRS} (ver ${Python_NumPy_VERSION})" ELSE "No") caffe_status("") endif() if(BUILD_matlab) diff --git a/docker/gpu/Dockerfile b/docker/gpu/Dockerfile index 448cd0d3945..f8ea01445ba 100644 --- a/docker/gpu/Dockerfile +++ b/docker/gpu/Dockerfile @@ -28,7 +28,6 @@ WORKDIR $CAFFE_ROOT RUN cd /opt && \ git clone https://github.com/CMU-Perceptual-Computing-Lab/caffe.git && \ cd $CAFFE_ROOT && \ - sed -i 's/set(python_version "2"/set(python_version "3"/g' CMakeLists.txt && \ # sed -i 's/set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")/set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -std=c++11")/g' CMakeLists.txt && \ # sed -i 's/cudnn.h CUDNN_VERSION_FILE_CONTENTS/cudnn_version.h CUDNN_VERSION_FILE_CONTENTS/g' cmake/Cuda.cmake && \ mkdir build && cd build && \ diff --git a/scripts/travis/configure-cmake.sh b/scripts/travis/configure-cmake.sh index d21c8b3af69..519830d0b8e 100644 --- a/scripts/travis/configure-cmake.sh +++ b/scripts/travis/configure-cmake.sh @@ -3,7 +3,7 @@ mkdir -p build cd build -ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open -Dpython_version=3" +ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open" if $WITH_IO ; then ARGS="$ARGS -DUSE_OPENCV=On -DUSE_LMDB=On -DUSE_LEVELDB=On" From 7c1dd7b7752071dfdb2c3346d94cc74e479aa1d3 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 16:46:16 +0100 Subject: [PATCH 24/46] (cmake) remove non-existing include dir var --- cmake/Dependencies.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 7a451aacd04..537e8d82661 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -46,7 +46,7 @@ include(cmake/ProtoBuf.cmake) # This code is taken from https://github.com/sh1r0/caffe-android-lib if(USE_HDF5) find_package(HDF5 COMPONENTS HL REQUIRED) - include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR}) + include_directories(SYSTEM ${HDF5_INCLUDE_DIRS}) list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}) add_definitions(-DUSE_HDF5) endif() From 7ec98be42d83f0346dfc6782f03bec1867e99693 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 16:54:17 +0100 Subject: [PATCH 25/46] (travis) add some echo's --- scripts/travis/build.sh | 3 +++ scripts/travis/configure-cmake.sh | 2 ++ scripts/travis/configure-make.sh | 2 ++ 3 files changed, 7 insertions(+) diff --git a/scripts/travis/build.sh b/scripts/travis/build.sh index bb9406f046c..4748ee12648 100755 --- a/scripts/travis/build.sh +++ b/scripts/travis/build.sh @@ -5,9 +5,12 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh if ! $WITH_CMAKE ; then + echo -e "\e[35m\e[1mmake --jobs $NUM_THREADS all test pycaffe warn\e[0m" make --jobs $NUM_THREADS all test pycaffe warn else + echo -e "\e[35m\e[1mcd build; make --jobs $NUM_THREADS all test.testbin\e[0m" cd build make --jobs $NUM_THREADS all test.testbin fi +echo -e "\e[35m\e[1mmake lint\e[0m" make lint diff --git a/scripts/travis/configure-cmake.sh b/scripts/travis/configure-cmake.sh index 519830d0b8e..cd34bdd2b33 100644 --- a/scripts/travis/configure-cmake.sh +++ b/scripts/travis/configure-cmake.sh @@ -1,5 +1,6 @@ # CMake configuration +echo -e "\e[35m\e[1mmkdir -p build; cd build\e[0m" mkdir -p build cd build @@ -24,4 +25,5 @@ else ARGS="$ARGS -DUSE_CUDNN=Off" fi +echo -e "\e[35m\e[1mcmake .. ${ARGS}\e[0m" cmake .. $ARGS diff --git a/scripts/travis/configure-make.sh b/scripts/travis/configure-make.sh index ae2aa717aba..d91ccf459e4 100644 --- a/scripts/travis/configure-make.sh +++ b/scripts/travis/configure-make.sh @@ -1,9 +1,11 @@ # raw Makefile configuration LINE () { + echo -e "\e[35m\e[1m$@\e[0m" echo "$@" >> Makefile.config } +echo -e "\e[35m\e[1mcp Makefile.config.example Makefile.config\e[0m" cp Makefile.config.example Makefile.config LINE "BLAS := open" From de642c19640880a74c526f7937750642cbc9f704 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 16:57:26 +0100 Subject: [PATCH 26/46] (travis) apt-get quite --- scripts/travis/install-deps.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/travis/install-deps.sh b/scripts/travis/install-deps.sh index 0bfd5c07326..db9a24276fa 100755 --- a/scripts/travis/install-deps.sh +++ b/scripts/travis/install-deps.sh @@ -5,8 +5,8 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh -apt-get -y update -apt-get install -y --no-install-recommends \ +apt-get -yq update +apt-get install -yq --no-install-recommends \ build-essential \ graphviz \ libboost-filesystem-dev \ @@ -23,17 +23,17 @@ apt-get install -y --no-install-recommends \ wget if $WITH_CMAKE ; then - apt-get install -y --no-install-recommends cmake + apt-get install -yq --no-install-recommends cmake fi # Python3 -apt-get install -y --no-install-recommends \ +apt-get install -yq --no-install-recommends \ python3-dev \ python3-numpy \ python3-skimage if $WITH_IO ; then - apt-get install -y --no-install-recommends \ + apt-get install -yq --no-install-recommends \ libleveldb-dev \ liblmdb-dev \ libopencv-dev \ @@ -54,7 +54,7 @@ if $WITH_CUDA ; then fi # update package lists - apt-get -y update + apt-get -yq update # install packages CUDA_PKG_VERSION="7-5" @@ -68,7 +68,7 @@ if $WITH_CUDA ; then ln -s /usr/local/cuda-$CUDA_VERSION /usr/local/cuda if $WITH_CUDNN ; then - apt-get install -y --no-install-recommends libcudnn7-dev + apt-get install -yq --no-install-recommends libcudnn7-dev fi fi From a8db6566a3fe71fca40c561eaf7e29d52752ca53 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 17:11:31 +0100 Subject: [PATCH 27/46] (actions) fix env vars --- .github/workflows/main.yml | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a6a5b6dca9f..0a5373c0829 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,9 @@ name: CI on: [push, pull_request, workflow_dispatch] +env: + NUM_THREADS: 4 + jobs: build: name: ${{ matrix.build_name }} @@ -17,27 +20,34 @@ jobs: # WITH_CUDA: false # WITH_CUDNN: false - BUILD_NAME: default-make + ENV: {} - BUILD_NAME: no-io-make - WITH_IO: false + ENV: + WITH_IO: false - BUILD_NAME: cuda-make - WITH_CUDA: true + ENV: + WITH_CUDA: true - BUILD_NAME: cudnn-make - WITH_CUDA: true - WITH_CUDNN: true + ENV: + WITH_CUDA: true + WITH_CUDNN: true - BUILD_NAME: default-cmake - WITH_CMAKE: true + ENV: + WITH_CMAKE: true - BUILD_NAME: no-io-cmake - WITH_CMAKE: true - WITH_IO: false + ENV: + WITH_CMAKE: true + WITH_IO: false - BUILD_NAME: cuda-cmake - WITH_CMAKE: true - WITH_CUDA: true + ENV: + WITH_CMAKE: true + WITH_CUDA: true - BUILD_NAME: cudnn-cmake - WITH_CMAKE: true - WITH_CUDA: true - WITH_CUDNN: true - env: - NUM_THREADS: 4 + ENV: + WITH_CMAKE: true + WITH_CUDA: true + WITH_CUDNN: true + env: ${{ matrix.env }} steps: - uses: actions/checkout@v3 - name: Defaults From 33f14acaeecae139c9c15f0feef91fce2dccaedf Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 17:36:05 +0100 Subject: [PATCH 28/46] Fix CV flags --- src/caffe/layers/window_data_layer.cpp | 2 +- src/caffe/test/test_io.cpp | 4 ++-- src/caffe/util/io.cpp | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/caffe/layers/window_data_layer.cpp b/src/caffe/layers/window_data_layer.cpp index 1bf3760e9fd..f41169debe4 100644 --- a/src/caffe/layers/window_data_layer.cpp +++ b/src/caffe/layers/window_data_layer.cpp @@ -290,7 +290,7 @@ void WindowDataLayer::load_batch(Batch* batch) { image_database_cache_[window[WindowDataLayer::IMAGE_INDEX]]; cv_img = DecodeDatumToCVMat(image_cached.second, true); } else { - cv_img = cv::imread(image.first, CV_LOAD_IMAGE_COLOR); + cv_img = cv::imread(image.first, cv::IMREAD_COLOR); if (!cv_img.data) { LOG(ERROR) << "Could not open or find file " << image.first; return; diff --git a/src/caffe/test/test_io.cpp b/src/caffe/test/test_io.cpp index c2c919e90dc..b80df287fba 100644 --- a/src/caffe/test/test_io.cpp +++ b/src/caffe/test/test_io.cpp @@ -20,8 +20,8 @@ class IOTest : public ::testing::Test {}; bool ReadImageToDatumReference(const string& filename, const int label, const int height, const int width, const bool is_color, Datum* datum) { cv::Mat cv_img; - int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : - CV_LOAD_IMAGE_GRAYSCALE); + int cv_read_flag = (is_color ? cv::IMREAD_COLOR : + cv::IMREAD_GRAYSCALE); cv::Mat cv_img_origin = cv::imread(filename, cv_read_flag); if (!cv_img_origin.data) { diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 485ff962134..8517396b76c 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -53,7 +53,7 @@ bool ReadProtoFromBinaryFile(const char* filename, Message* proto) { int fd = open(filename, O_RDONLY); CHECK_NE(fd, -1) << "File not found: " << filename; ZeroCopyInputStream* raw_input = new FileInputStream(fd); - CodedInputStream* coded_input = new CodedInputStream(raw_input); + CodedInputStream* coded_input = new CodedInputStream(raw_input); #if GOOGLE_PROTOBUF_VERSION >= 3011000 // Only take one parameter since protobuf 3.11 coded_input->SetTotalBytesLimit(kProtoReadBytesLimit); @@ -79,8 +79,8 @@ void WriteProtoToBinaryFile(const Message& proto, const char* filename) { cv::Mat ReadImageToCVMat(const string& filename, const int height, const int width, const bool is_color) { cv::Mat cv_img; - int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : - CV_LOAD_IMAGE_GRAYSCALE); + int cv_read_flag = (is_color ? cv::IMREAD_COLOR : + cv::IMREAD_GRAYSCALE); cv::Mat cv_img_origin = cv::imread(filename, cv_read_flag); if (!cv_img_origin.data) { LOG(ERROR) << "Could not open or find file " << filename; @@ -185,8 +185,8 @@ cv::Mat DecodeDatumToCVMat(const Datum& datum, bool is_color) { CHECK(datum.encoded()) << "Datum not encoded"; const string& data = datum.data(); std::vector vec_data(data.c_str(), data.c_str() + data.size()); - int cv_read_flag = (is_color ? CV_LOAD_IMAGE_COLOR : - CV_LOAD_IMAGE_GRAYSCALE); + int cv_read_flag = (is_color ? cv::IMREAD_COLOR : + cv::IMREAD_GRAYSCALE); cv_img = cv::imdecode(vec_data, cv_read_flag); if (!cv_img.data) { LOG(ERROR) << "Could not decode datum "; From 0b76e96ffbe8ec8dabdad05db6d051bf736a8c40 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 18:05:27 +0100 Subject: [PATCH 29/46] Adjust makefile to python3 --- Makefile.config.example | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Makefile.config.example b/Makefile.config.example index a9776b73df2..0db168074c0 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -68,26 +68,22 @@ BLAS := atlas # NOTE: this is required only if you will compile the python interface. # We need to be able to find Python.h and numpy/arrayobject.h. -PYTHON_INCLUDE := /usr/include/python2.7 \ - /usr/lib/python2.7/dist-packages/numpy/core/include +PYTHON_LIBRARIES := boost_python3 python3.8 +PYTHON_INCLUDE := /usr/include/python3.8 \ + /usr/lib/python3/dist-packages/numpy/core/include # Anaconda Python distribution is quite popular. Include path: # Verify anaconda location, sometimes it's in root. # ANACONDA_HOME := $(HOME)/anaconda # PYTHON_INCLUDE := $(ANACONDA_HOME)/include \ - # $(ANACONDA_HOME)/include/python2.7 \ - # $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include - -# Uncomment to use Python 3 (default is Python 2) -# PYTHON_LIBRARIES := boost_python3 python3.5m -# PYTHON_INCLUDE := /usr/include/python3.5m \ -# /usr/lib/python3.5/dist-packages/numpy/core/include + # $(ANACONDA_HOME)/include/python3.8 \ + # $(ANACONDA_HOME)/lib/python3.8/site-packages/numpy/core/include # We need to be able to find libpythonX.X.so or .dylib. PYTHON_LIB := /usr/lib /usr/local/lib # PYTHON_LIB := $(ANACONDA_HOME)/lib # Homebrew installs numpy in a non standard path (keg only) -# PYTHON_INCLUDE += $(dir $(shell python -c 'import numpy.core; print(numpy.core.__file__)'))/include +# PYTHON_INCLUDE += $(dir $(shell python3 -c 'import numpy.core; print(numpy.core.__file__)'))/include # PYTHON_LIB += $(shell brew --prefix numpy)/lib # Uncomment to support layers written in Python (will link against Python libs) From b47133f382ac530b10bfd2cb6716de130a5898c5 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 18:05:59 +0100 Subject: [PATCH 30/46] add CUDNN_STATUS_VERSION_MISMATCH --- include/caffe/util/cudnn.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/caffe/util/cudnn.hpp b/include/caffe/util/cudnn.hpp index cd3f93f6e28..0051d0c12e3 100644 --- a/include/caffe/util/cudnn.hpp +++ b/include/caffe/util/cudnn.hpp @@ -50,6 +50,10 @@ inline const char* cudnnGetErrorString(cudnnStatus_t status) { return "CUDNN_STATUS_RUNTIME_IN_PROGRESS"; case CUDNN_STATUS_RUNTIME_FP_OVERFLOW: return "CUDNN_STATUS_RUNTIME_FP_OVERFLOW"; +#endif +#if CUDNN_VERSION_MIN(8, 0, 1) + case CUDNN_STATUS_VERSION_MISMATCH: + return "CUDNN_STATUS_VERSION_MISMATCH"; #endif } return "Unknown cudnn status"; From ef9236a55af57ff2a071e3c63284323211c0886f Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 18:14:55 +0100 Subject: [PATCH 31/46] (cmake) fix python vars --- python/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c53299d265b..c73fcfa7b88 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -8,8 +8,8 @@ file(GLOB_RECURSE python_srcs ${PROJECT_SOURCE_DIR}/python/*.cpp) add_library(pycaffe SHARED ${python_srcs}) caffe_default_properties(pycaffe) set_target_properties(pycaffe PROPERTIES PREFIX "" OUTPUT_NAME "_caffe") -target_include_directories(pycaffe PUBLIC ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR}) -target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${PYTHON_LIBRARIES}) +target_include_directories(pycaffe PUBLIC ${Python_INCLUDE_DIRS} ${Python_NumPy_INCLUDE_DIRS}) +target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${Python_LIBRARIES}) if(UNIX OR APPLE) set(__linkname "${PROJECT_SOURCE_DIR}/python/caffe/_caffe.so") From dae8fa34524bde2db796e6b317841b0d4c786f80 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 20:05:22 +0100 Subject: [PATCH 32/46] Fix linting --- src/caffe/layers/cudnn_conv_layer.cpp | 47 +++++++++++++++++-------- src/caffe/layers/cudnn_deconv_layer.cpp | 8 ++--- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/caffe/layers/cudnn_conv_layer.cpp b/src/caffe/layers/cudnn_conv_layer.cpp index a4a69b3ac6a..a1812c17569 100644 --- a/src/caffe/layers/cudnn_conv_layer.cpp +++ b/src/caffe/layers/cudnn_conv_layer.cpp @@ -116,7 +116,7 @@ void CuDNNConvolutionLayer::Reshape( cudnnConvolutionFwdAlgoPerf_t fwd_algo_pref_[4]; cudnnConvolutionBwdDataAlgoPerf_t bwd_data_algo_pref_[4]; - //get memory sizes + // get memory sizes cudaMemGetInfo(&free_memory, &total_memory); #else // Specify workspace limit for kernels directly until we have a @@ -142,46 +142,63 @@ void CuDNNConvolutionLayer::Reshape( // Note: Copied from https://github.com/Qengineering/caffe/tree/ssd/src/caffe/layers #if CUDNN_VERSION_MIN(8, 0, 0) // choose forward algorithm for filter - // in forward filter the CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED is not implemented in cuDNN 8 - CUDNN_CHECK(cudnnGetConvolutionForwardAlgorithm_v7(handle_[0], bottom_descs_[i], filter_desc_, conv_descs_[i], top_descs_[i], 4, &RetCnt, fwd_algo_pref_)); + // in forward filter the CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED is not + // implemented in cuDNN 8 + CUDNN_CHECK(cudnnGetConvolutionForwardAlgorithm_v7(handle_[0], + bottom_descs_[i], + filter_desc_, + conv_descs_[i], + top_descs_[i], + 4, + &RetCnt, + fwd_algo_pref_)); found_conv_algorithm = false; - for(int n=0;n::Reshape( // // We have found that CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM is // // buggy. Thus, if this algo was chosen, choose winograd instead. If - // // winograd is not supported or workspace is larger than threshold, choose + // // winograd is not supported or workspace is larger than threshold, choose // NOLINT(whitespace/line_length) // // implicit_gemm instead. // if (fwd_algo_[i] == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM) { // size_t winograd_workspace_size; @@ -226,7 +226,7 @@ void CuDNNDeconvolutionLayer::Reshape( // &workspace_bwd_data_sizes_[i])); // } - // // reduce over all workspace sizes to get a maximum to allocate / reallocate + // // reduce over all workspace sizes to get a maximum to allocate / reallocate // NOLINT(whitespace/line_length) // size_t total_workspace_fwd = 0; // size_t total_workspace_bwd_data = 0; // size_t total_workspace_bwd_filter = 0; @@ -255,7 +255,7 @@ void CuDNNDeconvolutionLayer::Reshape( // // free the existing workspace and allocate a new (larger) one // cudaFree(this->workspaceData); - // cudaError_t err = cudaMalloc(&(this->workspaceData), workspaceSizeInBytes); + // cudaError_t err = cudaMalloc(&(this->workspaceData), workspaceSizeInBytes); // NOLINT(whitespace/line_length) // if (err != cudaSuccess) { // // force zero memory path // for (int i = 0; i < bottom.size(); i++) { @@ -278,7 +278,7 @@ void CuDNNDeconvolutionLayer::Reshape( // // if we succeed in the allocation, set pointer aliases for workspaces // for (int g = 0; g < (this->group_ * CUDNN_STREAMS_PER_GROUP); g++) { - // workspace[g] = reinterpret_cast(workspaceData) + g*max_workspace; + // workspace[g] = reinterpret_cast(workspaceData) + g*max_workspace; // NOLINT(whitespace/line_length) // } // } From a9790eefa032bf95989d39d439df120d143a3597 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 20:40:11 +0100 Subject: [PATCH 33/46] (CI) move scripts --- .github/workflows/main.yml | 14 +++++++------- scripts/{travis => ci}/build.sh | 0 scripts/{travis => ci}/configure-cmake.sh | 0 scripts/{travis => ci}/configure-make.sh | 0 scripts/{travis => ci}/configure.sh | 0 scripts/{travis => ci}/defaults.sh | 0 scripts/{travis => ci}/install-deps.sh | 0 scripts/{travis => ci}/install-python-deps.sh | 0 scripts/{travis => ci}/setup-venv.sh | 0 scripts/{travis => ci}/test.sh | 0 10 files changed, 7 insertions(+), 7 deletions(-) rename scripts/{travis => ci}/build.sh (100%) rename scripts/{travis => ci}/configure-cmake.sh (100%) rename scripts/{travis => ci}/configure-make.sh (100%) rename scripts/{travis => ci}/configure.sh (100%) rename scripts/{travis => ci}/defaults.sh (100%) rename scripts/{travis => ci}/install-deps.sh (100%) rename scripts/{travis => ci}/install-python-deps.sh (100%) rename scripts/{travis => ci}/setup-venv.sh (100%) rename scripts/{travis => ci}/test.sh (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0a5373c0829..817fecf8bb2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,19 +52,19 @@ jobs: - uses: actions/checkout@v3 - name: Defaults run: | - source ./scripts/travis/defaults.sh + source ./scripts/ci/defaults.sh - name: Install run: | - sudo -E ./scripts/travis/install-deps.sh - ./scripts/travis/setup-venv.sh ~/venv + sudo -E ./scripts/ci/install-deps.sh + ./scripts/ci/setup-venv.sh ~/venv source ~/venv/bin/activate - ./scripts/travis/install-python-deps.sh + ./scripts/ci/install-python-deps.sh - name: Configure run: | - ./scripts/travis/configure.sh + ./scripts/ci/configure.sh - name: Build run: | - ./scripts/travis/build.sh + ./scripts/ci/build.sh - name: Test run: | - ./scripts/travis/test.sh + ./scripts/ci/test.sh diff --git a/scripts/travis/build.sh b/scripts/ci/build.sh similarity index 100% rename from scripts/travis/build.sh rename to scripts/ci/build.sh diff --git a/scripts/travis/configure-cmake.sh b/scripts/ci/configure-cmake.sh similarity index 100% rename from scripts/travis/configure-cmake.sh rename to scripts/ci/configure-cmake.sh diff --git a/scripts/travis/configure-make.sh b/scripts/ci/configure-make.sh similarity index 100% rename from scripts/travis/configure-make.sh rename to scripts/ci/configure-make.sh diff --git a/scripts/travis/configure.sh b/scripts/ci/configure.sh similarity index 100% rename from scripts/travis/configure.sh rename to scripts/ci/configure.sh diff --git a/scripts/travis/defaults.sh b/scripts/ci/defaults.sh similarity index 100% rename from scripts/travis/defaults.sh rename to scripts/ci/defaults.sh diff --git a/scripts/travis/install-deps.sh b/scripts/ci/install-deps.sh similarity index 100% rename from scripts/travis/install-deps.sh rename to scripts/ci/install-deps.sh diff --git a/scripts/travis/install-python-deps.sh b/scripts/ci/install-python-deps.sh similarity index 100% rename from scripts/travis/install-python-deps.sh rename to scripts/ci/install-python-deps.sh diff --git a/scripts/travis/setup-venv.sh b/scripts/ci/setup-venv.sh similarity index 100% rename from scripts/travis/setup-venv.sh rename to scripts/ci/setup-venv.sh diff --git a/scripts/travis/test.sh b/scripts/ci/test.sh similarity index 100% rename from scripts/travis/test.sh rename to scripts/ci/test.sh From 9b9eea8853e0ff46ef0cdffc3280222df7e45811 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 20:45:22 +0100 Subject: [PATCH 34/46] (actions) no need for defaults --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 817fecf8bb2..e37764aaa4f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,9 +50,6 @@ jobs: env: ${{ matrix.env }} steps: - uses: actions/checkout@v3 - - name: Defaults - run: | - source ./scripts/ci/defaults.sh - name: Install run: | sudo -E ./scripts/ci/install-deps.sh From 091c6c5b9cf25f36ba2807842ca87ca17abb612f Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 20:47:18 +0100 Subject: [PATCH 35/46] (actions) log env vars --- scripts/ci/configure.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh index ef740c8982e..6990a48e32e 100755 --- a/scripts/ci/configure.sh +++ b/scripts/ci/configure.sh @@ -4,6 +4,11 @@ BASEDIR=$(dirname $0) source $BASEDIR/defaults.sh +echo -e "\e[35m\e[1mWITH_CMAKE: ${WITH_CMAKE}\e[0m" +echo -e "\e[35m\e[1mWITH_IO: ${WITH_IO}\e[0m" +echo -e "\e[35m\e[1mWITH_CUDA: ${WITH_CUDA}\e[0m" +echo -e "\e[35m\e[1mWITH_CUDNN: ${WITH_CUDNN}\e[0m" + if ! $WITH_CMAKE ; then source $BASEDIR/configure-make.sh else From 86babe240c33785a9db64f038efb9f5d7ccee61f Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 20:49:56 +0100 Subject: [PATCH 36/46] Drop pydotplus --- python/caffe/draw.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/python/caffe/draw.py b/python/caffe/draw.py index 0061f490bd9..73ae04abaa8 100644 --- a/python/caffe/draw.py +++ b/python/caffe/draw.py @@ -11,15 +11,7 @@ from caffe.proto import caffe_pb2 -""" -pydot is not supported under python 3 and pydot2 doesn't work properly. -pydotplus works nicely (pip install pydotplus) -""" -try: - # Try to load pydotplus - import pydotplus as pydot -except ImportError: - import pydot +import pydot # Internal layer and blob styles. LAYER_STYLE_DEFAULT = {'shape': 'record', From 851273e9fcf5725144acf227949800c7311ad41b Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 21:16:49 +0100 Subject: [PATCH 37/46] (actions) load virtualenv in each step --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e37764aaa4f..a5331280d9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,10 +58,13 @@ jobs: ./scripts/ci/install-python-deps.sh - name: Configure run: | + source ~/venv/bin/activate ./scripts/ci/configure.sh - name: Build run: | + source ~/venv/bin/activate ./scripts/ci/build.sh - name: Test run: | + source ~/venv/bin/activate ./scripts/ci/test.sh From 2b19ea8d9915b8f8c4b33d5159ecd78643e60792 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 21:57:58 +0100 Subject: [PATCH 38/46] Replace deprecated assertEquals by assertEqual --- python/caffe/test/test_coord_map.py | 56 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/python/caffe/test/test_coord_map.py b/python/caffe/test/test_coord_map.py index 613260e25df..7df1694020d 100644 --- a/python/caffe/test/test_coord_map.py +++ b/python/caffe/test/test_coord_map.py @@ -42,15 +42,15 @@ def test_conv_pool_deconv(self): n = coord_net_spec() # identity for 2x pool, 2x deconv ax, a, b = coord_map_from_to(n.deconv, n.data) - self.assertEquals(ax, 1) - self.assertEquals(a, 1) - self.assertEquals(b, 0) + self.assertEqual(ax, 1) + self.assertEqual(a, 1) + self.assertEqual(b, 0) # shift-by-one for 4x pool, 4x deconv n = coord_net_spec(pool=4, dstride=4) ax, a, b = coord_map_from_to(n.deconv, n.data) - self.assertEquals(ax, 1) - self.assertEquals(a, 1) - self.assertEquals(b, -1) + self.assertEqual(ax, 1) + self.assertEqual(a, 1) + self.assertEqual(b, -1) def test_pass(self): """ @@ -64,9 +64,9 @@ def test_pass(self): n.relu, num_output=10, kernel_size=1, stride=1, pad=0) for top in [n.relu, n.conv1x1]: ax_pass, a_pass, b_pass = coord_map_from_to(top, n.data) - self.assertEquals(ax, ax_pass) - self.assertEquals(a, a_pass) - self.assertEquals(b, b_pass) + self.assertEqual(ax, ax_pass) + self.assertEqual(a, a_pass) + self.assertEqual(b, b_pass) def test_padding(self): """ @@ -78,18 +78,18 @@ def test_padding(self): # conv padding n = coord_net_spec(pad=pad) _, a_pad, b_pad = coord_map_from_to(n.deconv, n.data) - self.assertEquals(a, a_pad) - self.assertEquals(b - pad, b_pad) + self.assertEqual(a, a_pad) + self.assertEqual(b - pad, b_pad) # deconv padding n = coord_net_spec(dpad=pad) _, a_pad, b_pad = coord_map_from_to(n.deconv, n.data) - self.assertEquals(a, a_pad) - self.assertEquals(b + pad, b_pad) + self.assertEqual(a, a_pad) + self.assertEqual(b + pad, b_pad) # pad both to cancel out n = coord_net_spec(pad=pad, dpad=pad) _, a_pad, b_pad = coord_map_from_to(n.deconv, n.data) - self.assertEquals(a, a_pad) - self.assertEquals(b, b_pad) + self.assertEqual(a, a_pad) + self.assertEqual(b, b_pad) def test_multi_conv(self): """ @@ -102,9 +102,9 @@ def test_multi_conv(self): pad=0) ax1, a1, b1 = coord_map_from_to(n.conv_data, n.data) ax2, a2, b2 = coord_map_from_to(n.conv_aux, n.aux) - self.assertEquals(ax1, ax2) - self.assertEquals(a1, a2) - self.assertEquals(b1, b2) + self.assertEqual(ax1, ax2) + self.assertEqual(a1, a2) + self.assertEqual(b1, b2) def test_rect(self): """ @@ -117,10 +117,10 @@ def test_rect(self): ax_5x5, a_5x5, b_5x5 = coord_map_from_to(n5x5.deconv, n5x5.data) ax_3x5, a_3x5, b_3x5 = coord_map_from_to(n3x5.deconv, n3x5.data) self.assertTrue(ax_3x3 == ax_5x5 == ax_3x5) - self.assertEquals(a_3x3, a_3x5[0]) - self.assertEquals(b_3x3, b_3x5[0]) - self.assertEquals(a_5x5, a_3x5[1]) - self.assertEquals(b_5x5, b_3x5[1]) + self.assertEqual(a_3x3, a_3x5[0]) + self.assertEqual(b_3x3, b_3x5[0]) + self.assertEqual(a_5x5, a_3x5[1]) + self.assertEqual(b_5x5, b_3x5[1]) def test_nd_conv(self): """ @@ -137,11 +137,11 @@ def test_nd_conv(self): n.deconv = L.Deconvolution( n.pool, num_output=10, kernel_size=4, stride=2, pad=0) ax, a, b = coord_map_from_to(n.deconv, n.data) - self.assertEquals(ax, 1) + self.assertEqual(ax, 1) self.assertTrue(len(a) == len(b)) self.assertTrue(np.all(a == 1)) - self.assertEquals(b[0] - 1, b[1]) - self.assertEquals(b[1] - 1, b[2]) + self.assertEqual(b[0] - 1, b[1]) + self.assertEqual(b[1] - 1, b[2]) def test_crop_of_crop(self): """ @@ -153,9 +153,9 @@ def test_crop_of_crop(self): ax, a, b = coord_map_from_to(n.deconv, n.data) n.crop = L.Crop(n.deconv, n.data, axis=2, offset=offset) ax_crop, a_crop, b_crop = coord_map_from_to(n.crop, n.data) - self.assertEquals(ax, ax_crop) - self.assertEquals(a, a_crop) - self.assertEquals(b + offset, b_crop) + self.assertEqual(ax, ax_crop) + self.assertEqual(a, a_crop) + self.assertEqual(b + offset, b_crop) def test_crop_helper(self): """ From 1bbcec2fda5c752ef375ad0c6f732f55662a5a50 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 10 Dec 2022 22:11:58 +0100 Subject: [PATCH 39/46] Replace deprecated assertRaisesRegexp by assertRaisesRegex --- python/caffe/test/test_net_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/caffe/test/test_net_spec.py b/python/caffe/test/test_net_spec.py index ffe71bacb08..20871985b3e 100644 --- a/python/caffe/test/test_net_spec.py +++ b/python/caffe/test/test_net_spec.py @@ -84,6 +84,6 @@ def test_type_error(self): """Test that a TypeError is raised when a Function input isn't a Top.""" data = L.DummyData(ntop=2) # data is a 2-tuple of Tops r = r"^Silence input 0 is not a Top \(type is <(type|class) 'tuple'>\)$" - with self.assertRaisesRegexp(TypeError, r): + with self.assertRaisesRegex(TypeError, r): L.Silence(data, ntop=0) # should raise: data is a tuple, not a Top L.Silence(*data, ntop=0) # shouldn't raise: each elt of data is a Top From 8e00b9f026e5e452bc6cdad50c3e82a64bdac417 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 11:42:09 +0100 Subject: [PATCH 40/46] (actions) run ubuntu 18.04 too --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5331280d9f..761135f5465 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,7 @@ jobs: strategy: fail-fast: false matrix: + os: [ubuntu-18.04, ubuntu-20.04] include: # Use a build matrix to test many builds in parallel # envvar defaults: From 3fdcb57ab9eb20c07c83032ce9b7dfaae72a1b26 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 11:43:19 +0100 Subject: [PATCH 41/46] (actions) run ubuntu 18.04 too --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 761135f5465..7175c3a30c6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ env: jobs: build: name: ${{ matrix.build_name }} - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: From d96e4faf4b587ebc564377d731058c9c968a1fc5 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 11:58:20 +0100 Subject: [PATCH 42/46] (actions) refactor matrix --- .github/workflows/main.yml | 50 +++++++++++--------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7175c3a30c6..162f70db40e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,47 +7,25 @@ env: jobs: build: - name: ${{ matrix.build_name }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-18.04, ubuntu-20.04] - include: - # Use a build matrix to test many builds in parallel - # envvar defaults: - # WITH_CMAKE: false - # WITH_IO: true - # WITH_CUDA: false - # WITH_CUDNN: false - - BUILD_NAME: default-make - ENV: {} - - BUILD_NAME: no-io-make - ENV: - WITH_IO: false - - BUILD_NAME: cuda-make - ENV: - WITH_CUDA: true - - BUILD_NAME: cudnn-make - ENV: - WITH_CUDA: true - WITH_CUDNN: true - - BUILD_NAME: default-cmake - ENV: - WITH_CMAKE: true - - BUILD_NAME: no-io-cmake - ENV: - WITH_CMAKE: true - WITH_IO: false - - BUILD_NAME: cuda-cmake - ENV: - WITH_CMAKE: true - WITH_CUDA: true - - BUILD_NAME: cudnn-cmake - ENV: - WITH_CMAKE: true - WITH_CUDA: true - WITH_CUDNN: true + # envvar defaults: + # WITH_CMAKE: false + # WITH_IO: true + # WITH_CUDA: false + # WITH_CUDNN: false + env: + - {} + - {WITH_IO: false} + - {WITH_CUDA: true} + - {WITH_CUDA: true, WITH_CUDNN: true} + - {WITH_CMAKE: true} + - {WITH_CMAKE: true, WITH_IO: false} + - {WITH_CMAKE: true, WITH_CUDA: true} + - {WITH_CMAKE: true, WITH_CUDA: true, WITH_CUDNN: true} env: ${{ matrix.env }} steps: - uses: actions/checkout@v3 From cb64f2a0ecb20d913a0a8e1b364afed6ed28f6b3 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 12:02:46 +0100 Subject: [PATCH 43/46] (actions) restore build name --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 162f70db40e..c1077a7c231 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,7 @@ env: jobs: build: + name: $BUILD_NAME - ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -18,14 +19,14 @@ jobs: # WITH_CUDA: false # WITH_CUDNN: false env: - - {} - - {WITH_IO: false} - - {WITH_CUDA: true} - - {WITH_CUDA: true, WITH_CUDNN: true} - - {WITH_CMAKE: true} - - {WITH_CMAKE: true, WITH_IO: false} - - {WITH_CMAKE: true, WITH_CUDA: true} - - {WITH_CMAKE: true, WITH_CUDA: true, WITH_CUDNN: true} + - {BUILD_NAME: default-make} + - {BUILD_NAME: no-io-make, WITH_IO: false} + - {BUILD_NAME: cuda-make, WITH_CUDA: true} + - {BUILD_NAME: cudnn-make, WITH_CUDA: true, WITH_CUDNN: true} + - {BUILD_NAME: default-cmake, WITH_CMAKE: true} + - {BUILD_NAME: no-io-cmake, WITH_CMAKE: true, WITH_IO: false} + - {BUILD_NAME: cuda-cmake, WITH_CMAKE: true, WITH_CUDA: true} + - {BUILD_NAME: cudnn-cmake, WITH_CMAKE: true, WITH_CUDA: true, WITH_CUDNN: true} env: ${{ matrix.env }} steps: - uses: actions/checkout@v3 From cf0122a0c537affeff4141a300a4c46de59e2d79 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 12:03:35 +0100 Subject: [PATCH 44/46] (actions) fix build name --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1077a7c231..4b7cfe4e12d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ env: jobs: build: - name: $BUILD_NAME - ${{ matrix.os }} + name: ${{ matrix.env.BUILD_NAME }} - ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false From d58cc3d0b994cbd75039d4eb134f51266eb08cd2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 12:08:25 +0100 Subject: [PATCH 45/46] (ci) use `python3 -m virtualenv` to get it working in 18.04 --- scripts/ci/setup-venv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/setup-venv.sh b/scripts/ci/setup-venv.sh index 8f737a0165e..1d757cf1de0 100755 --- a/scripts/ci/setup-venv.sh +++ b/scripts/ci/setup-venv.sh @@ -11,4 +11,4 @@ VENV_DIR=${1:-~/venv} PYTHON_EXE='/usr/bin/python3' # use --system-site-packages so that Python will use deb packages -virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages +python3 -m virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages From 631422c55beba129fc0b8598cd2a596fdf16e4b1 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 12 Dec 2022 12:17:58 +0100 Subject: [PATCH 46/46] (actions) disable make job for now --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4b7cfe4e12d..d0a20b64907 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,10 +19,10 @@ jobs: # WITH_CUDA: false # WITH_CUDNN: false env: - - {BUILD_NAME: default-make} - - {BUILD_NAME: no-io-make, WITH_IO: false} - - {BUILD_NAME: cuda-make, WITH_CUDA: true} - - {BUILD_NAME: cudnn-make, WITH_CUDA: true, WITH_CUDNN: true} + # - {BUILD_NAME: default-make} + # - {BUILD_NAME: no-io-make, WITH_IO: false} + # - {BUILD_NAME: cuda-make, WITH_CUDA: true} + # - {BUILD_NAME: cudnn-make, WITH_CUDA: true, WITH_CUDNN: true} - {BUILD_NAME: default-cmake, WITH_CMAKE: true} - {BUILD_NAME: no-io-cmake, WITH_CMAKE: true, WITH_IO: false} - {BUILD_NAME: cuda-cmake, WITH_CMAKE: true, WITH_CUDA: true}