fix(build): pytorch with custom spdlog

This change uses by default upstream spdlog 1.8.2

This is required for ubuntu 20.04 and RedHat distro as there hdoesn't ship a working version of libspdlog/libfmt.

Author: sileht

CMakeLists.txt

@@ -20,7 +20,7 @@ option(USE_CUDA_CV "use CUDA with OpenCV (Requires OpenCV build for CUDA)" OFF)
 option(USE_SIMSEARCH "build index and search services" OFF)
 option(USE_ANNOY "use annoy as indexer" OFF)
 option(USE_FAISS "use FAISS as indexer" ON)
-option(BUILD_SPDLOG "build SPDLOG instead of using system library" OFF)
+option(BUILD_SPDLOG "build SPDLOG instead of using system library" ON)
 option(USE_BOOST_BACKTRACE "use boost backtrace" ON)
 
 if (USE_CAFFE)
@@ -187,13 +187,16 @@ if (BUILD_SPDLOG)
         URL https://github.com/gabime/spdlog/archive/v1.8.2.tar.gz
         URL_HASH SHA256=e20e6bd8f57e866eaf25a5417f0a38a116e537f1a77ac7b5409ca2b180cec0d5
         BUILD_IN_SOURCE true
-        INSTALL_COMMAND ""
+        CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/spdlog/build -DCMAKE_POSITION_INDEPENDENT_CODE=ON
     )
-    set(SPDLOG_LIB_DEPS ${CMAKE_BINARY_DIR}/spdlog/src/spdlog/libspdlog.a)
-    set(SPDLOG_CAFFE_LIB_DIR ${CMAKE_BINARY_DIR}/spdlog/src/spdlog)
+    set(SPDLOG_LIB_DEPS ${CMAKE_BINARY_DIR}/spdlog/build/lib/libspdlog.a)
+    set(SPDLOG_LIB_DIR ${CMAKE_BINARY_DIR}/spdlog/build/lib)
     set(SPDLOG_CAFFE_LIB_DEPS :libspdlog.a)
-    set(SPDLOG_INCLUDE_DIR ${CMAKE_BINARY_DIR}/spdlog/src/spdlog/include)
+    set(SPDLOG_INCLUDE_DIR ${CMAKE_BINARY_DIR}/spdlog/build/include)
     include_directories(SYSTEM ${SPDLOG_INCLUDE_DIR})
+else()
+    find_package(spdlog 1.8.2 CONFIG REQUIRED)
+    set(SPDLOG_LIB_DEPS spdlog::spdlog)
 endif()
 
 # hdf5
@@ -627,7 +630,7 @@ elseif(USE_CAFFE)
         echo "INCLUDE_DIRS+=${PROTOBUF_INCLUDE_DIR}" >> Makefile.config &&
         echo "CUDA_ARCH:=${CUDA_ARCH}" >> Makefile.config &&
         echo "LIBRARIES+=${SPDLOG_CAFFE_LIB_DEPS}" >> Makefile.config &&
-        echo "LIBRARY_DIRS+=${SPDLOG_CAFFE_LIB_DIR}" >> Makefile.config &&
+        echo "LIBRARY_DIRS+=${SPDLOG_LIB_DIR}" >> Makefile.config &&
         echo "LIBRARY_DIRS+=${PROTOBUF_LIB_DIR}" >> Makefile.config
     BUILD_COMMAND 
         ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${PROTOBUF_LIB_DIR}:$ENV{LD_LIBRARY_PATH} PATH=${PROTOBUF_LIB_DIR}:$ENV{PATH} make -j${N} proto &&
@@ -760,7 +763,13 @@ if (USE_TORCH)
       PATCH_COMMAND test -f ${PYTORCH_COMPLETE} && echo Skipping || git apply ${PYTORCH_PATCHES} && echo Applying ${PYTORCH_PATCHES}
       CONFIGURE_COMMAND ""
       BUILD_COMMAND ""
-      COMMAND test -f ${PYTORCH_COMPLETE} && echo Skipping || PATH=${PROTOBUF_LIB_DIR}:$ENV{PATH} BUILD_CUSTOM_PROTOBUF=0 GLIBCXX_USE_CXX11_ABI=1 BUILD_TEST=0 USE_CUDA=${PYTORCH_USE_CUDA} BUILD_CAFFE2_OPS=0 USE_DDLOG=1 CAFFE2_LINK_LOCAL_PROTOBUF=0 MAX_JOBS=8 python3 ../pytorch/tools/build_libtorch.py
+      COMMAND test -f ${PYTORCH_COMPLETE} && echo Skipping
+        || PATH=${PROTOBUF_LIB_DIR}:$ENV{PATH}
+        BUILD_CUSTOM_PROTOBUF=0 GLIBCXX_USE_CXX11_ABI=1 BUILD_TEST=0
+        USE_CUDA=${PYTORCH_USE_CUDA}
+        BUILD_CAFFE2_OPS=0 USE_DDLOG=1 CAFFE2_LINK_LOCAL_PROTOBUF=0
+        "CMAKE_CXX_FLAGS=-isystem${SPDLOG_INCLUDE_DIR}"
+        MAX_JOBS=8 python3 ../pytorch/tools/build_libtorch.py
       INSTALL_COMMAND ""
     )
 
@@ -793,7 +802,8 @@ if (USE_TORCH)
     GIT_CONFIG advice.detachedHead=false
     CMAKE_ARGS -DWITH_CUDA=${PYTORCH_USE_CUDA} -DCMAKE_PREFIX_PATH=${TORCH_LOCATION} -DCMAKE_INSTALL_PREFIX=${TORCHVISION_LOCATION}
       -DCMAKE_INCLUDE_PATH=${PROJECT_INCLUDE_DIR} -DCMAKE_LIBRARY_PATH=${PROTOBUF_LIB_DIR}
-      "-DCMAKE_CXX_FLAGS=-isystem ${CMAKE_BINARY_DIR}/pytorch/src/pytorch/" "-DCMAKE_CUDA_FLAGS=-isystem ${CMAKE_BINARY_DIR}/pytorch/src/pytorch/"
+      "-DCMAKE_CXX_FLAGS=-isystem ${CMAKE_BINARY_DIR}/pytorch/src/pytorch/ -isystem ${SPDLOG_INCLUDE_DIR}"
+      "-DCMAKE_CUDA_FLAGS=-isystem ${CMAKE_BINARY_DIR}/pytorch/src/pytorch/ -isystem ${SPDLOG_INCLUDE_DIR}"
       -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
     DEPENDS protobuf pytorch
   )
@@ -1042,7 +1052,7 @@ include_directories("${PROJECT_SOURCE_DIR}/src")
 include_directories(${CMAKE_BINARY_DIR}/src)
 add_subdirectory(src)
 
-set(COMMON_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ${HTTP_INCLUDE_DIR} ${PROTOBUF_INCLUDE_DIR})
+set(COMMON_INCLUDE_DIRS ${Boost_INCLUDE_DIRS} ${HTTP_INCLUDE_DIR} ${PROTOBUF_INCLUDE_DIR} ${SPDLOG_INCLUDE_DIR})
 
 if (CUDA_FOUND)
     list(APPEND COMMON_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS})

ci/Jenkinsfile.prebuilt-cache

@@ -35,7 +35,7 @@ cmake ..  \
     -DUSE_TENSORRT=ON  \
     -DUSE_TENSORRT_OSS=ON  \
     -DCUDA_ARCH="-gencode arch=compute_61,code=sm_61"
-schedtool -B -n 1 -e ionice -n 1 make -j24 protobuf caffe_dd pytorch Multicore-TSNE ncnn xgboost cpp-netlib tensorrt-oss oatpp oatpp-swagger oatpp-zlib
+schedtool -B -n 1 -e ionice -n 1 make -j24 protobuf spdlog caffe_dd pytorch Multicore-TSNE ncnn xgboost cpp-netlib tensorrt-oss oatpp oatpp-swagger oatpp-zlib
 ccache -s
 
 # Remove cmake files

ci/Jenkinsfile.unittests

@@ -69,7 +69,7 @@ cmake .. \
 export PATH="/usr/lib/ccache/:$PATH"
 export TMPDIR=$(pwd)/build/tmp
 cd build
-schedtool -B -n 1 -e ionice -n 1 make -j24 protobuf caffe_dd pytorch Multicore-TSNE faisslib ncnn xgboost cpp-netlib tensorrt-oss oatpp oatpp-swagger oatpp-zlib
+schedtool -B -n 1 -e ionice -n 1 make -j24 protobuf spdlog caffe_dd pytorch Multicore-TSNE faisslib ncnn xgboost cpp-netlib tensorrt-oss oatpp oatpp-swagger oatpp-zlib
 # TODO(sileht): we should create the prebuilt artefacts here after each successful master branch build
 schedtool -B -n 1 -e ionice -n 1 make -j 6
 ccache -s

patches/pytorch/pytorch_16_new_logger.patch

@@ -3,11 +3,12 @@ new file mode 100644
 index 0000000000..56ebbf9441
 --- /dev/null
 +++ b/c10/util/logging_is_dd_log.h
-@@ -0,0 +1,60 @@
+@@ -0,0 +1,61 @@
 +#ifndef DD_LOG_H
 +#define DD_LOG_H
 +#include <set>
 +#include <iomanip>
++#include <map>
 +
 +#include "caffe/llogging.h"
 +
@@ -69,7 +70,7 @@ new file mode 100644
 index 0000000000..ff97118ff2
 --- /dev/null
 +++ b/caffe/llogging.h
-@@ -0,0 +1,211 @@
+@@ -0,0 +1,217 @@
 +/**
 + * Author: Emmanuel Benazera <[email protected]>
 + */
@@ -78,8 +79,14 @@ index 0000000000..ff97118ff2
 +#define LLOGGING_H
 +
 +#include <spdlog/spdlog.h>
++#ifdef USE_SYSLOG
++#include <spdlog/sinks/syslog_sink.h>
++#else
++#include <spdlog/sinks/stdout_sinks.h>
++#endif
 +#include <boost/algorithm/string.hpp>
 +#include <iostream>
++#include <map>
 +
 +class DateLogger {
 + public:

src/CMakeLists.txt

@@ -121,6 +121,9 @@ endif()
 
 add_library(ddetect ${ddetect_SOURCES})
 add_dependencies(ddetect protobuf caffe_pb_h)
+if (BUILD_SPDLOG)
+  add_dependencies(ddetect spdlog)
+endif()
 if (USE_DLIB)
   add_dependencies(ddetect dlib)
 endif()

src/backends/torch/llogging.h

@@ -10,11 +10,10 @@
 #endif
 
 #include <spdlog/spdlog.h>
-
 #ifdef USE_DD_SYSLOG
-#if SPDLOG_VER_MAJOR > 0
-#include <spdlog/sinks/syslog_sinks.h>
-#endif
+#include <spdlog/sinks/syslog_sink.h>
+#else
+#include <spdlog/sinks/stdout_sinks.h>
 #endif
 
 #include <boost/algorithm/string.hpp>