Skip to content

Commit 22e7fe0

Browse files
vera121vera121
authored
Windows py38 (#12)
* initial trials for build win with py38 locally * use .dll instead of .lib for build * Revert "use .dll instead of .lib for build" This reverts commit e439852. * revert all path hard-code changes * adjust py38 setup to fix compile errors on server * refine win build script for py38/vs2019 * use ninja instead of vs; add make clean up * save the setup that could link successful (but still has layer missing issues in caffe infererence) * fix the python caffe layer type missing issues * fix error for build pytest * clear up the conda path setup * windows path / \ handling fix * add boost fix also in build script * force to make install for windows to work around layer type missing issues for .exe evoke * refine codes to keep better backwards compatibility * try to make appveyor regression also able to work * further try appveyor fix * try to fix appveyor * try to fix appveyor * fix appveyor build error for py36 * refine hdf5 set after conda installed version change
1 parent 27d25f1 commit 22e7fe0

File tree

15 files changed

+256
-39
lines changed

15 files changed

+256
-39
lines changed

‎CMakeLists.txt‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ set(CAFFE_TARGET_VERSION "1.0.0" CACHE STRING "Caffe logical version")
88
set(CAFFE_TARGET_SOVERSION "1.0.0" CACHE STRING "Caffe soname version")
99
add_definitions(-DCAFFE_VERSION=${CAFFE_TARGET_VERSION})
1010

11+
if(MSVC)
12+
set(CONDA_LIB_PATH $ENV{SYNOPSYS_CAFFE_HOME}/Miniconda3/Library/lib)
13+
set(CONDA_INCLUDE_PATH $ENV{SYNOPSYS_CAFFE_HOME}/Miniconda3/Library/include)
14+
string(REPLACE "\\" "/" CONDA_LIB_PATH "${CONDA_LIB_PATH}")
15+
string(REPLACE "\\" "/" CONDA_INCLUDE_PATH "${CONDA_INCLUDE_PATH}")
16+
endif()
17+
1118
# ---[ Python3
1219
find_package(Python3 COMPONENTS Interpreter Development)
1320

@@ -24,6 +31,16 @@ message(STATUS "PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}")
2431

2532
include_directories(${PYTHON_INCLUDE_DIRS})
2633

34+
if(MSVC)
35+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
36+
add_compile_options(
37+
$<$<CONFIG:>:/MD> #---------|
38+
$<$<CONFIG:Release>:/MD> #--|
39+
$<$<CONFIG:Debug>:/MDd> #---|
40+
)
41+
endif()
42+
endif()
43+
2744
# ---[ Using cmake scripts and modules
2845
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
2946

@@ -78,14 +95,30 @@ if(MSVC AND BUILD_SHARED_LIBS)
7895
endif()
7996

8097
# ---[ Prebuild dependencies on windows
81-
include(cmake/WindowsDownloadPrebuiltDependencies.cmake)
98+
if("${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
99+
include(cmake/WindowsDownloadPrebuiltDependencies.cmake)
100+
endif()
82101

83102
# This code is taken from https://github.com/sh1r0/caffe-android-lib
84103
caffe_option(USE_HDF5 "Build with hdf5" ON)
85104

86105
# ---[ Dependencies
87106
include(cmake/Dependencies.cmake)
88107

108+
#if(MSVC)
109+
# set(CompilerFlags
110+
# CMAKE_CXX_FLAGS
111+
# CMAKE_CXX_FLAGS_DEBUG
112+
# CMAKE_CXX_FLAGS_RELEASE
113+
# CMAKE_C_FLAGS
114+
# CMAKE_C_FLAGS_DEBUG
115+
# CMAKE_C_FLAGS_RELEASE
116+
# )
117+
# foreach(CompilerFlag ${CompilerFlags})
118+
# string(REPLACE "/MD" "/MT" ${CompilerFlag} "${${CompilerFlag}}")
119+
# endforeach()
120+
#endif()
121+
89122
# ---[ Flags
90123
if(UNIX OR APPLE)
91124
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")
@@ -146,7 +179,7 @@ add_custom_target(lint COMMAND ${CMAKE_COMMAND} -DPYTHON_EXECUTABLE=${PYTHON_EXE
146179

147180
# ---[ pytest target
148181
if(BUILD_python)
149-
add_custom_target(pytest COMMAND python3 -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python )
182+
add_custom_target(pytest COMMAND python -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python)
150183
add_dependencies(pytest pycaffe)
151184
endif()
152185

@@ -166,3 +199,5 @@ caffe_print_configuration_summary()
166199
# ---[ Export configs generation
167200
caffe_generate_export_configs()
168201

202+
message(STATUS "Caffe_INCLUDE_DIRS ====== ${Caffe_INCLUDE_DIRS}")
203+
message(STATUS "Caffe_LINKER_LIBS ====== ${Caffe_LINKER_LIBS}")

‎cmake/Dependencies.cmake‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,25 @@
1-
# These lists are later turned into target properties on main caffe library target
1+
# These lists are later turned into target properties on main caffe library target
22
set(Caffe_LINKER_LIBS "")
33
set(Caffe_INCLUDE_DIRS "")
44
set(Caffe_DEFINITIONS "")
55
set(Caffe_COMPILE_OPTIONS "")
66

77
# ---[ Boost
8-
find_package( Boost COMPONENTS python3 system thread filesystem regex )
8+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
9+
find_package(Boost COMPONENTS python38 system thread filesystem regex)
10+
else()
11+
find_package(Boost COMPONENTS python3 system thread filesystem regex)
12+
endif()
13+
message(STATUS "boost find: include dir -> ${Boost_INCLUDE_DIRS}, lib -> ${Boost_LIBRARIES}")
914
list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${Boost_INCLUDE_DIRS})
1015
list(APPEND Caffe_DEFINITIONS PUBLIC -DBOOST_ALL_NO_LIB)
11-
list(APPEND Caffe_LINKER_LIBS PUBLIC ${Boost_LIBRARIES})
16+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
17+
set(Boost_LIBRARY ${CONDA_LIB_PATH}/boost_python38.lib)
18+
list(APPEND Boost_LIBRARY ${CONDA_LIB_PATH}/boost_system.lib ${CONDA_LIB_PATH}/boost_thread.lib ${CONDA_LIB_PATH}/boost_filesystem.lib ${CONDA_LIB_PATH}/boost_regex.lib)
19+
list(APPEND Caffe_LINKER_LIBS PUBLIC ${Boost_LIBRARY})
20+
else()
21+
list(APPEND Caffe_LINKER_LIBS PUBLIC ${Boost_LIBRARIES})
22+
endif()
1223

1324
if(DEFINED MSVC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0.40629.0)
1425
# Required for VS 2013 Update 4 or earlier.
@@ -35,7 +46,6 @@ if(USE_OPENMP)
3546
list(APPEND Caffe_COMPILE_OPTIONS PRIVATE ${OpenMP_CXX_FLAGS})
3647
endif()
3748

38-
3949
# ---[ Google-glog
4050
include("cmake/External/glog.cmake")
4151
list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${GLOG_INCLUDE_DIRS})
@@ -56,8 +66,10 @@ if(MSVC)
5666
list(APPEND CMAKE_MODULE_PATH ${HDF5_DIR})
5767
endif()
5868
find_package(HDF5 COMPONENTS C HL REQUIRED)
59-
set(HDF5_LIBRARIES hdf5-shared)
60-
set(HDF5_HL_LIBRARIES hdf5_hl-shared)
69+
if("${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
70+
set(HDF5_LIBRARIES hdf5-shared)
71+
set(HDF5_HL_LIBRARIES hdf5_hl-shared)
72+
endif()
6173
else()
6274
find_package(HDF5 COMPONENTS HL REQUIRED)
6375
endif()
@@ -98,6 +110,7 @@ if(USE_LEVELDB)
98110
list(APPEND Caffe_DEFINITIONS PUBLIC -DUSE_LEVELDB)
99111
endif()
100112

113+
101114
# ---[ Snappy
102115
if(USE_LEVELDB)
103116
find_package(Snappy REQUIRED)
@@ -126,7 +139,7 @@ endif()
126139

127140
# ---[ OpenCV
128141
if(USE_OPENCV)
129-
find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs videoio)
142+
find_package(OpenCV COMPONENTS core highgui imgproc imgcodecs videoio)
130143
if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
131144
find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
132145
endif()
@@ -169,7 +182,10 @@ endif()
169182

170183
# ---[ Python
171184
if(BUILD_python)
172-
if(NOT "${python_version}" VERSION_LESS "3.0.0")
185+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
186+
find_package(Boost COMPONENTS python38)
187+
find_package(NumPy)
188+
elseif(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.0.0" OR NOT "${python_version}" VERSION_LESS "3.0.0")
173189
# use python3
174190
find_package(PythonInterp 3.0)
175191
find_package(PythonLibs 3.0)
@@ -203,7 +219,8 @@ if(BUILD_python)
203219
find_package(NumPy 1.7.1)
204220
find_package(Boost 1.46 COMPONENTS python)
205221
endif()
206-
if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND Boost_PYTHON_FOUND)
222+
223+
if(PYTHONLIBS_FOUND AND NUMPY_FOUND AND (Boost_PYTHON_FOUND OR Boost_PYTHON3_FOUND OR Boost_PYTHON38_FOUND))
207224
set(HAVE_PYTHON TRUE)
208225
if(Boost_USE_STATIC_LIBS AND MSVC)
209226
list(APPEND Caffe_DEFINITIONS PUBLIC -DBOOST_PYTHON_STATIC_LIB)

‎cmake/Modules/FindGlog.cmake‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,18 @@ find_path(GLOG_INCLUDE_DIR glog/logging.h
1717
PATHS ${GLOG_ROOT_DIR})
1818

1919
if(MSVC)
20+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
21+
# required for py38 when glog version updates (could also be added in libraries\lib\cmake\glog\glog-config.cmake instead)
22+
set (glog_LIBRARY ${CONDA_LIB_PATH}/glog.lib)
23+
set (glog_LIBRARIES ${glog_LIBRARY})
24+
set (glog_INCLUDE_DIRS ${glog_INCLUDE_DIR})
25+
endif()
26+
2027
# rely on glog-config.cmake
2128
find_package(glog NO_MODULE)
22-
2329
set(GLOG_LIBRARY ${glog_LIBRARIES})
2430
set(GLOG_INCLUDE_DIR ${glog_INCLUDE_DIRS})
31+
add_compile_definitions(GLOG_NO_ABBREVIATED_SEVERITIES)
2532
else()
2633
find_library(GLOG_LIBRARY glog
2734
PATHS ${GLOG_ROOT_DIR}

‎cmake/Modules/FindLevelDB.cmake‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,14 @@
77
# Look for the header file.
88
if(MSVC)
99
find_package(LevelDB NO_MODULE)
10-
set(LevelDB_INCLUDE ${LevelDB_INCLUDE_DIRS})
11-
set(LevelDB_LIBRARY ${LevelDB_LIBRARIES})
10+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
11+
set(LevelDB_LIBRARY ${CONDA_LIB_PATH}/leveldb.lib)
12+
set(LevelDB_INCLUDE ${CONDA_INCLUDE_PATH}/leveldb)
13+
else()
14+
set(LevelDB_INCLUDE ${LevelDB_INCLUDE_DIRS})
15+
set(LevelDB_LIBRARY ${LevelDB_LIBRARIES})
16+
endif()
17+
1218
else()
1319
find_path(LevelDB_INCLUDE NAMES leveldb/db.h
1420
PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include

‎cmake/Modules/FindOpenBLAS.cmake‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@ FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PA
3737
FIND_LIBRARY(OpenBLAS_LIB NAMES ${OpenBLAS_LIB_NAMES} PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
3838

3939
SET(OpenBLAS_FOUND ON)
40+
if(MSVC)
41+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
42+
set(OpenBLAS_INCLUDE_DIR ${CONDA_INCLUDE_PATH}/openblas)
43+
set(OpenBLAS_LIB ${CONDA_LIB_PATH}/openblas.lib)
44+
endif()
45+
endif()
4046

4147
# Check include files
4248
IF(NOT OpenBLAS_INCLUDE_DIR)

‎cmake/Modules/FindSnappy.cmake‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
if(MSVC)
1111
# rely on snappy-config.cmake
1212
find_package(Snappy NO_MODULE)
13+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
14+
set(Snappy_INCLUDE_DIR ${CONDA_INCLUDE_PATH})
15+
set(Snappy_LIBRARIES ${CONDA_LIB_PATH}/snappy.lib)
16+
endif()
1317
else()
1418
find_path(Snappy_INCLUDE_DIR NAMES snappy.h
1519
PATHS ${SNAPPY_ROOT_DIR} ${SNAPPY_ROOT_DIR}/include)

‎cmake/ProtoBuf.cmake‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,22 @@
33

44
if(MSVC)
55
# search using protobuf-config.cmake
6-
find_package( Protobuf REQUIRED NO_MODULE)
6+
find_package(Protobuf REQUIRED NO_MODULE)
77
set(PROTOBUF_INCLUDE_DIR ${PROTOBUF_INCLUDE_DIRS})
8+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
9+
set(PROTOBUF_LIBRARIES ${CONDA_LIB_PATH}/libprotobuf.lib)
10+
# To solve linking extern issues: https://stackoverflow.com/questions/13733604/visual-studio-2010-c-w-google-protocol-buffers-cannot-find-60-externals-can/16065204#16065204
11+
add_compile_definitions(PROTOBUF_USE_DLLS)
12+
endif()
813
else()
9-
find_package( Protobuf REQUIRED )
14+
find_package(Protobuf REQUIRED)
1015
endif()
1116
list(APPEND Caffe_INCLUDE_DIRS PUBLIC ${PROTOBUF_INCLUDE_DIR})
12-
list(APPEND Caffe_LINKER_LIBS PUBLIC ${PROTOBUF_LIBRARIES})
17+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
18+
list(APPEND Caffe_LINKER_LIBS PUBLIC ${PROTOBUF_LIBRARIES} ${CONDA_LIB_PATH}/libprotoc.lib ${CONDA_LIB_PATH}/libprotobuf-lite.lib)
19+
else()
20+
list(APPEND Caffe_LINKER_LIBS PUBLIC ${PROTOBUF_LIBRARIES})
21+
endif()
1322

1423
# As of Ubuntu 14.04 protoc is no longer a part of libprotobuf-dev package
1524
# and should be installed separately as in: sudo apt-get install protobuf-compiler
@@ -31,8 +40,8 @@ endif()
3140
message(STATUS "PROTOBUF_INCLUDE_DIR ====== ${PROTOBUF_INCLUDE_DIR}")
3241
message(STATUS "PROTOBUF_LIBRARIES ====== ${PROTOBUF_LIBRARIES}")
3342
message(STATUS "PROTOBUF_PROTOC_EXECUTABLE ====== ${PROTOBUF_PROTOC_EXECUTABLE}")
34-
message(STATUS "Caffe_INCLUDE_DIRS ====== ${Caffe_INCLUDE_DIRS}")
35-
message(STATUS "Caffe_LINKER_LIBS ====== ${Caffe_LINKER_LIBS}")
43+
#message(STATUS "Caffe_INCLUDE_DIRS ====== ${Caffe_INCLUDE_DIRS}")
44+
#message(STATUS "Caffe_LINKER_LIBS ====== ${Caffe_LINKER_LIBS}")
3645

3746

3847
# place where to generate protobuf sources

‎cmake/WindowsDownloadPrebuiltDependenciesLocal.cmake‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
set(DEPENDENCIES_VERSION 1.1.0)
2+
caffe_option(USE_PREBUILT_DEPENDENCIES "Download and use the prebuilt dependencies" ON IF MSVC)
3+
4+
if(USE_PREBUILT_DEPENDENCIES)
5+
# Determine the python version
6+
if(BUILD_python)
7+
if(NOT PYTHONINTERP_FOUND)
8+
if(NOT "${python_version}" VERSION_LESS "3.8.0")
9+
find_package(PythonInterp 3.8)
10+
elseif(NOT "${python_version}" VERSION_LESS "3.6.0")
11+
find_package(PythonInterp 3.6)
12+
elseif(NOT "${python_version}" VERSION_LESS "3.5.0")
13+
find_package(PythonInterp 3.5)
14+
else()
15+
find_package(PythonInterp 2.7)
16+
endif()
17+
endif()
18+
set(_pyver ${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR})
19+
else()
20+
message(STATUS "Building without python. Prebuilt dependencies will default to Python 2.7")
21+
set(_pyver 27)
22+
endif()
23+
24+
# prebuilt packages path
25+
set(CAFFE_DEPENDENCIES_DIR "C:\\Users\\yche\\Desktop\\Projects\\caffe-builder\\build_v140_x64")
26+
27+
if(EXISTS ${CAFFE_DEPENDENCIES_DIR}/libraries/caffe-builder-config.cmake)
28+
include(${CAFFE_DEPENDENCIES_DIR}/libraries/caffe-builder-config.cmake)
29+
else()
30+
message(FATAL_ERROR "Something went wrong while dowloading dependencies could not open caffe-builder-config.cmake")
31+
endif()
32+
endif()
33+

‎include/caffe/util/mkl_alternate.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@
1111
#include <Accelerate/Accelerate.h>
1212
#else
1313
extern "C" {
14+
#if _MSC_VER > 1900
15+
#include <openblas/cblas.h> // for py38 env need to add "openblas/"
16+
#else
1417
#include <cblas.h>
18+
#endif
1519
}
1620
#endif // USE_ACCELERATE
1721

‎python/CMakeLists.txt‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,24 @@ set_target_properties(pycaffe PROPERTIES PREFIX "" OUTPUT_NAME "_caffe")
1111
if(MSVC)
1212
set_target_properties(pycaffe PROPERTIES SUFFIX ".pyd")
1313
endif()
14-
target_include_directories(pycaffe PUBLIC ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR})
15-
target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${PYTHON_LIBRARIES})
14+
15+
if(MSVC)
16+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
17+
add_compile_options(
18+
$<$<CONFIG:>:/MD> #---------|
19+
$<$<CONFIG:Release>:/MD> #--|
20+
$<$<CONFIG:Debug>:/MDd> #---|
21+
)
22+
endif()
23+
endif()
24+
25+
if(NOT "${PYTHON_VERSION_STRING}" VERSION_LESS "3.8.0")
26+
target_include_directories(pycaffe PUBLIC ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR} ${Caffe_INCLUDE_DIR})
27+
target_link_libraries(pycaffe PUBLIC ${Caffe_LINKER_LIBS} ${Caffe_LINK} ${PYTHON_LIBRARIES})
28+
else()
29+
target_include_directories(pycaffe PUBLIC ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIR})
30+
target_link_libraries(pycaffe PUBLIC ${Caffe_LINK} ${PYTHON_LIBRARIES})
31+
endif()
1632

1733
if(UNIX OR APPLE)
1834
set(__linkname "${PROJECT_SOURCE_DIR}/python/caffe/_caffe.so")

0 commit comments

Comments
 (0)