Skip to content

Commit 675b303

Browse files
keroro824keroro824
authored
Merge pull request #11 from sarthakpati/master
CMake support
2 parents 4d67a5e + 3d0b86e commit 675b303

File tree

4 files changed

+158
-14
lines changed

4 files changed

+158
-14
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,5 @@ mnist.scale
1818
*/*/cmake-build-debug
1919
*/cppobjs
2020
*/runme
21+
bin/*
22+
bin_linux/*

‎CMakeLists.txt‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
CMAKE_MINIMUM_REQUIRED(VERSION 3.0 FATAL_ERROR)
2+
3+
PROJECT( HashDeepLearning )
4+
5+
SET( CMAKE_CXX_STANDARD 11 )
6+
7+
# cross-platform OpenMP find
8+
IF(APPLE)
9+
# MESSAGE (STATUS "${OpenMP_LIBRARIES}")
10+
IF ("${OpenMP_LIBRARIES}" STREQUAL "")
11+
FIND_LIBRARY(OpenMP_LIBRARY
12+
NAMES omp
13+
)
14+
FIND_PATH(OpenMP_INCLUDE_DIR
15+
omp.h
16+
)
17+
MARK_AS_ADVANCED(OpenMP_LIBRARY OpenMP_INCLUDE_DIR)
18+
INCLUDE(FINDPACKAGEHANDLESTANDARDARGS)
19+
FIND_PACKAGE_HANDLE_STANDARD_ARGS(OpenMP DEFAULT_MSG
20+
OpenMP_LIBRARY OpenMP_INCLUDE_DIR)
21+
IF (OpenMP_FOUND)
22+
# MESSAGE (STATUS "OpenMP FOUND")
23+
SET(OpenMP_LIBRARIES ${OpenMP_LIBRARY} CACHE STRING "Cache OpenMP Lib" FORCE)
24+
SET(OpenMP_INCLUDE_DIRS ${OpenMP_INCLUDE_DIR} CACHE STRING "Cache OpenMP Include" FORCE)
25+
SET(OpenMP_COMPILE_OPTIONS -XPREPROCESSOR -FOpenMP -LOMP)
26+
ADD_LIBRARY(OpenMP::OpenMP SHARED IMPORTED)
27+
SET_TARGET_PROPERTIES(OpenMP::OpenMP PROPERTIES
28+
IMPORTED_LOCATION ${OpenMP_LIBRARIES}
29+
INTERFACE_INCLUDE_DIRECTORIES "${OpenMP_INCLUDE_DIRS}"
30+
INTERFACE_COMPILE_OPTIONS "${OpenMP_COMPILE_OPTIONS}"
31+
)
32+
ADD_DEFINITIONS(-D_OPENMP="${OpenMP_CXX_VERSION}")
33+
SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" )
34+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
35+
ENDIF()
36+
ENDIF()
37+
ELSE()
38+
FIND_PACKAGE(OpenMP REQUIRED)
39+
SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}" )
40+
SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}" )
41+
ENDIF()
42+
43+
# build dependencies
44+
INCLUDE( ExternalProject )
45+
46+
FIND_PACKAGE( Git REQUIRED )
47+
48+
OPTION( USE_GIT_PROTOCOL "If behind a firewall turn this off to use https instead." OFF )
49+
50+
function(functionInstallExternalCMakeProject ep_name)
51+
ExternalProject_Get_Property(${ep_name} binary_dir)
52+
install(SCRIPT ${binary_dir}/cmake_install.cmake)
53+
endfunction()
54+
55+
ExternalProject_Add(
56+
ZLIB
57+
DEPENDS ""
58+
GIT_REPOSITORY https://github.com/madler/zlib.git
59+
GIT_TAG v1.2.11
60+
SOURCE_DIR ZLIB-source
61+
BINARY_DIR ZLIB-build
62+
UPDATE_COMMAND ""
63+
PATCH_COMMAND ""
64+
# INSTALL_COMMAND ""
65+
CMAKE_GENERATOR ${gen}
66+
CMAKE_ARGS
67+
-DCMAKE_INSTALL_PREFIX:STRING=${PROJECT_BINARY_DIR}/ep
68+
-DINSTALL_BIN_DIR:STRING=${PROJECT_BINARY_DIR}/ep/bin
69+
-DINSTALL_INC_DIR:STRING=${PROJECT_BINARY_DIR}/ep/include
70+
-DINSTALL_LIB_DIR:STRING=${PROJECT_BINARY_DIR}/ep/lib
71+
-DINSTALL_MAN_DIR:STRING=${PROJECT_BINARY_DIR}/ep/share/man
72+
-DINSTALL_PKGCONFIG_DIR:STRING=${PROJECT_BINARY_DIR}/ep/share/pkgconfig
73+
-DCMAKE_BUILD_TYPE:STRING=Release
74+
)
75+
functionInstallExternalCMakeProject(ZLIB)
76+
77+
# set the expected zlib libraries
78+
IF( WIN32 )
79+
SET( ZLIB_LIB_DEBUG ${PROJECT_BINARY_DIR}/ep/lib/zlibstaticd.lib )
80+
SET( ZLIB_LIB_RELEASE ${PROJECT_BINARY_DIR}/ep/lib/zlibstatic.lib )
81+
ELSE()
82+
SET( ZLIB_LIB_DEBUG ${PROJECT_BINARY_DIR}/ep/lib/libz.a )
83+
SET( ZLIB_LIB_RELEASE ${PROJECT_BINARY_DIR}/ep/lib/libz.a )
84+
ENDIF()
85+
86+
ExternalProject_Add(
87+
CNPY
88+
DEPENDS ZLIB
89+
GIT_REPOSITORY https://github.com/sarthakpati/cnpy.git
90+
# GIT_TAG v1.2.11
91+
SOURCE_DIR CNPY-source
92+
BINARY_DIR CNPY-build
93+
UPDATE_COMMAND ""
94+
PATCH_COMMAND ""
95+
# INSTALL_COMMAND ""
96+
CMAKE_GENERATOR ${gen}
97+
CMAKE_ARGS
98+
-DZLIB_INCLUDE_DIR:STRING=${PROJECT_BINARY_DIR}/ep/include
99+
-DZLIB_LIBRARY_DEBUG:STRING=${ZLIB_LIB_DEBUG}
100+
-DZLIB_LIBRARY_RELEASE:STRING=${ZLIB_LIB_RELEASE}
101+
-DCMAKE_INSTALL_PREFIX:STRING=${PROJECT_BINARY_DIR}/ep
102+
-DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}
103+
-DCMAKE_BUILD_TYPE:STRING=Release
104+
)
105+
functionInstallExternalCMakeProject(CNPY)
106+
107+
INCLUDE_DIRECTORIES( ${PROJECT_BINARY_DIR}/ep/include )
108+
109+
# set CNPY lib path
110+
IF( WIN32 )
111+
SET( CNPY_LIB ${PROJECT_BINARY_DIR}/ep/lib/cnpy.lib )
112+
ELSE()
113+
SET( CNPY_LIB ${PROJECT_BINARY_DIR}/ep/lib/libcnpy.a )
114+
ENDIF()
115+
116+
# now build SLIDE
117+
FILE( GLOB_RECURSE SLIDE_SOURCES "${PROJECT_SOURCE_DIR}/SLIDE/*.cpp" )
118+
FILE( GLOB_RECURSE SLIDE_HEADERS "${PROJECT_SOURCE_DIR}/SLIDE/*.h" )
119+
120+
# add library to decouple compilation
121+
ADD_LIBRARY( SLIDE_LIB ${SLIDE_HEADERS} ${SLIDE_SOURCES} )
122+
ADD_DEPENDENCIES( SLIDE_LIB CNPY )
123+
TARGET_LINK_LIBRARIES( SLIDE_LIB ${CNPY_LIB} )
124+
125+
# add executable
126+
SET( SLIDE_EXE_NAME runme )
127+
ADD_EXECUTABLE( ${SLIDE_EXE_NAME} ${PROJECT_SOURCE_DIR}/SLIDE/main.cpp )
128+
ADD_DEPENDENCIES( ${SLIDE_EXE_NAME} SLIDE_LIB )
129+
TARGET_LINK_LIBRARIES(
130+
${SLIDE_EXE_NAME}
131+
SLIDE_LIB
132+
${CNPY_LIB}
133+
${ZLIB_LIB_RELEASE} ) # TBD: this should be changed to use ${ZLIB_LIBRARIES} for debug portability on Windows
134+
INSTALL( TARGETS ${SLIDE_EXE_NAME} DESTINATION bin )

‎README.md‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,25 +28,32 @@ python python_examples/example_sampled_softmax.py
2828

2929
## Running SLIDE
3030

31-
For simplicity, please refer to the our [Docker](https://hub.docker.com/repository/docker/ottovonxu/slide) image with all environments and a dataset installed [Amazon-670K](https://drive.google.com/open?id=0B3lPMIHmG6vGdUJwRzltS1dvUVk). To replicate the experiment, please type ```docker pull ottovonxu/slide:v3```
31+
### Dependencies
3232

33-
Firstly, [CNPY](https://github.com/rogersce/cnpy) package needs to be installed.
33+
- CMake v3.0 and above
34+
- C++11 Compliant compiler
35+
- Linux: Ubuntu 16.04 and newer
36+
- Transparent Huge Pages must be enabled.
37+
- SLIDE requires approximately 900 2MB pages, and 10 1GB pages: ([Instructions](https://wiki.debian.org/Hugepages))
3438

35-
Additionally, Transparent Huge Pages must be enabled. SLIDE requires approximately 900 2MB pages, and 10 1GB pages.
39+
### Notes:
3640

41+
- For simplicity, please refer to the our [Docker](https://hub.docker.com/repository/docker/ottovonxu/slide) image with all environments installed. To replicate the experiment without setting Hugepages, please download [Amazon-670K](https://drive.google.com/open?id=0B3lPMIHmG6vGdUJwRzltS1dvUVk) in path ```/home/code/HashingDeepLearning/dataset/Amazon```
3742

38-
Please see the [Instructions](https://wiki.debian.org/Hugepages) to enable Hugepages on Ubuntu.
43+
- Also, note that only Skylake or newer architectures support Hugepages. For older Haswell processors, we need to remove the flag `-mavx512f` from the `OPT_FLAGS` line in Makefile. You can also revert to the commit `2d10d46b5f6f1eda5d19f27038a596446fc17cee` to ignore the HugePages optimization and still use SLIDE (which could lead to a 30% slower performance).
3944

40-
Also, note that only Skylake or newer architectures support Hugepages. For older Haswell processors, we need to remove the flag `-mavx512f` from the `OPT_FLAGS` line in Makefile. You can also revert to the commit `2d10d46b5f6f1eda5d19f27038a596446fc17cee` to ignore the HugePages optmization and still use SLIDE (which could lead to a 30% slower performance).
45+
- This version builds all dependencies (which currently are [ZLIB](https://github.com/madler/zlib/tree/v1.2.11) and [CNPY](https://github.com/sarthakpati/cnpy)).
4146

47+
### Commands
4248

49+
Change the paths in ```./SLIDE/Config_amz.csv``` appropriately.
4350

44-
Run
45-
46-
```make```
47-
48-
```./runme Config_amz.csv```
49-
50-
Note that `Makefile` needs to be modified based on the CNPY path. Also the `trainData, testData, logFile` in Config_amz.csv needs to be changed accordingly too.
51-
52-
51+
```bash
52+
git clone https://github.com/sarthakpati/HashingDeepLearning.git
53+
cd HashingDeepLearning
54+
mkdir bin
55+
cd bin
56+
cmake ..
57+
make
58+
./runme ../SLIDE/Config_amz.csv
59+
```

‎SLIDE/Node.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <iostream>
55
#include <cmath>
66
#include <sys/mman.h>
7+
#include <asm-generic/mman-common.h>
78

89

910
using namespace std;

0 commit comments

Comments
 (0)