Skip to content
This repository was archived by the owner on Sep 27, 2019. It is now read-only.

Commit 20cd414

Browse files
mengranwomengranwo
authored
Merge branch 'master' into cuckoohash
2 parents 546c51a + 7699877 commit 20cd414

21 files changed

+586
-120
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ docs/dev
306306
# Protobuf files
307307
*.pb.cc
308308
*.pb.h
309+
*.pb
309310

310311
# Third party
311312
third_party/gflags/

‎Jenkinsfile

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ pipeline {
1717
sh 'python script/validators/source_validator.py'
1818
sh 'mkdir build'
1919
sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
20-
sh 'cd build && make check -j4 || true'
20+
sh 'cd build && make check -j4'
2121
// sh 'cd build && cp -pr test /job/' // special tests collection step just for this stage
2222
sh 'cd build && make install'
2323
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
@@ -33,7 +33,7 @@ pipeline {
3333
sh 'python script/validators/source_validator.py'
3434
sh 'mkdir build'
3535
sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
36-
sh 'cd build && make check -j4 || true'
36+
sh 'cd build && make check -j4'
3737
sh 'cd build && make install'
3838
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
3939
sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
@@ -48,7 +48,7 @@ pipeline {
4848
sh 'python script/validators/source_validator.py'
4949
sh 'mkdir build'
5050
sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
51-
sh 'cd build && make check -j4 || true'
51+
sh 'cd build && make check -j4'
5252
sh 'cd build && make install'
5353
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
5454
sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
@@ -63,7 +63,7 @@ pipeline {
6363
sh 'python script/validators/source_validator.py'
6464
sh 'mkdir build'
6565
sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
66-
sh 'cd build && make check -j4 || true'
66+
sh 'cd build && make check -j4'
6767
sh 'cd build && make install'
6868
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
6969
sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
@@ -88,7 +88,7 @@ pipeline {
8888
// sh 'python script/validators/source_validator.py'
8989
// sh 'mkdir build'
9090
// sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
91-
// sh 'cd build && make check -j4 || true'
91+
// sh 'cd build && make check -j4'
9292
// sh 'cd build && make install'
9393
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
9494
// sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
@@ -103,7 +103,7 @@ pipeline {
103103
// sh 'python script/validators/source_validator.py'
104104
// sh 'mkdir build'
105105
// sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
106-
// sh 'cd build && make check -j4 || true'
106+
// sh 'cd build && make check -j4'
107107
// sh 'cd build && make install'
108108
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
109109
// sh 'sudo apt-get -qq update && sudo apt-get -qq -y --no-install-recommends install wget default-jdk default-jre' // prerequisites for jdbc_validator
@@ -118,7 +118,7 @@ pipeline {
118118
// sh 'python script/validators/source_validator.py'
119119
// sh 'mkdir build'
120120
// sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
121-
// sh 'cd build && make check -j4 || true'
121+
// sh 'cd build && make check -j4'
122122
// sh 'cd build && make install'
123123
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
124124
// sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
@@ -133,7 +133,7 @@ pipeline {
133133
// sh 'python script/validators/source_validator.py'
134134
// sh 'mkdir build'
135135
// sh 'cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
136-
// sh 'cd build && make check -j4 || true'
136+
// sh 'cd build && make check -j4'
137137
// sh 'cd build && make install'
138138
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
139139
// sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
@@ -148,7 +148,7 @@ pipeline {
148148
sh 'python ./script/validators/source_validator.py'
149149
sh 'mkdir build'
150150
sh 'cd build && PATH=/usr/lib64/llvm4.0/bin:$PATH cmake -DCMAKE_CXX_FLAGS="-isystem /usr/include/llvm4.0" -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
151-
sh 'cd build && make check -j4 || true'
151+
sh 'cd build && make check -j4'
152152
sh 'cd build && make install'
153153
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
154154
sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
@@ -163,7 +163,7 @@ pipeline {
163163
sh 'python ./script/validators/source_validator.py'
164164
sh 'mkdir build'
165165
sh 'cd build && PATH=/usr/lib64/llvm4.0/bin:$PATH cmake -DCMAKE_CXX_FLAGS="-isystem /usr/include/llvm4.0" -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
166-
sh 'cd build && make check -j4 || true'
166+
sh 'cd build && make check -j4'
167167
sh 'cd build && make install'
168168
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
169169
sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
@@ -178,7 +178,7 @@ pipeline {
178178
// sh 'python script/validators/source_validator.py'
179179
// sh 'mkdir build'
180180
// sh 'cd build && cmake3 -DCMAKE_BUILD_TYPE=Debug -DCOVERALLS=False .. && make -j4'
181-
// sh 'cd build && make check -j4 || true'
181+
// sh 'cd build && make check -j4'
182182
// sh 'cd build && make install'
183183
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
184184
// sh 'sudo yum -q install -y wget java-devel' // prerequisites for jdbc_validator
@@ -193,7 +193,7 @@ pipeline {
193193
// sh 'python script/validators/source_validator.py'
194194
// sh 'mkdir build'
195195
// sh 'cd build && cmake3 -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
196-
// sh 'cd build && make check -j4 || true'
196+
// sh 'cd build && make check -j4'
197197
// sh 'cd build && make install'
198198
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
199199
// sh 'sudo yum -q install -y wget java-devel' // prerequisites for jdbc_validator

‎cmake/Dependencies.cmake

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ include(cmake/CapnProtoMacros.cmake)
3131
# ---[ Google-protobuf
3232
include(cmake/ProtoBuf.cmake)
3333

34+
# --[ tensorflow
35+
find_library(TFlowC
36+
NAMES tensorflow
37+
PATHS "/usr/local/lib")
38+
list(APPEND Peloton_LINKER_LIBS ${TFlowC})
39+
3440
# ---[ Libevent
3541
find_package(Libevent REQUIRED)
3642
include_directories(SYSTEM ${LIBEVENT_INCLUDE_DIRS})

‎cmake/ProtoBuf.cmake

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
# Finds Google Protocol Buffers library and compilers and extends
22
# the standard cmake script with version and python generation support
3-
4-
find_package( Protobuf REQUIRED )
3+
find_package( Protobuf REQUIRED)
54
include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR})
65
list(APPEND Peloton_LINKER_LIBS ${PROTOBUF_LIBRARIES})
76

‎script/installation/packages.sh

Lines changed: 93 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
## * OSX
1818
## =================================================================
1919

20+
2021
# Determine OS platform
2122
UNAME=$(uname | tr "[:upper:]" "[:lower:]")
2223
# If Linux, try to determine specific distribution
@@ -36,6 +37,51 @@ fi
3637
unset UNAME
3738
DISTRO=$(echo $DISTRO | tr "[:lower:]" "[:upper:]")
3839
TMPDIR=/tmp
40+
TF_VERSION="1.4.0"
41+
TF_TYPE="cpu"
42+
43+
44+
function install_protobuf3.4.0() {
45+
# Install Relevant tooling
46+
# Remove any old versions of protobuf
47+
DISTRIB=$1 # ubuntu/fedora
48+
if [ "$DISTRIB" == "ubuntu" ]
49+
then
50+
sudo apt-get --yes --force-yes remove --purge libprotobuf-dev protobuf-compiler
51+
elif [ "$DISTRIB" == "fedora" ]
52+
then
53+
sudo dnf -q remove -y protobuf protobuf-devel protobuf-compiler
54+
else
55+
echo "Only Ubuntu and Fedora is supported currently!"
56+
return 0
57+
fi
58+
wget -O protobuf-cpp-3.4.0.tar.gz https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
59+
tar -xzf protobuf-cpp-3.4.0.tar.gz
60+
cd protobuf-3.4.0
61+
./autogen.sh && ./configure && make -j4 && sudo make install && sudo ldconfig
62+
cd ..
63+
# Cleanup
64+
rm -rf protobuf-3.4.0 protobuf-cpp-3.4.0.tar.gz
65+
}
66+
67+
# Utility function for installing tensorflow components of python/C++
68+
function install_tf() {
69+
TFCApiFile=$1
70+
TFBinaryURL=$2
71+
LinkerConfigCmd=$3
72+
TARGET_DIRECTORY="/usr/local"
73+
# Install Tensorflow Python Binary
74+
sudo -E pip3 install --upgrade ${TFBinaryURL}
75+
76+
# Install C-API
77+
TFCApiURL="https://storage.googleapis.com/tensorflow/libtensorflow/${TFCApiFile}"
78+
wget -O $TFCApiFile $TFCApiURL
79+
sudo tar -C $TARGET_DIRECTORY -xzf $TFCApiFile || true
80+
# Configure the Linker
81+
eval $LinkerConfigCmd
82+
# Cleanup
83+
rm -rf ${TFCApiFile}
84+
}
3985

4086
## ------------------------------------------------
4187
## UBUNTU
@@ -75,13 +121,19 @@ if [ "$DISTRO" = "UBUNTU" ]; then
75121
if [ "$MAJOR_VER" == "14" ]; then
76122
PKG_CMAKE="cmake3"
77123
FORCE_Y="--force-yes"
124+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp34-cp34m-linux_x86_64.whl"
125+
fi
126+
if [ "$MAJOR_VER" == "16" ]; then
127+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
78128
fi
79129
# Fix for llvm on Ubuntu 17.x
80130
if [ "$MAJOR_VER" == "17" ]; then
81131
PKG_LLVM="llvm-3.9"
82132
PKG_CLANG="clang-3.8"
133+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
83134
fi
84-
135+
TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
136+
LinkerConfigCmd="sudo ldconfig"
85137
sudo apt-get -qq $FORCE_Y --ignore-missing -y install \
86138
$PKG_CMAKE \
87139
$PKG_LLVM \
@@ -92,9 +144,7 @@ if [ "$DISTRO" = "UBUNTU" ]; then
92144
flex \
93145
valgrind \
94146
lcov \
95-
protobuf-compiler \
96147
libgflags-dev \
97-
libprotobuf-dev \
98148
libevent-dev \
99149
libboost-dev \
100150
libboost-thread-dev \
@@ -103,7 +153,19 @@ if [ "$DISTRO" = "UBUNTU" ]; then
103153
libpqxx-dev \
104154
libedit-dev \
105155
libssl-dev \
106-
postgresql-client
156+
postgresql-client \
157+
python3-pip \
158+
curl \
159+
autoconf \
160+
automake \
161+
libtool \
162+
make \
163+
g++ \
164+
unzip
165+
# Install version of protobuf needed by C-API
166+
install_protobuf3.4.0 "ubuntu"
167+
# Install tensorflow
168+
install_tf "$TFCApiFile" "$TFBinaryURL" "$LinkerConfigCmd"
107169

108170
## ------------------------------------------------
109171
## DEBIAN
@@ -140,14 +202,15 @@ elif [[ "$DISTRO" == *"FEDORA"* ]]; then
140202
26) LLVM="llvm";;
141203
*) LLVM="llvm4.0";;
142204
esac
143-
205+
TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
206+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp36-cp36m-linux_x86_64.whl"
207+
LinkerConfigCmd="sudo ldconfig"
144208
sudo dnf -q install -y \
145209
git \
146210
gcc-c++ \
147211
make \
148212
cmake \
149213
gflags-devel \
150-
protobuf-devel \
151214
bison \
152215
flex \
153216
libevent-devel \
@@ -166,7 +229,16 @@ elif [[ "$DISTRO" == *"FEDORA"* ]]; then
166229
libasan \
167230
libtsan \
168231
libubsan \
169-
libatomic
232+
libatomic \
233+
python3-pip \
234+
curl \
235+
autoconf \
236+
automake \
237+
libtool
238+
# Install version of protobuf needed by C-API
239+
install_protobuf3.4.0 "fedora"
240+
# Install tensorflow
241+
install_tf "$TFCApiFile" "$TFBinaryURL" "$LinkerConfigCmd"
170242

171243
## ------------------------------------------------
172244
## REDHAT
@@ -194,18 +266,17 @@ elif [[ "$DISTRO" == *"REDHAT"* ]] && [[ "${DISTRO_VER%.*}" == "7" ]]; then
194266
fi
195267
popd; popd
196268
return 0
197-
}
269+
}
198270

199-
# Package download paths
271+
#Package download paths
200272
PKGS=(
201273
"https://github.com/schuhschuh/gflags/archive/v2.0.tar.gz"
202274
)
203-
204-
# Add EPEL repository first
275+
#Add EPEL repository first
205276
sudo yum -q -y install epel-release
206277
sudo yum -q -y upgrade epel-release
207278

208-
# Simple installations via yum
279+
#Simple installations via yum
209280
sudo yum -q -y install \
210281
git \
211282
gcc-c++ \
@@ -245,7 +316,9 @@ elif [ "$DISTRO" = "DARWIN" ]; then
245316
echo "Installing homebrew..."
246317
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
247318
fi
248-
319+
TFBinaryURL="https://storage.googleapis.com/tensorflow/mac/${TF_TYPE}/tensorflow-${TF_VERSION}-py3-none-any.whl"
320+
TFCApiFile="libtensorflow-${TF_TYPE}-darwin-x86_64-${TF_VERSION}.tar.gz"
321+
LinkerConfigCmd="sudo update_dyld_shared_cache"
249322
brew install git
250323
brew install cmake
251324
brew install gflags
@@ -260,6 +333,13 @@ elif [ "$DISTRO" = "DARWIN" ]; then
260333
brew install libedit
261334
brew install [email protected]
262335
brew install postgresql
336+
brew install curl
337+
brew install wget
338+
brew install python
339+
brew upgrade python
340+
# Brew installs correct version of Protobuf(3.5.1 >= 3.4.0)
341+
# So we can directly install tensorflow
342+
install_tf "$TFCApiFile" "$TFBinaryURL" "$LinkerConfigCmd"
263343

264344
## ------------------------------------------------
265345
## UNKNOWN

‎src/CMakeLists.txt

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
##################################################################################
2-
# SRC CMAKELISTS
2+
#SRC CMAKELISTS
33
##################################################################################
44

5-
# --[ Cap'nProto library
5+
#--[Cap'nProto library
66

77
file(GLOB capnp_files include/capnp/*.capnp)
88
if (capnp_files)
@@ -34,6 +34,22 @@ peloton_default_properties(peloton-proto)
3434

3535
add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/libpg_query/ libpg_query.a)
3636

37+
##################################################################################
38+
39+
# --[ tensorflow
40+
# Generate essential model files
41+
set(MODEL_GEN_DIR ${CMAKE_CURRENT_LIST_DIR}/brain/modelgen)
42+
set(MODEL_GEN_PY ${MODEL_GEN_DIR}/model_generator.py)
43+
set(MODEL_GEN_SETTINGS ${MODEL_GEN_DIR}/settings.json)
44+
set(MODEL_GEN_COMMAND python3 ${MODEL_GEN_PY} ${MODEL_GEN_SETTINGS} ${MODEL_GEN_DIR})
45+
46+
message(STATUS "Tensorflow models being generated")
47+
execute_process(COMMAND ${MODEL_GEN_COMMAND}
48+
RESULT_VARIABLE RETURN_VALUE)
49+
50+
if (NOT RETURN_VALUE EQUAL 0)
51+
message(FATAL_ERROR "Failed to generate tensorflow models.")
52+
endif()
3753

3854
##################################################################################
3955

0 commit comments

Comments
 (0)