Skip to content
This repository was archived by the owner on Sep 27, 2019. It is now read-only.

Commit 80cc371

Browse files
pervazeapervazea
committed
Merge branch 'master' of https://github.com/cmu-db/peloton into junit
2 parents d983c3c + 6e8526c commit 80cc371

File tree

11 files changed

+395
-45
lines changed

11 files changed

+395
-45
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ docs/dev
308308
# Protobuf files
309309
*.pb.cc
310310
*.pb.h
311+
*.pb
311312

312313
# Third party
313314
third_party/gflags/

‎Jenkinsfile

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -141,35 +141,35 @@ pipeline {
141141
// }
142142
// }
143143

144-
stage('Fedora 27/gcc-7.2.1/llvm-4.0.1 (Debug)') {
145-
agent { docker { image 'fedora:27' } }
146-
steps {
147-
sh 'sudo /bin/bash -c "source ./script/installation/packages.sh"'
148-
sh 'python ./script/validators/source_validator.py'
149-
sh 'mkdir build'
150-
sh 'cd build && PATH=/usr/lib64/llvm4.0/bin:$PATH cmake -DCMAKE_CXX_FLAGS="-isystem /usr/include/llvm4.0" -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
151-
sh 'cd build && make check -j4'
152-
sh 'cd build && make install'
153-
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
154-
sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
155-
sh 'cd build && python ../script/validators/jdbc_validator.py'
156-
}
157-
}
144+
// stage('Fedora 27/gcc-7.2.1/llvm-4.0.1 (Debug)') {
145+
// agent { docker { image 'fedora:27' } }
146+
// steps {
147+
// sh 'sudo /bin/bash -c "source ./script/installation/packages.sh"'
148+
// sh 'python ./script/validators/source_validator.py'
149+
// sh 'mkdir build'
150+
// sh 'cd build && PATH=/usr/lib64/llvm4.0/bin:$PATH cmake -DCMAKE_CXX_FLAGS="-isystem /usr/include/llvm4.0" -DCMAKE_BUILD_TYPE=Debug -DUSE_SANITIZER=Address -DCOVERALLS=False .. && make -j4'
151+
// sh 'cd build && make check -j4'
152+
// sh 'cd build && make install'
153+
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
154+
// sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
155+
// sh 'cd build && python ../script/validators/jdbc_validator.py'
156+
// }
157+
// }
158158

159-
stage('Fedora 27/gcc-7.2.1/llvm-4.0.1 (Release)') {
160-
agent { docker { image 'fedora:27' } }
161-
steps {
162-
sh 'sudo /bin/bash -c "source ./script/installation/packages.sh"'
163-
sh 'python ./script/validators/source_validator.py'
164-
sh 'mkdir build'
165-
sh 'cd build && PATH=/usr/lib64/llvm4.0/bin:$PATH cmake -DCMAKE_CXX_FLAGS="-isystem /usr/include/llvm4.0" -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
166-
sh 'cd build && make check -j4'
167-
sh 'cd build && make install'
168-
sh 'cd build && bash ../script/testing/psql/psql_test.sh'
169-
sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
170-
sh 'cd build && python ../script/validators/jdbc_validator.py'
171-
}
172-
}
159+
// stage('Fedora 27/gcc-7.2.1/llvm-4.0.1 (Release)') {
160+
// agent { docker { image 'fedora:27' } }
161+
// steps {
162+
// sh 'sudo /bin/bash -c "source ./script/installation/packages.sh"'
163+
// sh 'python ./script/validators/source_validator.py'
164+
// sh 'mkdir build'
165+
// sh 'cd build && PATH=/usr/lib64/llvm4.0/bin:$PATH cmake -DCMAKE_CXX_FLAGS="-isystem /usr/include/llvm4.0" -DCMAKE_BUILD_TYPE=Release -DCOVERALLS=False .. && make -j4'
166+
// sh 'cd build && make check -j4'
167+
// sh 'cd build && make install'
168+
// sh 'cd build && bash ../script/testing/psql/psql_test.sh'
169+
// sh 'sudo dnf -q install -y wget java-devel' // prerequisites for jdbc_validator
170+
// sh 'cd build && python ../script/validators/jdbc_validator.py'
171+
// }
172+
// }
173173

174174
// stage('CentOS 7/gcc-4.8.5/llvm-3.9.1 (Debug)') {
175175
// agent { docker { image 'centos:7' } }

‎cmake/Dependencies.cmake

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ include(cmake/CapnProtoMacros.cmake)
3131
# ---[ Google-protobuf
3232
include(cmake/ProtoBuf.cmake)
3333

34+
# --[ tensorflow
35+
find_library(TFlowC
36+
NAMES tensorflow
37+
PATHS "/usr/local/lib")
38+
list(APPEND Peloton_LINKER_LIBS ${TFlowC})
39+
3440
# ---[ Libevent
3541
find_package(Libevent REQUIRED)
3642
include_directories(SYSTEM ${LIBEVENT_INCLUDE_DIRS})

‎cmake/ProtoBuf.cmake

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
# Finds Google Protocol Buffers library and compilers and extends
22
# the standard cmake script with version and python generation support
3-
4-
find_package( Protobuf REQUIRED )
3+
find_package( Protobuf REQUIRED)
54
include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR})
65
list(APPEND Peloton_LINKER_LIBS ${PROTOBUF_LIBRARIES})
76

‎script/installation/packages.sh

Lines changed: 93 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
## * OSX
1818
## =================================================================
1919

20+
2021
# Determine OS platform
2122
UNAME=$(uname | tr "[:upper:]" "[:lower:]")
2223
# If Linux, try to determine specific distribution
@@ -36,6 +37,51 @@ fi
3637
unset UNAME
3738
DISTRO=$(echo $DISTRO | tr "[:lower:]" "[:upper:]")
3839
TMPDIR=/tmp
40+
TF_VERSION="1.4.0"
41+
TF_TYPE="cpu"
42+
43+
44+
function install_protobuf3.4.0() {
45+
# Install Relevant tooling
46+
# Remove any old versions of protobuf
47+
DISTRIB=$1 # ubuntu/fedora
48+
if [ "$DISTRIB" == "ubuntu" ]
49+
then
50+
sudo apt-get --yes --force-yes remove --purge libprotobuf-dev protobuf-compiler
51+
elif [ "$DISTRIB" == "fedora" ]
52+
then
53+
sudo dnf -q remove -y protobuf protobuf-devel protobuf-compiler
54+
else
55+
echo "Only Ubuntu and Fedora is supported currently!"
56+
return 0
57+
fi
58+
wget -O protobuf-cpp-3.4.0.tar.gz https://github.com/google/protobuf/releases/download/v3.4.0/protobuf-cpp-3.4.0.tar.gz
59+
tar -xzf protobuf-cpp-3.4.0.tar.gz
60+
cd protobuf-3.4.0
61+
./autogen.sh && ./configure && make -j4 && sudo make install && sudo ldconfig
62+
cd ..
63+
# Cleanup
64+
rm -rf protobuf-3.4.0 protobuf-cpp-3.4.0.tar.gz
65+
}
66+
67+
# Utility function for installing tensorflow components of python/C++
68+
function install_tf() {
69+
TFCApiFile=$1
70+
TFBinaryURL=$2
71+
LinkerConfigCmd=$3
72+
TARGET_DIRECTORY="/usr/local"
73+
# Install Tensorflow Python Binary
74+
sudo -E pip3 install --upgrade ${TFBinaryURL}
75+
76+
# Install C-API
77+
TFCApiURL="https://storage.googleapis.com/tensorflow/libtensorflow/${TFCApiFile}"
78+
wget -O $TFCApiFile $TFCApiURL
79+
sudo tar -C $TARGET_DIRECTORY -xzf $TFCApiFile || true
80+
# Configure the Linker
81+
eval $LinkerConfigCmd
82+
# Cleanup
83+
rm -rf ${TFCApiFile}
84+
}
3985

4086
## ------------------------------------------------
4187
## UBUNTU
@@ -75,13 +121,19 @@ if [ "$DISTRO" = "UBUNTU" ]; then
75121
if [ "$MAJOR_VER" == "14" ]; then
76122
PKG_CMAKE="cmake3"
77123
FORCE_Y="--force-yes"
124+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp34-cp34m-linux_x86_64.whl"
125+
fi
126+
if [ "$MAJOR_VER" == "16" ]; then
127+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
78128
fi
79129
# Fix for llvm on Ubuntu 17.x
80130
if [ "$MAJOR_VER" == "17" ]; then
81131
PKG_LLVM="llvm-3.9"
82132
PKG_CLANG="clang-3.8"
133+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp35-cp35m-linux_x86_64.whl"
83134
fi
84-
135+
TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
136+
LinkerConfigCmd="sudo ldconfig"
85137
sudo apt-get -qq $FORCE_Y --ignore-missing -y install \
86138
$PKG_CMAKE \
87139
$PKG_LLVM \
@@ -92,9 +144,7 @@ if [ "$DISTRO" = "UBUNTU" ]; then
92144
flex \
93145
valgrind \
94146
lcov \
95-
protobuf-compiler \
96147
libgflags-dev \
97-
libprotobuf-dev \
98148
libevent-dev \
99149
libboost-dev \
100150
libboost-thread-dev \
@@ -103,7 +153,19 @@ if [ "$DISTRO" = "UBUNTU" ]; then
103153
libpqxx-dev \
104154
libedit-dev \
105155
libssl-dev \
106-
postgresql-client
156+
postgresql-client \
157+
python3-pip \
158+
curl \
159+
autoconf \
160+
automake \
161+
libtool \
162+
make \
163+
g++ \
164+
unzip
165+
# Install version of protobuf needed by C-API
166+
install_protobuf3.4.0 "ubuntu"
167+
# Install tensorflow
168+
install_tf "$TFCApiFile" "$TFBinaryURL" "$LinkerConfigCmd"
107169

108170
## ------------------------------------------------
109171
## DEBIAN
@@ -140,14 +202,15 @@ elif [[ "$DISTRO" == *"FEDORA"* ]]; then
140202
26) LLVM="llvm";;
141203
*) LLVM="llvm4.0";;
142204
esac
143-
205+
TFCApiFile="libtensorflow-${TF_TYPE}-linux-x86_64-${TF_VERSION}.tar.gz"
206+
TFBinaryURL="https://storage.googleapis.com/tensorflow/linux/${TF_TYPE}/tensorflow-${TF_VERSION}-cp36-cp36m-linux_x86_64.whl"
207+
LinkerConfigCmd="sudo ldconfig"
144208
sudo dnf -q install -y \
145209
git \
146210
gcc-c++ \
147211
make \
148212
cmake \
149213
gflags-devel \
150-
protobuf-devel \
151214
bison \
152215
flex \
153216
libevent-devel \
@@ -166,7 +229,16 @@ elif [[ "$DISTRO" == *"FEDORA"* ]]; then
166229
libasan \
167230
libtsan \
168231
libubsan \
169-
libatomic
232+
libatomic \
233+
python3-pip \
234+
curl \
235+
autoconf \
236+
automake \
237+
libtool
238+
# Install version of protobuf needed by C-API
239+
install_protobuf3.4.0 "fedora"
240+
# Install tensorflow
241+
install_tf "$TFCApiFile" "$TFBinaryURL" "$LinkerConfigCmd"
170242

171243
## ------------------------------------------------
172244
## REDHAT
@@ -194,18 +266,17 @@ elif [[ "$DISTRO" == *"REDHAT"* ]] && [[ "${DISTRO_VER%.*}" == "7" ]]; then
194266
fi
195267
popd; popd
196268
return 0
197-
}
269+
}
198270

199-
# Package download paths
271+
#Package download paths
200272
PKGS=(
201273
"https://github.com/schuhschuh/gflags/archive/v2.0.tar.gz"
202274
)
203-
204-
# Add EPEL repository first
275+
#Add EPEL repository first
205276
sudo yum -q -y install epel-release
206277
sudo yum -q -y upgrade epel-release
207278

208-
# Simple installations via yum
279+
#Simple installations via yum
209280
sudo yum -q -y install \
210281
git \
211282
gcc-c++ \
@@ -245,7 +316,9 @@ elif [ "$DISTRO" = "DARWIN" ]; then
245316
echo "Installing homebrew..."
246317
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
247318
fi
248-
319+
TFBinaryURL="https://storage.googleapis.com/tensorflow/mac/${TF_TYPE}/tensorflow-${TF_VERSION}-py3-none-any.whl"
320+
TFCApiFile="libtensorflow-${TF_TYPE}-darwin-x86_64-${TF_VERSION}.tar.gz"
321+
LinkerConfigCmd="sudo update_dyld_shared_cache"
249322
brew install git
250323
brew install cmake
251324
brew install gflags
@@ -260,6 +333,13 @@ elif [ "$DISTRO" = "DARWIN" ]; then
260333
brew install libedit
261334
brew install [email protected]
262335
brew install postgresql
336+
brew install curl
337+
brew install wget
338+
brew install python
339+
brew upgrade python
340+
# Brew installs correct version of Protobuf(3.5.1 >= 3.4.0)
341+
# So we can directly install tensorflow
342+
install_tf "$TFCApiFile" "$TFBinaryURL" "$LinkerConfigCmd"
263343

264344
## ------------------------------------------------
265345
## UNKNOWN

‎src/CMakeLists.txt

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
##################################################################################
2-
# SRC CMAKELISTS
2+
#SRC CMAKELISTS
33
##################################################################################
44

5-
# --[ Cap'nProto library
5+
#--[Cap'nProto library
66

77
file(GLOB capnp_files include/capnp/*.capnp)
88
if (capnp_files)
@@ -34,6 +34,22 @@ peloton_default_properties(peloton-proto)
3434

3535
add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/libpg_query/ libpg_query.a)
3636

37+
##################################################################################
38+
39+
# --[ tensorflow
40+
# Generate essential model files
41+
set(MODEL_GEN_DIR ${CMAKE_CURRENT_LIST_DIR}/brain/modelgen)
42+
set(MODEL_GEN_PY ${MODEL_GEN_DIR}/model_generator.py)
43+
set(MODEL_GEN_SETTINGS ${MODEL_GEN_DIR}/settings.json)
44+
set(MODEL_GEN_COMMAND python3 ${MODEL_GEN_PY} ${MODEL_GEN_SETTINGS} ${MODEL_GEN_DIR})
45+
46+
message(STATUS "Tensorflow models being generated")
47+
execute_process(COMMAND ${MODEL_GEN_COMMAND}
48+
RESULT_VARIABLE RETURN_VALUE)
49+
50+
if (NOT RETURN_VALUE EQUAL 0)
51+
message(FATAL_ERROR "Failed to generate tensorflow models.")
52+
endif()
3753

3854
##################################################################################
3955

0 commit comments

Comments
 (0)