Skip to content

Commit 4eb8273

Browse files
peroneperone
committed
initial commit
0 parents  commit 4eb8273

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+24613
-0
lines changed

‎.gitignore‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
*.pth
2+
cmake-build-debug/
3+
.idea/
4+
db/
5+
__pycache__/
6+
*.pyc
7+
.DS_Store
8+
build/
9+
*.log
10+

‎CMakeLists.txt‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
cmake_minimum_required(VERSION 3.5)
2+
3+
project(euclidesdb CXX)
4+
5+
# ----[ Project versioning
6+
set(EUCLIDESDB_VERSION_MAJOR 0)
7+
set(EUCLIDESDB_VERSION_MINOR 1)
8+
set(EUCLIDESDB_VERSION_PATCH 0)
9+
set(EUCLIDESDB_VERSION_STRING ${EUCLIDESDB_VERSION_MAJOR}.${EUCLIDESDB_VERSION_MINOR}.${EUCLIDESDB_VERSION_PATCH})
10+
11+
# ----[ C++11 requirements, no gnu++11
12+
set(CMAKE_CXX_STANDARD 11)
13+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
14+
set(CMAKE_CXX_EXTENSIONS OFF)
15+
set(CMAKE_COLOR_MAKEFILE ON)
16+
17+
Include(FindProtobuf)
18+
19+
# ----[ libtorch should be a symbolic link inside the building
20+
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
21+
list(APPEND CMAKE_PREFIX_PATH ./libtorch)
22+
23+
include(TargetCopyFiles)
24+
25+
# ----[ Requirements
26+
find_package(Protobuf REQUIRED)
27+
find_package(GRPC REQUIRED)
28+
find_package(LevelDB REQUIRED)
29+
find_package(Torch REQUIRED)
30+
find_package(PythonInterp 3.6 REQUIRED)
31+
32+
# ---[ gRPC Protocols
33+
set(PROTOS
34+
${CMAKE_CURRENT_SOURCE_DIR}/source/protos/euclidesproto.proto
35+
)
36+
37+
# ---[ This is needed due to protobuf compilation
38+
include_directories(${CMAKE_CURRENT_BINARY_DIR})
39+
40+
# ---[ Protobuf and gRPC generation
41+
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTOS})
42+
grpc_generate_cpp(GRPC_SRCS GRPC_HDRS ${CMAKE_CURRENT_BINARY_DIR} ${PROTOS})
43+
protobuf_generate_python(PROTO_PY ${PROTOS})
44+
45+
add_custom_target(generate_proto ALL DEPENDS ${PROTO_PY})
46+
47+
# ----[ Set properties for generated files
48+
set_source_files_properties(
49+
${PROTO_SRCS}
50+
${PROTO_HDRS}
51+
PROPERTIES GENERATED TRUE)
52+
53+
set_source_files_properties(
54+
${PROTO_PY}
55+
PROPERTIES GENERATED TRUE)
56+
57+
set_source_files_properties(
58+
${GRPC_SRCS}
59+
${GRPC_HDRS}
60+
PROPERTIES GENERATED TRUE)
61+
62+
# ----[ Generate a header with versioning based on the template
63+
configure_file (
64+
"${CMAKE_CURRENT_SOURCE_DIR}/source/EuclidesConfig.h.in"
65+
"${PROJECT_BINARY_DIR}/EuclidesConfig.h"
66+
)
67+
68+
# ----[ Add all sources, both local and external
69+
file(GLOB CPP_FILES source/*.cpp source/external/*.cpp)
70+
71+
# ----[ Copy example configuration file
72+
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/source/external)
73+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/source/euclidesdb.conf
74+
${PROJECT_BINARY_DIR}/euclidesdb.conf COPYONLY)
75+
76+
# ----[ Copy all python files to the build
77+
add_custom_target(pythongenproto ALL)
78+
set(PYGRPC_DEST "${PROJECT_BINARY_DIR}/python")
79+
add_custom_command(TARGET pythongenproto PRE_BUILD
80+
COMMAND "./gen_protobuf.sh"
81+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/source/python
82+
COMMENT "Running Python Proto generation...")
83+
84+
add_custom_target(pythonfiles ALL
85+
DEPENDS pythongenproto)
86+
add_copy_directory(pythonfiles source/python
87+
DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/python
88+
GLOB *.py *.jpg
89+
EXCLUDE *.pth *.pyc __pycache__)
90+
91+
# ----[ Add the main server executable
92+
add_executable(${PROJECT_NAME}
93+
${CPP_FILES}
94+
${PROTO_SRCS}
95+
${GRPC_SRCS})
96+
97+
# ----[ Add the ./lib to the RPATH for Apple and Unix
98+
if(APPLE)
99+
set_target_properties(${PROJECT_NAME} PROPERTIES
100+
MACOSX_RPATH 1
101+
BUILD_WITH_INSTALL_RPATH 1
102+
INSTALL_RPATH "@loader_path/lib")
103+
elseif(UNIX)
104+
set_target_properties(${PROJECT_NAME} PROPERTIES
105+
BUILD_WITH_INSTALL_RPATH 1
106+
INSTALL_RPATH "$ORIGIN/lib")
107+
endif()
108+
109+
# ----[ Directives fro easyloggingcpp
110+
target_compile_options(${PROJECT_NAME} PRIVATE -DELPP_FEATURE_PERFORMANCE_TRACKING -DELPP_THREAD_SAFE)
111+
112+
# ----[ Link it to the required dependencies
113+
target_link_libraries(${PROJECT_NAME}
114+
${LevelDB_LIBRARIES}
115+
${TORCH_LIBRARIES}
116+
gRPC::grpc++_reflection
117+
protobuf::libprotobuf)
118+
119+
add_dependencies(${PROJECT_NAME} generate_proto)
120+
121+
# ----[ Copy libraries to the build directory
122+
add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
123+
COMMAND ${CMAKE_COMMAND} -E copy_directory
124+
${CMAKE_SOURCE_DIR}/libtorch/lib $<TARGET_FILE_DIR:${PROJECT_NAME}>/lib)
125+
126+
install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION euclidesdb)
127+
128+
# ----[ Copy libtorch libraries
129+
if(APPLE)
130+
install(DIRECTORY ${CMAKE_SOURCE_DIR}/libtorch/lib DESTINATION euclidesdb
131+
FILES_MATCHING PATTERN "*.dylib")
132+
elseif(UNIX)
133+
install(DIRECTORY ${CMAKE_SOURCE_DIR}/libtorch/lib DESTINATION euclidesdb
134+
FILES_MATCHING PATTERN "*.so*")
135+
endif()
136+
137+
# ----[ Copy the models and sample config
138+
install(DIRECTORY ${CMAKE_SOURCE_DIR}/models DESTINATION euclidesdb
139+
FILES_MATCHING PATTERN "*.*" PATTERN "*.pth" EXCLUDE)
140+
141+
install(DIRECTORY ${CMAKE_SOURCE_DIR}/models DESTINATION euclidesdb
142+
FILES_MATCHING PATTERN "*.*" PATTERN "*.pth" EXCLUDE)
143+
144+
install(FILES ${CMAKE_SOURCE_DIR}/source/sample_config.conf DESTINATION euclidesdb)
145+
146+
# ----[ Packaging with CPack
147+
execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE)
148+
message("Building for ${ARCHITECTURE}")
149+
150+
include(InstallRequiredSystemLibraries)
151+
set(CPACK_PACKAGE_NAME "euclidesdb")
152+
set(CPACK_PACKAGE_VENDOR "EuclidesDB")
153+
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "EuclidesDB")
154+
set(CPACK_PACKAGE_VERSION ${EUCLIDESDB_VERSION_STRING})
155+
set(CPACK_PACKAGE_VERSION_MAJOR "${EUCLIDESDB_VERSION_MAJOR}")
156+
set(CPACK_PACKAGE_VERSION_MINOR "${EUCLIDESDB_VERSION_MINOR}")
157+
set(CPACK_PACKAGE_VERSION_PATCH "${EUCLIDESDB_VERSION_PATCH}")
158+
set(CPACK_PACKAGE_INSTALL_DIRECTORY "euclidesdb")
159+
set(CPACK_PACKAGE_CONTACT "christian.perone@gmail.com")
160+
161+
set(CPACK_GENERATOR "TGZ;ZIP;STGZ")
162+
163+
if(APPLE)
164+
set(CPACK_GENERATOR "${CPACK_GENERATOR};DragNDrop")
165+
endif()
166+
167+
include(CPack)

‎LICENSE‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Copyright 2018 Christian S. Perone
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.

‎README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<img src="/docs/source/_static/img/logo.png" width="300">
2+
3+
[![Documentation Status](https://readthedocs.org/projects/euclidesdb/badge/?version=latest)](http://euclidesdb.readthedocs.io/en/latest/?badge=latest)
4+
5+
# Welcome to EuclidesDB
6+
EuclidesDB is a multi-model machine learning feature database that is tight coupled with PyTorch and provides a backend for including and querying data on the model feature space.
7+
8+
## Getting Started
9+
- [Official Documentation](http://euclidesdb.readthedocs.io)

‎cmake/FindGRPC.cmake‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
#
2+
# Locate and configure the gRPC library
3+
#
4+
# Adds the following targets:
5+
#
6+
# gRPC::grpc - gRPC library
7+
# gRPC::grpc++ - gRPC C++ library
8+
# gRPC::grpc++_reflection - gRPC C++ reflection library
9+
# gRPC::grpc_cpp_plugin - C++ generator plugin for Protocol Buffers
10+
#
11+
12+
#
13+
# Generates C++ sources from the .proto files
14+
#
15+
# grpc_generate_cpp (<SRCS> <HDRS> <DEST> [<ARGN>...])
16+
#
17+
# SRCS - variable to define with autogenerated source files
18+
# HDRS - variable to define with autogenerated header files
19+
# DEST - directory where the source files will be created
20+
# ARGN - .proto files
21+
#
22+
function(GRPC_GENERATE_CPP SRCS HDRS DEST)
23+
if(NOT ARGN)
24+
message(SEND_ERROR "Error: GRPC_GENERATE_CPP() called without any proto files")
25+
return()
26+
endif()
27+
28+
if(GRPC_GENERATE_CPP_APPEND_PATH)
29+
# Create an include path for each file specified
30+
foreach(FIL ${ARGN})
31+
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
32+
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
33+
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
34+
if(${_contains_already} EQUAL -1)
35+
list(APPEND _protobuf_include_path -I ${ABS_PATH})
36+
endif()
37+
endforeach()
38+
else()
39+
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
40+
endif()
41+
42+
if(DEFINED PROTOBUF_IMPORT_DIRS)
43+
foreach(DIR ${PROTOBUF_IMPORT_DIRS})
44+
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
45+
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
46+
if(${_contains_already} EQUAL -1)
47+
list(APPEND _protobuf_include_path -I ${ABS_PATH})
48+
endif()
49+
endforeach()
50+
endif()
51+
52+
set(${SRCS})
53+
set(${HDRS})
54+
foreach(FIL ${ARGN})
55+
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
56+
get_filename_component(FIL_WE ${FIL} NAME_WE)
57+
58+
list(APPEND ${SRCS} "${DEST}/${FIL_WE}.grpc.pb.cc")
59+
list(APPEND ${HDRS} "${DEST}/${FIL_WE}.grpc.pb.h")
60+
61+
add_custom_command(
62+
OUTPUT "${DEST}/${FIL_WE}.grpc.pb.cc"
63+
"${DEST}/${FIL_WE}.grpc.pb.h"
64+
COMMAND protobuf::protoc
65+
ARGS --grpc_out ${DEST} ${_protobuf_include_path} --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} ${ABS_FIL}
66+
DEPENDS ${ABS_FIL} protobuf::protoc gRPC::grpc_cpp_plugin
67+
COMMENT "Running C++ gRPC compiler on ${FIL}"
68+
VERBATIM )
69+
endforeach()
70+
71+
set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
72+
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
73+
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
74+
endfunction()
75+
76+
# By default have GRPC_GENERATE_CPP macro pass -I to protoc
77+
# for each directory where a proto file is referenced.
78+
if(NOT DEFINED GRPC_GENERATE_CPP_APPEND_PATH)
79+
set(GRPC_GENERATE_CPP_APPEND_PATH TRUE)
80+
endif()
81+
82+
# Find gRPC include directory
83+
find_path(GRPC_INCLUDE_DIR grpc/grpc.h)
84+
mark_as_advanced(GRPC_INCLUDE_DIR)
85+
86+
# Find gRPC library
87+
if(APPLE)
88+
find_library(GRPC_LIBRARY NAMES grpc)
89+
else()
90+
find_library(GRPC_LIBRARY NAMES libgrpc.a grpc)
91+
endif()
92+
93+
mark_as_advanced(GRPC_LIBRARY)
94+
add_library(gRPC::grpc UNKNOWN IMPORTED)
95+
set_target_properties(gRPC::grpc PROPERTIES
96+
INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
97+
INTERFACE_LINK_LIBRARIES "-lpthread;-ldl"
98+
IMPORTED_LOCATION ${GRPC_LIBRARY}
99+
)
100+
101+
# Find gRPC C++ library
102+
if(APPLE)
103+
find_library(GRPC_GRPC++_LIBRARY NAMES grpc++)
104+
else()
105+
find_library(GRPC_GRPC++_LIBRARY NAMES libgrpc++.a grpc++)
106+
endif()
107+
108+
mark_as_advanced(GRPC_GRPC++_LIBRARY)
109+
add_library(gRPC::grpc++ UNKNOWN IMPORTED)
110+
set_target_properties(gRPC::grpc++ PROPERTIES
111+
INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
112+
INTERFACE_LINK_LIBRARIES gRPC::grpc
113+
IMPORTED_LOCATION ${GRPC_GRPC++_LIBRARY}
114+
)
115+
116+
# Find gRPC C++ reflection library
117+
if(APPLE)
118+
find_library(GRPC_GRPC++_REFLECTION_LIBRARY NAMES grpc++_reflection)
119+
else()
120+
find_library(GRPC_GRPC++_REFLECTION_LIBRARY NAMES libgrpc++_reflection.a grpc++_reflection)
121+
endif()
122+
123+
mark_as_advanced(GRPC_GRPC++_REFLECTION_LIBRARY)
124+
add_library(gRPC::grpc++_reflection UNKNOWN IMPORTED)
125+
set_target_properties(gRPC::grpc++_reflection PROPERTIES
126+
INTERFACE_INCLUDE_DIRECTORIES ${GRPC_INCLUDE_DIR}
127+
INTERFACE_LINK_LIBRARIES gRPC::grpc++
128+
IMPORTED_LOCATION ${GRPC_GRPC++_REFLECTION_LIBRARY}
129+
)
130+
131+
# Find gRPC CPP generator
132+
find_program(GRPC_CPP_PLUGIN NAMES grpc_cpp_plugin)
133+
mark_as_advanced(GRPC_CPP_PLUGIN)
134+
add_executable(gRPC::grpc_cpp_plugin IMPORTED)
135+
set_target_properties(gRPC::grpc_cpp_plugin PROPERTIES
136+
IMPORTED_LOCATION ${GRPC_CPP_PLUGIN}
137+
)
138+
139+
include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)
140+
FIND_PACKAGE_HANDLE_STANDARD_ARGS(gRPC DEFAULT_MSG
141+
GRPC_LIBRARY GRPC_INCLUDE_DIR GRPC_GRPC++_REFLECTION_LIBRARY GRPC_CPP_PLUGIN)

‎cmake/FindLevelDB.cmake‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# - Find LevelDB
2+
#
3+
# LevelDB_INCLUDES - List of LevelDB includes
4+
# LevelDB_LIBRARIES - List of libraries when using LevelDB.
5+
# LevelDB_FOUND - True if LevelDB found.
6+
7+
# Look for the header file.
8+
find_path(LevelDB_INCLUDE NAMES leveldb/db.h
9+
PATHS $ENV{LEVELDB_ROOT}/include /opt/local/include /usr/local/include /usr/include
10+
DOC "Path in which the file leveldb/db.h is located." )
11+
12+
# Look for the library.
13+
find_library(LevelDB_LIBRARY NAMES leveldb
14+
PATHS /usr/lib $ENV{LEVELDB_ROOT}/lib
15+
DOC "Path to leveldb library." )
16+
17+
include(FindPackageHandleStandardArgs)
18+
find_package_handle_standard_args(LevelDB DEFAULT_MSG LevelDB_INCLUDE LevelDB_LIBRARY)
19+
20+
if(LEVELDB_FOUND)
21+
message(STATUS "Found LevelDB (include: ${LevelDB_INCLUDE}, library: ${LevelDB_LIBRARY})")
22+
set(LevelDB_INCLUDES ${LevelDB_INCLUDE})
23+
set(LevelDB_LIBRARIES ${LevelDB_LIBRARY})
24+
mark_as_advanced(LevelDB_INCLUDE LevelDB_LIBRARY)
25+
26+
if(EXISTS "${LevelDB_INCLUDE}/leveldb/db.h")
27+
file(STRINGS "${LevelDB_INCLUDE}/leveldb/db.h" __version_lines
28+
REGEX "static const int k[^V]+Version[ \t]+=[ \t]+[0-9]+;")
29+
30+
foreach(__line ${__version_lines})
31+
if(__line MATCHES "[^k]+kMajorVersion[ \t]+=[ \t]+([0-9]+);")
32+
set(LEVELDB_VERSION_MAJOR ${CMAKE_MATCH_1})
33+
elseif(__line MATCHES "[^k]+kMinorVersion[ \t]+=[ \t]+([0-9]+);")
34+
set(LEVELDB_VERSION_MINOR ${CMAKE_MATCH_1})
35+
endif()
36+
endforeach()
37+
38+
if(LEVELDB_VERSION_MAJOR AND LEVELDB_VERSION_MINOR)
39+
set(LEVELDB_VERSION "${LEVELDB_VERSION_MAJOR}.${LEVELDB_VERSION_MINOR}")
40+
endif()
41+
42+
endif()
43+
endif()

0 commit comments

Comments
 (0)