Skip to content

Commit e1c8e49

Browse files
dolfim-ibmdolfim-ibm
authored
feat: build using system deps (#33)
 --------- Signed-off-by: Michele Dolfi <[email protected]>
1 parent 8d903ba commit e1c8e49

File tree

13 files changed

+294
-153
lines changed

13 files changed

+294
-153
lines changed

‎.github/scripts/build_rhel.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
3+
set -e # trigger failure on error - do not remove!
4+
set -x # display command on output
5+
6+
# Build the sdist
7+
poetry build -f sdist
8+
9+
# Compile the wheel from sdist in centos stream
10+
11+
docker build -f - . <<EOF
12+
FROM quay.io/centos/centos:stream9
13+
RUN dnf config-manager --set-enabled crb
14+
# RUN dnf copr -y enable cheimes/deepsearch-glm rhel-9-x86_64
15+
RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
16+
&& dnf clean all
17+
RUN dnf install -y --nodocs \
18+
gcc gcc-c++ git make cmake pkgconfig glibc-devel \
19+
python3.11 python3.11-pip python3.11-devel \
20+
libjpeg-turbo-devel libpng-devel qpdf-devel json-devel utf8cpp-devel zlib-devel \
21+
loguru-devel \
22+
&& dnf clean all
23+
24+
RUN mkdir /src
25+
COPY ./dist/*.tar.gz /src/
26+
27+
RUN USE_SYSTEM_DEPS=ON pip3.11 install /src/docling_parse*.tar.gz \
28+
&& python3.11 -c 'from docling_parse.docling_parse import pdf_parser'
29+
30+
COPY ./tests /src/tests
31+
RUN cd /src \
32+
&& pip3.11 install pytest \
33+
&& pytest
34+
EOF

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,6 @@ jobs:
2121
uses: ./.github/workflows/checks.yml
2222
build-wheels:
2323
uses: ./.github/workflows/wheels.yml
24+
rhel-build:
25+
uses: ./.github/workflows/rhel.yml
2426

‎.github/workflows/rhel.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
on:
2+
workflow_call:
3+
4+
jobs:
5+
run-checks:
6+
runs-on: ubuntu-latest
7+
steps:
8+
- uses: actions/checkout@v4
9+
- uses: ./.github/actions/setup-poetry
10+
- name: Run build in docker
11+
run: ./.github/scripts/build_rhel.sh
12+
shell: bash

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,6 @@ pyrightconfig.json
287287
[Ll]ib
288288
[Ll]ib64
289289
[Ll]ocal
290-
[Ss]cripts
291290
pyvenv.cfg
292291
pip-selfcheck.json
293292

‎CMakeLists.txt‎

Lines changed: 38 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -84,63 +84,73 @@ message(STATUS "cxx-flags : " ${CMAKE_CXX_FLAGS})
8484
message(STATUS "cxx-standard: " ${CMAKE_CXX_STANDARD})
8585

8686
# ************************
87-
# *** libraries ***
87+
# *** dependencies ***
8888
# ************************
8989

90-
if(NOT EXISTS ${EXTERNALS_PREFIX_PATH})
91-
execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH})
92-
execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/bin)
93-
execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/lib)
94-
execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/include)
95-
execute_process(COMMAND mkdir ${EXTERNALS_PREFIX_PATH}/resources)
90+
option(USE_SYSTEM_DEPS OFF "If enabled, the build will find and link to system dependencies, otherwise they are sourced from the original repos and compiled on the fly.")
91+
92+
# prepare setup for building external packages
93+
if(NOT USE_SYSTEM_DEPS)
94+
if(NOT EXISTS ${EXTERNALS_PREFIX_PATH})
95+
file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH})
96+
file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/bin)
97+
file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/lib)
98+
file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/include)
99+
file(MAKE_DIRECTORY ${EXTERNALS_PREFIX_PATH}/resources)
100+
endif()
96101
endif()
97102

98-
include(cmake/subdirlist.cmake)
99103

104+
# include dependencies
100105
include(cmake/extlib_json.cmake)
101106
include(cmake/extlib_utf8.git.cmake)
102107
include(cmake/extlib_jpeg.cmake)
103-
# include(cmake/extlib_qpdf_v10.cmake)
104108
include(cmake/extlib_qpdf_v11.cmake)
105-
# include(cmake/extlib_pybind11.cmake)
106109

110+
# aggregate the targets created by the dependencies
111+
set(DEPENDENCIES qpdf jpeg utf8 json)
112+
113+
114+
# ************************
115+
# *** libraries ***
116+
# ************************
117+
118+
# define LIB_LINK and OS_DEPENDENCIES
107119
include(cmake/os_opts.cmake)
120+
list(APPEND DEPENDENCIES ${OS_DEPENDENCIES})
108121

109122
# ***********************
110123
# *** includes ***
111124
# ***********************
112125

113-
include_directories(${EXTERNALS_PREFIX_PATH}/include)
114-
115-
subdirlist(EXTERNALS_INCLUDES ${EXTERNALS_PREFIX_PATH}/include)
116-
include_directories(${EXTERNALS_INCLUDES})
117-
118-
message(STATUS "subdirs: " ${SUBDIRS})
126+
# define subdirlist utility
127+
include(cmake/subdirlist.cmake)
119128

120129
include_directories(${TOPLEVEL_PREFIX_PATH}/src)
121130

122131
subdirlist(SUBDIRS ${TOPLEVEL_PREFIX_PATH}/src)
132+
message(STATUS "subdirs: " ${SUBDIRS})
123133
include_directories(${SUBDIRS})
124134

125135
# **********************
126136
# *** Executables ***
127137
# **********************
128138

129139
add_executable(parse.exe "${TOPLEVEL_PREFIX_PATH}/app/parse.cpp")
130-
131140
set_property(TARGET parse.exe PROPERTY CXX_STANDARD 20)
132-
133-
target_link_libraries(parse.exe ${LIB_LINK})
141+
add_dependencies(parse.exe ${DEPENDENCIES})
142+
target_include_directories(parse.exe INTERFACE ${DEPENDENCIES})
143+
target_link_libraries(parse.exe ${DEPENDENCIES} ${LIB_LINK})
134144

135145
# **********************
136146
# *** Libraries ***
137147
# **********************
138148

139-
add_library(parse STATIC "${TOPLEVEL_PREFIX_PATH}/app/parse.cpp")
140-
141-
set_target_properties(parse PROPERTIES POSITION_INDEPENDENT_CODE ON)
142-
143-
target_link_libraries(parse ${LIB_LINK})
149+
add_library(libparse STATIC "${TOPLEVEL_PREFIX_PATH}/app/parse.cpp")
150+
add_dependencies(libparse ${DEPENDENCIES})
151+
target_include_directories(libparse INTERFACE ${DEPENDENCIES})
152+
set_target_properties(libparse PROPERTIES POSITION_INDEPENDENT_CODE ON)
153+
target_link_libraries(libparse ${DEPENDENCIES} ${LIB_LINK})
144154

145155
# ***************************
146156
# *** Python-binding ***
@@ -151,11 +161,10 @@ target_link_libraries(parse ${LIB_LINK})
151161
find_package(pybind11 CONFIG REQUIRED)
152162

153163
pybind11_add_module(docling_parse "${TOPLEVEL_PREFIX_PATH}/app/pybind_parse.cpp")
154-
155-
add_dependencies(docling_parse json utf8) # we have to list the headers-only deps here
156-
target_compile_definitions(docling_parse PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})
157-
158-
target_link_libraries(docling_parse PRIVATE parse)
164+
add_dependencies(docling_parse libparse)
165+
target_include_directories(docling_parse INTERFACE ${DEPENDENCIES})
166+
target_compile_definitions(docling_parse PRIVATE VERSION_INFO=${CMAKE_PROJECT_VERSION})
167+
target_link_libraries(docling_parse PRIVATE libparse)
159168

160169
# *****************
161170
# *** Install ***

‎app/parse.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ Email: {taa, dol, cau, ttt}@zurich.ibm.com
1212
Date: 18/08/2017
1313
**********************************************************/
1414

15-
#include <utf8/utf8.h>
15+
#include <utf8.h>
1616

1717
#include "include_libs.h"
1818

‎build.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,15 @@ def run(cmd: List[str], cwd: str="./"):
4040

4141
def build_local(num_threads: int):
4242

43+
USE_SYSTEM_DEPS = os.getenv("USE_SYSTEM_DEPS", "OFF")
44+
45+
4346
print("python prefix: ", sys.exec_prefix)
4447
print("python executable: ", sys.executable)
4548
config_cmd = [
4649
"cmake",
4750
"-B", f"{BUILD_DIR}",
51+
f"-DUSE_SYSTEM_DEPS={USE_SYSTEM_DEPS}",
4852
f"-DPYTHON_EXECUTABLE={sys.executable}",
4953
]
5054
config_cmd.extend(get_pybind11_cmake_args())

‎cmake/extlib_jpeg.cmake‎

Lines changed: 43 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,54 @@
11

22
message(STATUS "entering in extlib_jpeg.cmake")
33

4-
include(ExternalProject)
5-
include(CMakeParseArguments)
4+
set(ext_name "jpeg")
65

7-
set(JPEG_URL https://github.com/libjpeg-turbo/libjpeg-turbo.git)
8-
set(JPEG_TAG 3.0.3)
6+
if(USE_SYSTEM_DEPS)
7+
find_package(PkgConfig)
8+
pkg_check_modules(libjpeg REQUIRED IMPORTED_TARGET libjpeg)
99

10-
ExternalProject_Add(extlib_jpeg
11-
PREFIX extlib_jpeg
10+
add_library(${ext_name} ALIAS PkgConfig::libjpeg)
11+
#set_target_properties(${ext_name} PROPERTIES INTERFACE_LINK_LIBRARIES "${libjpeg_LIBRARIES}")
12+
#set_target_properties(${ext_name} PROPERTIES INTERFACE_LINK_DIRECTORIES "${libjpeg_LIBRARY_DIRS}")
13+
#set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${libjpeg_INCLUDEDIR}")
1214

13-
UPDATE_COMMAND ""
14-
GIT_REPOSITORY ${JPEG_URL}
15-
GIT_TAG ${JPEG_TAG}
15+
else()
16+
include(ExternalProject)
17+
include(CMakeParseArguments)
1618

17-
BUILD_ALWAYS OFF
19+
set(JPEG_URL https://github.com/libjpeg-turbo/libjpeg-turbo.git)
20+
set(JPEG_TAG 3.0.3)
1821

19-
INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
22+
ExternalProject_Add(extlib_jpeg
2023

21-
CMAKE_ARGS \\
22-
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \\
23-
-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \\
24-
-DCMAKE_C_FLAGS=${ENV_ARCHFLAGS} \\
25-
-DARMV8_BUILD=ON \\
26-
-DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\
27-
-DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH}
24+
PREFIX extlib_jpeg
2825

29-
BUILD_IN_SOURCE ON
30-
LOG_DOWNLOAD ON
31-
# LOG_BUILD ON
32-
)
26+
UPDATE_COMMAND ""
27+
GIT_REPOSITORY ${JPEG_URL}
28+
GIT_TAG ${JPEG_TAG}
3329

34-
add_library(jpeg STATIC IMPORTED)
35-
set_target_properties(jpeg PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libjpeg.a)
36-
add_dependencies(jpeg extlib_jpeg)
30+
BUILD_ALWAYS OFF
31+
32+
INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
33+
34+
CMAKE_ARGS \\
35+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \\
36+
-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \\
37+
-DCMAKE_C_FLAGS=${ENV_ARCHFLAGS} \\
38+
-DARMV8_BUILD=ON \\
39+
-DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\
40+
-DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH}
41+
42+
BUILD_IN_SOURCE ON
43+
LOG_DOWNLOAD ON
44+
# LOG_BUILD ON
45+
)
46+
47+
add_library(${ext_name} STATIC IMPORTED)
48+
add_dependencies(${ext_name} extlib_jpeg)
49+
set_target_properties(${ext_name} PROPERTIES
50+
IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libjpeg.a
51+
INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include
52+
)
53+
54+
endif()

‎cmake/extlib_json.cmake‎

Lines changed: 33 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,44 @@
11

22
message(STATUS "entering in extlib_json.cmake")
33

4-
include(ExternalProject)
5-
include(CMakeParseArguments)
4+
set(ext_name "json")
65

7-
set(JSON_URL https://github.com/nlohmann/json.git)
8-
set(JSON_TAG v3.11.3)
6+
if(USE_SYSTEM_DEPS)
7+
# this will define the nlohmann_json::nlohmann_json target
8+
find_package(nlohmann_json REQUIRED)
99

10-
ExternalProject_Add(extlib_json
11-
PREFIX extlib_json
10+
add_library(${ext_name} INTERFACE IMPORTED)
11+
add_dependencies(${ext_name} nlohmann_json::nlohmann_json)
1212

13-
GIT_REPOSITORY ${JSON_URL}
14-
GIT_TAG ${JSON_TAG}
13+
else()
1514

16-
UPDATE_COMMAND ""
17-
CONFIGURE_COMMAND ""
15+
include(ExternalProject)
16+
include(CMakeParseArguments)
1817

19-
BUILD_COMMAND ""
20-
BUILD_ALWAYS OFF
18+
set(JSON_URL https://github.com/nlohmann/json.git)
19+
set(JSON_TAG v3.11.3)
2120

22-
INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
23-
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/include/ ${EXTERNALS_PREFIX_PATH}/include/
21+
ExternalProject_Add(extlib_json
22+
23+
PREFIX extlib_json
24+
25+
GIT_REPOSITORY ${JSON_URL}
26+
GIT_TAG ${JSON_TAG}
27+
28+
UPDATE_COMMAND ""
29+
CONFIGURE_COMMAND ""
30+
31+
BUILD_COMMAND ""
32+
BUILD_ALWAYS OFF
33+
34+
INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
35+
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/include/ ${EXTERNALS_PREFIX_PATH}/include/
2436
)
2537

26-
add_library(json INTERFACE)
27-
add_custom_target(install_extlib_json DEPENDS extlib_json)
28-
add_dependencies(json install_extlib_json)
38+
add_library(${ext_name} INTERFACE IMPORTED)
39+
add_dependencies(${ext_name} extlib_json)
40+
set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include)
41+
42+
endif()
43+
44+

0 commit comments

Comments
 (0)