Skip to content

Commit 90afb06

Browse files
smistadsmistad
authored
Pyfast compilation improvements (#222)
* Improved compilation time of pyFAST by building the extension only with cmake and swig. setup.py now only does packaging of the wheel. * Added tests using POCL running with Github Actions * Added python wheel import tests for all OS
1 parent 33e29ea commit 90afb06

File tree

12 files changed

+502
-380
lines changed

12 files changed

+502
-380
lines changed

‎.github/workflows/CI-mac-arm64.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,47 @@ jobs:
119119
run: |
120120
twine upload --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} ${{github.workspace}}/build/python/dist/pyFAST-*.whl
121121
122+
test-python-wheel:
123+
name: Test Python Wheel
124+
needs: [build]
125+
strategy:
126+
fail-fast: false
127+
matrix:
128+
python-version: ['3.8', '3.10', '3.12']
129+
runs-on: macos-14
130+
steps:
131+
- name: Install dependencies
132+
run: |
133+
brew reinstall zlib
134+
brew install openslide pcre libomp
135+
# OpenCL on apple silicon only supports GPU which
136+
# the GitHub nodes don't have, thus we install pocl
137+
# which supports CPUs
138+
brew install pocl
139+
- name: Set up Python ${{ matrix.python-version }}
140+
uses: actions/setup-python@v5
141+
with:
142+
python-version: ${{ matrix.python-version }}
143+
- name: Display Python version
144+
run: python -c "import sys; print(sys.version)"
145+
- name: Download wheel artifact
146+
uses: actions/download-artifact@v4
147+
with:
148+
name: 'Python wheel'
149+
path: ${{github.workspace}}/download/
150+
- name: Create virtual environment and install wheel
151+
run: |
152+
cd ${{github.workspace}}
153+
python -m venv venv # Get error if not using virtual environment for some reason
154+
source venv/bin/activate
155+
python -m pip install --prefer-binary ${{github.workspace}}/download/pyFAST-*.whl
156+
- name: Import FAST with Python
157+
run: |
158+
cd ${{github.workspace}}
159+
source venv/bin/activate
160+
# This will fail because there are no OpenCL plaforms available, thus we add || true
161+
python -c "import fast" || true
162+
122163
# test-cpp:
123164
# name: Run C++ Tests
124165
# needs: [build]

‎.github/workflows/CI-mac-x86_64.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,43 @@ jobs:
115115
run: |
116116
twine upload --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} ${{github.workspace}}/build/python/dist/pyFAST-*.whl
117117
118+
test-python-wheel:
119+
name: Test Python Wheel
120+
needs: [build]
121+
strategy:
122+
fail-fast: false
123+
matrix:
124+
python-version: ['3.6', '3.10', '3.x']
125+
runs-on: macos-13
126+
steps:
127+
- name: Install dependencies
128+
run: |
129+
brew reinstall zlib
130+
brew reinstall libtiff
131+
brew install openslide pcre libomp
132+
- name: Download wheel artifact
133+
uses: actions/download-artifact@v4
134+
with:
135+
name: 'Python wheel'
136+
path: ${{github.workspace}}/download/
137+
- name: Set up Python ${{ matrix.python-version }}
138+
uses: actions/setup-python@v5
139+
with:
140+
python-version: ${{ matrix.python-version }}
141+
- name: Display Python version
142+
run: python -c "import sys; print(sys.version)"
143+
- name: Create virtual environment and install wheel
144+
run: |
145+
cd ${{github.workspace}}
146+
python -m venv venv # Get error if not using virtual environment for some reason
147+
source venv/bin/activate
148+
python -m pip install ${{github.workspace}}/download/pyFAST-*.whl
149+
- name: Import FAST with Python
150+
run: |
151+
cd ${{github.workspace}}
152+
source venv/bin/activate
153+
python -c "import fast"
154+
118155
# test-cpp:
119156
# name: Run C++ Tests
120157
# needs: [build]

‎.github/workflows/CI-ubuntu.yml‎

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: CI Ubuntu 18.04
1+
name: CI Ubuntu
22

33
on:
44
push:
@@ -16,7 +16,7 @@ env:
1616

1717
jobs:
1818
build:
19-
name: Build
19+
name: Build on Ubuntu 18.04
2020
runs-on: ubuntu-20.04
2121

2222
steps:
@@ -170,6 +170,39 @@ jobs:
170170
run: |
171171
twine upload --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} ${{github.workspace}}/build/pyFAST-*.whl
172172
173+
test-python-wheel:
174+
name: Test Python Wheel
175+
needs: [build]
176+
strategy:
177+
fail-fast: false
178+
matrix:
179+
os: [ubuntu-24.04, ubuntu-22.04]
180+
python-version: ['3.8', '3.10', '3.12', '3.x']
181+
runs-on: ${{ matrix.os }}
182+
steps:
183+
- name: Install dependencies
184+
run: |
185+
sudo apt install libpocl2 libopenslide0 xvfb libgl1 libopengl0 libusb-1.0-0 libxcb-xinerama0
186+
- name: Download wheel artifact
187+
uses: actions/download-artifact@v4
188+
with:
189+
name: 'Python wheel'
190+
path: ${{github.workspace}}/download/
191+
- name: Set up Python ${{ matrix.python-version }}
192+
uses: actions/setup-python@v5
193+
with:
194+
python-version: ${{ matrix.python-version }}
195+
- name: Display Python version
196+
run: python -c "import sys; print(sys.version)"
197+
- name: Install wheel
198+
run: |
199+
cd ${{github.workspace}}
200+
python -m pip install ${{github.workspace}}/download/pyFAST-*.whl
201+
- name: Import FAST with Python
202+
run: |
203+
cd ${{github.workspace}}
204+
python -c "import fast"
205+
173206
test-cpp:
174207
name: Run C++ Tests
175208
needs: [build]
@@ -203,20 +236,17 @@ jobs:
203236
cd fast_*
204237
cd fast/bin/
205238
./testFAST ~[visual]
206-
- name: Cleanup
207-
if: always()
208-
run: |
209-
rm -Rf ${{github.workspace}}
210-
rm -Rf $HOME/FAST/kernel_binaries/*
211239
212240
test-python:
213241
name: Run Python Tests
214242
needs: [build]
215243
runs-on: ubuntu-24.04
216244
steps:
245+
- name: Checkout
246+
uses: actions/checkout@v4
217247
- name: Install dependencies
218248
run: |
219-
sudo apt install libopenslide0 libpocl2t64 xvfb libgl1 libopengl0 libusb-1.0-0 libxcb-xinerama0
249+
sudo apt install python3-virtualenv libopenslide0 libpocl2t64 xvfb libgl1 libopengl0 libusb-1.0-0 libxcb-xinerama0
220250
- name: Download artifacts
221251
uses: actions/download-artifact@v4
222252
with:
@@ -231,18 +261,17 @@ jobs:
231261
source venv/bin/activate
232262
pip3 install pytest
233263
pip3 install ${{github.workspace}}/download/pyFAST-*.whl
264+
- name: Download test data
265+
run: |
266+
cd ${{github.workspace}}/tmp/
267+
source venv/bin/activate
268+
python -c "import fast;fast.downloadTestDataIfNotExists()"
234269
- name: Run tests
235270
env:
236271
DISPLAY: ':1'
237272
run: |
238-
Xvfb "$DISPLAY" -screen 0 1024x768x24 &
273+
#Xvfb "$DISPLAY" -screen 0 1024x768x24 &
239274
cd ${{github.workspace}}/tmp/
240275
source venv/bin/activate
241276
pytest ../source/FAST/
242-
- name: Cleanup
243-
if: always()
244-
run: |
245-
rm -Rf ${{github.workspace}}/download/
246-
rm -Rf ${{github.workspace}}/tmp/
247-
rm -Rf $HOME/FAST/kernel_binaries/*
248277

‎.github/workflows/CI-windows.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,3 +224,37 @@ jobs:
224224
run: |
225225
twine upload --username __token__ --password ${{ secrets.PYPI_API_TOKEN }} ${{github.workspace}}/build/python/dist/pyFAST-*.whl
226226
227+
test-python-wheel:
228+
name: Test Python Wheel
229+
needs: [build]
230+
strategy:
231+
fail-fast: false
232+
matrix:
233+
python-version: ['3.6', '3.10', '3.12']
234+
runs-on: windows-2019
235+
steps:
236+
- name: Download wheel artifact
237+
uses: actions/download-artifact@v4
238+
with:
239+
name: 'Python wheel'
240+
path: ${{github.workspace}}/download/
241+
- name: Set up Python ${{ matrix.python-version }}
242+
uses: actions/setup-python@v5
243+
with:
244+
python-version: ${{ matrix.python-version }}
245+
- name: Display Python version
246+
run: python -c "import sys; print(sys.version)"
247+
- name: Install wheel
248+
shell: powershell
249+
run: |
250+
cd ${{github.workspace}}/download/
251+
Get-ChildItem -Path ".\*.whl" | ForEach-Object { python -m pip install $_.FullName }
252+
- name: Import FAST with Python
253+
shell: powershell
254+
run: |
255+
try {
256+
python -c "import fast"
257+
} catch {
258+
Write-Warning "Import failed"
259+
}
260+
exit 0

‎cmake/InstallFAST.cmake‎

Lines changed: 20 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ if(FAST_BUILD_TESTS)
2121
# Install test executable
2222
install(TARGETS testFAST
2323
DESTINATION fast/bin
24-
COMPONENT fast
24+
COMPONENT fast_tests
2525
)
2626
endif()
2727

@@ -30,11 +30,11 @@ endif()
3030
# Install dependency libraries
3131
install(FILES ${PROJECT_BINARY_DIR}/FASTExport.hpp
3232
DESTINATION fast/include
33-
COMPONENT fast
33+
COMPONENT fast_headers
3434
)
3535
install(FILES ${PROJECT_BINARY_DIR}/FASTVersion.hpp
3636
DESTINATION fast/include
37-
COMPONENT fast
37+
COMPONENT fast_headers
3838
)
3939
if(WIN32)
4040
install(DIRECTORY ${PROJECT_BINARY_DIR}/bin/
@@ -53,40 +53,24 @@ elseif(APPLE)
5353
install(DIRECTORY ${PROJECT_BINARY_DIR}/lib/
5454
DESTINATION fast/lib/
5555
COMPONENT fast
56-
FILES_MATCHING PATTERN "*.so*")
56+
FILES_MATCHING PATTERN "*.so*"
57+
PATTERN "_fast.abi3.so" EXCLUDE)
5758
install(SCRIPT cmake/FixRPaths.cmake COMPONENT fast)
58-
if(FAST_SIGN_CODE)
59-
install(CODE "
60-
file(GLOB installedSOs
61-
\"$ENV\{DESTDIR\}/$\{CMAKE_INSTALL_PREFIX\}/fast/lib/*.dylib*\"
62-
\"$ENV\{DESTDIR\}/$\{CMAKE_INSTALL_PREFIX\}/fast/lib/*.so*\"
63-
\"$ENV\{DESTDIR\}/$\{CMAKE_INSTALL_PREFIX\}/fast/bin/*\"
64-
)
65-
66-
foreach(SO $\{installedSOs\})
67-
message(\"-- Signing $\{SO\}\")
68-
execute_process(COMMAND codesign --force --options runtime,library -s \"Developer ID Application: Erik Smistad (85JK2HDMY2)\" --timestamp --signature-size=12000 $\{SO\} RESULT_VARIABLE res OUTPUT_VARIABLE out ERROR_VARIABLE err)
69-
if (NOT res EQUAL 0)
70-
message(\"Unable to sign $\{SO\} - $\{err\}\")
71-
endif ()
72-
endforeach()
73-
message(\"Binaries signed\")
74-
" COMPONENT fast)
75-
endif()
7659
else()
7760
install(DIRECTORY ${PROJECT_BINARY_DIR}/lib/
7861
DESTINATION fast/lib/
7962
COMPONENT fast
80-
FILES_MATCHING PATTERN "*.so*")
63+
FILES_MATCHING PATTERN "*.so*"
64+
PATTERN "_fast.abi3.so" EXCLUDE)
8165
# Fix RPaths on install
82-
install(SCRIPT cmake/FixRPaths.cmake COMPONENT fast)
66+
install(SCRIPT cmake/FixRPaths.cmake COMPONENT fast)
8367
endif()
8468

8569
# Install Qt plugins
8670
if(FAST_MODULE_Visualization)
8771
install(DIRECTORY ${PROJECT_BINARY_DIR}/plugins/
8872
DESTINATION fast/plugins/
89-
COMPONENT fast
73+
COMPONENT fast
9074
)
9175

9276
# Install qt moc
@@ -100,19 +84,20 @@ if(WIN32)
10084
install(FILES ${PROJECT_BINARY_DIR}/bin/idc.exe
10185
PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ
10286
DESTINATION fast/bin
87+
COMPONENT fast
10388
)
10489
endif()
10590
endif()
10691

10792
# Install headers
10893
install(DIRECTORY ${FAST_SOURCE_DIR}
10994
DESTINATION fast/include/FAST/
110-
COMPONENT fast
95+
COMPONENT fast_headers
11196
FILES_MATCHING PATTERN "*.hpp"
11297
)
11398
install(DIRECTORY ${FAST_SOURCE_DIR}
11499
DESTINATION fast/include/FAST/
115-
COMPONENT fast
100+
COMPONENT fast_headers
116101
FILES_MATCHING PATTERN "*.h"
117102
)
118103

@@ -163,28 +148,28 @@ endif()
163148
foreach(INCLUDE_FOLDER ${INCLUDE_FOLDERS})
164149
install(DIRECTORY ${PROJECT_BINARY_DIR}/include/${INCLUDE_FOLDER}/
165150
DESTINATION fast/include/${INCLUDE_FOLDER}/
166-
OPTIONAL
167-
COMPONENT fast
151+
OPTIONAL
152+
COMPONENT fast_headers
168153
FILES_MATCHING PATTERN "*.h"
169154
)
170155
install(DIRECTORY ${PROJECT_BINARY_DIR}/include/${INCLUDE_FOLDER}/
171156
DESTINATION fast/include/${INCLUDE_FOLDER}/
172-
OPTIONAL
173-
COMPONENT fast
157+
OPTIONAL
158+
COMPONENT fast_headers
174159
FILES_MATCHING PATTERN "*.hpp"
175160
)
176161
install(DIRECTORY ${PROJECT_BINARY_DIR}/include/${INCLUDE_FOLDER}/
177162
DESTINATION fast/include/${INCLUDE_FOLDER}/
178-
OPTIONAL
179-
COMPONENT fast
163+
OPTIONAL
164+
COMPONENT fast_headers
180165
FILES_MATCHING REGEX "/[^.]+$" # Files with no extension
181166
)
182167
endforeach()
183168

184169
# Install created headers
185170
install(FILES ${PROJECT_BINARY_DIR}/ProcessObjectList.hpp
186171
DESTINATION fast/include/FAST/
187-
COMPONENT fast
172+
COMPONENT fast_headers
188173
)
189174

190175
# Install OpenCL kernels
@@ -284,7 +269,7 @@ install(FILES ${PROJECT_SOURCE_DIR}/cmake/InstallFiles/README_default.md
284269
if(WIN32)
285270
install(FILES ${PROJECT_SOURCE_DIR}/cmake/InstallFiles/MSVC_redis_files_license.txt
286271
DESTINATION fast/licenses/MSVC/
287-
COMPONENT fast
272+
COMPONENT fast
288273
)
289274
endif()
290275

‎cmake/Macros.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ macro (fast_add_example NAME)
9999
target_link_libraries(${NAME} FAST)
100100
install(TARGETS ${NAME}
101101
DESTINATION fast/bin
102-
COMPONENT fast
102+
COMPONENT fast_examples
103103
)
104104
if(WIN32)
105105
file(APPEND ${PROJECT_BINARY_DIR}/runAllExamples.bat "bin\\${NAME}.exe\r\n")

0 commit comments

Comments
 (0)