Skip to content

Commit a2ecab7

Browse files
NightTsarinaNightTsarina
authored
Merge pull request twmht#2 from NightTsarina/fix-github-actions
Fixes to test framework and GitHub workflow
2 parents 0828318 + 2f942de commit a2ecab7

File tree

4 files changed

+190
-103
lines changed

4 files changed

+190
-103
lines changed

‎.github/workflows/build.yml‎

Lines changed: 68 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,131 +1,99 @@
11
# vim:ts=2:sw=2:et:ai:sts=2
22
name: 'Build'
33

4-
on: ['push', 'pull_request']
4+
on:
5+
push:
56

67
jobs:
7-
test:
8-
name: 'Test'
8+
# Build the RocksDB C library and cache the result.
9+
librocksdb:
10+
name: 'Build librocksdb'
911
runs-on: ${{ matrix.os }}
12+
env:
13+
LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }}
1014
strategy:
1115
matrix:
1216
os: [ubuntu-latest]
13-
#py_ver: ['3.7', '3.8', '3.9']
14-
#rocksdb_ver: ['v6.14.6', 'v6.11.4', 'v5.17.2']
15-
py_ver: ['3.9']
16-
rocksdb_ver: ['v6.14.6']
17+
rocksdb_ver: ['v6.14.6', 'v6.11.4']
1718

1819
steps:
19-
- uses: actions/checkout@v2
20-
name: 'Checkout source repository'
21-
22-
- uses: actions/setup-python@v2
23-
name: 'Set up Python ${{ matrix.py_ver }}'
20+
- uses: actions/cache@v2
21+
id: cache-librocksdb
2422
with:
25-
python-version: ${{ matrix.py_ver }}
23+
key: ${{ matrix.os }}-librocksdb-${{ matrix.rocksdb_ver }}
24+
path: ${{ env.LIBROCKSDB_PATH }}
2625

2726
- name: 'Install dependencies'
28-
run: |
29-
sudo apt install -y libsnappy-dev libbz2-dev liblz4-dev libz-dev libgflags-dev libzstd-dev
30-
31-
- name: 'Install RocksDB ${{ matrix.rocksdb_ver }}'
32-
run: |
33-
pushd /opt &&
34-
git clone https://github.com/facebook/rocksdb &&
35-
cd rocksdb &&
36-
git reset --hard ${{ matrix.rocksdb_ver }} &&
27+
if: steps.cache-librocksdb.outputs.cache-hit != 'true'
28+
run: >
29+
sudo apt install -y libsnappy-dev libbz2-dev liblz4-dev libz-dev
30+
libgflags-dev libzstd-dev
31+
32+
- name: 'Clone & build RocksDB ${{ matrix.rocksdb_ver }}'
33+
if: steps.cache-librocksdb.outputs.cache-hit != 'true'
34+
run: >
35+
git clone https://github.com/facebook/rocksdb --depth 1
36+
--branch ${{ matrix.rocksdb_ver }} ${{ env.LIBROCKSDB_PATH }} &&
37+
pushd ${{ env.LIBROCKSDB_PATH }} &&
3738
CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4 &&
38-
make shared_lib &&
39-
sudo make install-shared &&
4039
popd
4140
42-
- name: Install python-rocksdb
43-
run: |
44-
sudo python3 setup.py install
45-
46-
- name: Run tests
47-
run: |
48-
python3 setup.py test
49-
50-
build_wheels:
51-
name: 'Build wheels'
52-
runs-on: ubuntu-latest
41+
test:
42+
name: 'Build and test python-rocksdb'
43+
needs: librocksdb
44+
runs-on: ${{ matrix.os }}
45+
env:
46+
LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }}
47+
strategy:
48+
matrix:
49+
os: [ubuntu-latest]
50+
py_ver: ['3.7', '3.8', '3.9']
51+
rocksdb_ver: ['v6.14.6', 'v6.11.4']
5352

5453
steps:
5554
- uses: actions/checkout@v2
5655
name: 'Checkout source repository'
5756

5857
- uses: actions/setup-python@v2
59-
name: 'Set up Python 3.9'
58+
name: 'Set up Python ${{ matrix.py_ver }}'
6059
with:
61-
python-version: '3.9'
62-
63-
- name: 'Install cibuildwheel'
64-
run: |
65-
python3 -m pip install cibuildwheel==1.7.1
60+
python-version: ${{ matrix.py_ver }}
6661

67-
- name: 'Build wheels'
68-
run: |
69-
python3 -m cibuildwheel --output-dir dist
70-
env:
71-
ROCKSDB_VERSION: 'v6.14.6'
72-
CIBW_MANYLINUX_X86_64_IMAGE: 'manylinux2014'
73-
CIBW_BUILD: 'cp3*'
74-
CIBW_SKIP: '*-manylinux_i686'
75-
CIBW_TEST_REQUIRES: '.[test] pytest'
76-
CIBW_TEST_COMMAND: 'python3 -m pytest {project}/rocksdb/tests'
77-
CIBW_BEFORE_BUILD: |
78-
yum install -y bzip2-devel lz4-devel snappy-devel zlib-devel &&
79-
pushd /opt &&
80-
git clone https://github.com/facebook/rocksdb &&
81-
cd rocksdb &&
82-
git reset --hard $ROCKSDB_VERSION &&
83-
CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4 &&
84-
make install-shared &&
85-
popd
62+
- name: 'Install C libraries'
63+
# XXX(Tina): The non-development versions are sufficient, but package
64+
# names are difficult to predict.
65+
run: >
66+
sudo apt install -y libsnappy-dev libbz2-dev liblz4-dev libz-dev
67+
libgflags-dev libzstd-dev
8668
87-
- uses: actions/upload-artifact@v2
88-
name: 'Upload build artifacts'
69+
# Recover the pre-built C library.
70+
- uses: actions/cache@v2
71+
id: cache-librocksdb
8972
with:
90-
path: 'dist/*.whl'
91-
73+
key: ${{ matrix.os }}-librocksdb-${{ matrix.rocksdb_ver }}
74+
path: ${{ env.LIBROCKSDB_PATH }}
9275

93-
build_sdist:
94-
name: 'Build source distribution'
95-
runs-on: 'ubuntu-latest'
96-
steps:
97-
- uses: actions/checkout@v2
98-
name: 'Checkout source repository'
99-
100-
- uses: actions/setup-python@v2
101-
name: 'Set up Python 3.9'
102-
with:
103-
python-version: '3.9'
104-
105-
- name: 'Build sdist'
76+
- name: 'Install RocksDB ${{ matrix.rocksdb_ver }}'
77+
if: steps.cache-librocksdb.outputs.cache-hit == 'true'
78+
# DO NOT FORGET to call `ldconfig`!
10679
run: |
107-
python3 setup.py sdist
108-
109-
- uses: actions/upload-artifact@v2
110-
name: 'Upload build artifacts'
111-
with:
112-
path: 'dist/*.tar.gz'
80+
pushd ${{ env.LIBROCKSDB_PATH }} &&
81+
sudo make install-shared &&
82+
sudo ldconfig &&
83+
popd
11384
85+
- name: Build and install python-rocksdb
86+
# Use `pip install` instead of `setup.py` so build-dependencies from
87+
# `pyproject.toml` are installed, in particular `Cython`, without which
88+
# the build fails in confusing ways.
89+
run: |
90+
python3 -m pip install '.[test]'
11491
115-
# upload_pypi:
116-
# name: 'Upload packages'
117-
# needs: ['build_wheels', 'build_sdist']
118-
# runs-on: 'ubuntu-latest'
119-
# if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
120-
# steps:
121-
# - uses: actions/download-artifact@v2
122-
# name: 'Download artifacts'
123-
# with:
124-
# name: 'artifact'
125-
# path: 'dist'
126-
#
127-
# - uses: pypa/gh-action-pypi-publish@master
128-
# name: 'Publish built packages'
129-
# with:
130-
# user: '__token__'
131-
# password: '${{ secrets.PYPI_API_TOKEN }}'
92+
- name: Run tests
93+
# Use `--pyargs` to interpret parameter as module to import, not as a
94+
# path, and do not use `python3 -m pytest`. This way we prevent
95+
# importing the module from the current directory instead of the
96+
# installed package, and failing when it cannot find the shared
97+
# library.
98+
run: |
99+
pytest --pyargs rocksdb

‎.github/workflows/dist.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# vim:ts=2:sw=2:et:ai:sts=2
2+
name: 'Build distribution'
3+
4+
on:
5+
# Only run when pushing to main/merging PRs.
6+
push:
7+
branches:
8+
- main
9+
10+
jobs:
11+
build_wheels:
12+
name: 'Build wheels'
13+
runs-on: ${{ matrix.os }}
14+
env:
15+
LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }}
16+
strategy:
17+
matrix:
18+
os: [ubuntu-latest]
19+
rocksdb_ver: ['v6.14.6']
20+
21+
steps:
22+
- uses: actions/checkout@v2
23+
name: 'Checkout source repository'
24+
25+
- uses: actions/setup-python@v2
26+
name: 'Set up Python 3.9'
27+
with:
28+
python-version: '3.9'
29+
30+
- name: 'Install cibuildwheel'
31+
run: |
32+
python3 -m pip install cibuildwheel==1.7.1
33+
34+
- name: 'Build wheels'
35+
run: |
36+
python3 -m cibuildwheel --output-dir dist
37+
env:
38+
CIBW_MANYLINUX_X86_64_IMAGE: 'manylinux2014'
39+
CIBW_BUILD: 'cp3*'
40+
CIBW_SKIP: '*-manylinux_i686'
41+
# Install python package and test-deps.
42+
CIBW_TEST_REQUIRES: '.[test] pytest'
43+
# Use `--pyargs` to interpret parameter as module to import, not as a
44+
# path, and do not use `python3 -m pytest`. This way we prevent
45+
# importing the module from the current directory instead of the
46+
# installed package, and failing when it cannot find the shared
47+
# library.
48+
CIBW_TEST_COMMAND: 'pytest --pyargs rocksdb'
49+
# Avoid re-building the C library in every iteration by testing for
50+
# the build directory.
51+
CIBW_BEFORE_BUILD: >
52+
yum install -y bzip2-devel lz4-devel snappy-devel zlib-devel
53+
python3-Cython &&
54+
test -d ${{ env.LIBROCKSDB_PATH }} || (
55+
git clone https://github.com/facebook/rocksdb --depth 1
56+
--branch ${{ matrix.rocksdb_ver }} ${{ env.LIBROCKSDB_PATH }} &&
57+
cd ${{ env.LIBROCKSDB_PATH }} &&
58+
CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4
59+
) &&
60+
pushd ${{ env.LIBROCKSDB_PATH }} &&
61+
make install-shared &&
62+
ldconfig &&
63+
popd
64+
65+
- uses: actions/upload-artifact@v2
66+
name: 'Upload build artifacts'
67+
with:
68+
path: 'dist/*.whl'
69+
70+
build_sdist:
71+
name: 'Build source distribution'
72+
runs-on: 'ubuntu-latest'
73+
steps:
74+
- uses: actions/checkout@v2
75+
name: 'Checkout source repository'
76+
77+
- uses: actions/setup-python@v2
78+
name: 'Set up Python 3.9'
79+
with:
80+
python-version: '3.9'
81+
82+
- name: 'Build sdist'
83+
run: |
84+
python3 setup.py sdist
85+
86+
- uses: actions/upload-artifact@v2
87+
name: 'Upload build artifacts'
88+
with:
89+
path: 'dist/*.tar.gz'
90+
91+
92+
# upload_pypi:
93+
# name: 'Upload packages'
94+
# needs: ['build_wheels', 'build_sdist']
95+
# runs-on: 'ubuntu-latest'
96+
# if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
97+
# steps:
98+
# - uses: actions/download-artifact@v2
99+
# name: 'Download artifacts'
100+
# with:
101+
# name: 'artifact'
102+
# path: 'dist'
103+
#
104+
# - uses: pypa/gh-action-pypi-publish@master
105+
# name: 'Publish built packages'
106+
# with:
107+
# user: '__token__'
108+
# password: '${{ secrets.PYPI_API_TOKEN }}'

‎setup.cfg‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ where = .
4141
doc =
4242
sphinx
4343
sphinx_rtd_theme
44+
test =
45+
pytest
4446

4547
[build_sphinx]
4648
source-dir = docs

‎tox.ini‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,24 @@
11
[tox]
22
envlist = py35,py36,py37,py38,py39
33
minversion = 2.0
4+
isolated_build = True
5+
skipsdist = True
46

57
[testenv]
6-
deps = pytest
7-
commands = pytest {posargs:rocksdb/tests}
8+
# Install the module in `.` and its `test` extra dependencies from
9+
# setup.cfg.
10+
deps =
11+
.[test]
12+
changedir = /
13+
# Point directly to the installed package, and do not use `python3 -m pytest`.
14+
# This way we prevent importing the module from the current directory instead
15+
# of the installed package, and failing when it cannot find the shared library.
16+
commands = pytest {envsitepackagesdir}/rocksdb
817

918
[testenv:docs]
1019
deps = .[doc]
1120
commands = python3 setup.py build_sphinx -W
1221

1322
[pytest]
14-
addopts = --verbose
23+
addopts = --verbose --pyargs
1524
norecursedirs = .tox

0 commit comments

Comments
 (0)