Skip to content

Commit cd92151

Browse files
ihnortonihnorton
committed
Bump to TileDB 2.18.0, fix build system issues
1 parent 85f0a30 commit cd92151

File tree

10 files changed

+71
-56
lines changed

10 files changed

+71
-56
lines changed

‎.github/workflows/ci.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ jobs:
1414
shell: bash
1515
strategy:
1616
matrix:
17-
os: [ubuntu-latest, macos-11, windows-latest]
17+
os: [ubuntu-latest, macos-12, windows-latest]
1818
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
1919
fail-fast: false
2020
env:
2121
# 11.7 necessary due to: https://github.com/actions/setup-python/issues/682#issuecomment-1604261330
22-
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-11' && contains(fromJSON('["3.7", "3.8"]'), matrix.python-version) && '11.7' || '10.15' }}
23-
#MACOSX_DEPLOYMENT_TARGET: "10.15"
22+
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-12' && contains(fromJSON('["3.7", "3.8"]'), matrix.python-version) && '11.7' || '11' }}
23+
#MACOSX_DEPLOYMENT_TARGET: "10.11"
2424
steps:
2525
- name: Checkout TileDB-Py `dev`
2626
uses: actions/checkout@v3
@@ -31,11 +31,11 @@ jobs:
3131
python-version: ${{ matrix.python-version }}
3232

3333
- run: rm -rf $RUNNER_TOOL_CACHE/Python/3.7.17
34-
if: matrix.os == 'macos-11' && matrix.python-version == '3.7'
34+
if: matrix.os == 'macos-12' && matrix.python-version == '3.7'
3535
- uses: actions/setup-python@v4
3636
with:
3737
python-version: 3.7.17
38-
if: matrix.os == 'macos-11' && matrix.python-version == '3.7'
38+
if: matrix.os == 'macos-12' && matrix.python-version == '3.7'
3939

4040
- name: Print Python version
4141
run: |
@@ -53,7 +53,7 @@ jobs:
5353
# - https://github.com/actions/runner-images/pull/7125
5454
- name: "Install homebrew dependencies"
5555
run: brew install pkg-config
56-
if: matrix.os == 'macos-11'
56+
if: matrix.os == 'macos-12'
5757

5858
- name: "Install dependencies"
5959
run: python -m pip install --upgrade -r misc/requirements_ci.txt
@@ -93,7 +93,7 @@ jobs:
9393
python setup.py build_ext --inplace
9494
python setup.py develop
9595
fi
96-
if: matrix.os == 'macos-11'
96+
if: matrix.os == 'macos-12'
9797

9898
- name: "Download TileDB From Tarball And Build TileDB-Py (Linux)"
9999
run: |

‎.github/workflows/daily-test-build-numpy.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
shell: bash
1717
strategy:
1818
matrix:
19-
os: [ubuntu-latest, macos-11, windows-latest]
19+
os: [ubuntu-latest, macos-12, windows-latest]
2020
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
2121
include:
2222
# https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg
@@ -34,7 +34,7 @@ jobs:
3434
env:
3535
# 11.7 necessary due to: https://github.com/actions/setup-python/issues/682#issuecomment-1604261330
3636
#MACOSX_DEPLOYMENT_TARGET: "10.15"
37-
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-11' && contains(fromJson('["3.7", "3.8"]'), matrix.python-version) && '11.7' || '10.15' }}
37+
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-11' && contains(fromJson('["3.7", "3.8"]'), matrix.python-version) && '11.7' || '11' }}
3838
steps:
3939
- name: Checkout TileDB-Py `dev`
4040
uses: actions/checkout@v3

‎.github/workflows/daily-test-build.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,14 @@ jobs:
3131
runs-on: ${{ matrix.os }}
3232
strategy:
3333
matrix:
34-
os: [ubuntu-latest, macos-11, windows-latest]
34+
os: [ubuntu-latest, macos-12, windows-latest]
3535
libtiledb_version: ["dev", "release-2.16"]
3636
uninstall_pandas: [true, false]
3737
fail-fast: false
3838

3939
env:
4040
TILEDB_VERSION: ${{ matrix.libtiledb_version }}
41-
MACOSX_DEPLOYMENT_TARGET: "10.15"
41+
MACOSX_DEPLOYMENT_TARGET: "11"
4242

4343
steps:
4444
- name: Set up Python
@@ -59,7 +59,7 @@ jobs:
5959
# - https://github.com/actions/runner-images/pull/7125
6060
- name: "Install homebrew dependencies"
6161
run: brew install pkg-config
62-
if: matrix.os == 'macos-11'
62+
if: matrix.os == 'macos-12'
6363

6464
- name: Checkout TileDB-Py `dev`
6565
uses: actions/checkout@v3

‎azure-pipelines.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
stages:
2-
- template: ./misc/azure-ci.yml
2+
#- template: ./misc/azure-ci.yml
33
- template: ./misc/azure-release.yml
44
#- template: ./misc/azure-publish.yml

‎misc/azure-ci.yml

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ stages:
88
strategy:
99
matrix:
1010
mac:
11-
imageName: "macOS-11"
11+
imageName: "macOS-12"
1212
python.version: "3.10"
13-
MACOSX_DEPLOYMENT_TARGET: 10.15
13+
MACOSX_DEPLOYMENT_TARGET: 11
1414
windows:
1515
imageName: "windows-latest"
1616
python.version: "3.10"
@@ -56,16 +56,14 @@ stages:
5656
set TILEDB_FORCE_ALL_DEPS=ON
5757
# vcvarsall is necessary so that numpy uses the correct compiler
5858
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
59-
python setup.py build_ext --inplace
60-
python setup.py install
59+
pip install -e -vv .
6160
displayName: "Build TileDB and TileDB-Py extension (Windows)"
6261
condition: eq(variables['Agent.OS'], 'Windows_NT')
6362
6463
- bash: |
6564
unset SYSTEM
6665
set -xeo pipefail
67-
python setup.py build_ext --inplace --werror
68-
python setup.py install
66+
pip install -e -vv .
6967
displayName: "Build TileDB and TileDB-Py extension (POSIX)"
7068
condition: ne(variables['Agent.OS'], 'Windows_NT')
7169

‎misc/azure-release.yml

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ stages:
66
LIBTILEDB_VERSION: dev
77
LIBTILEDB_SHA: dev
88
${{ else }}:
9-
TILEDBPY_VERSION: 0.23.4
9+
TILEDBPY_VERSION: 0.24.0
1010
# NOTE: *must* update both LIBTILEDB_VERSION and LIBTILEDB_SHA
11-
LIBTILEDB_VERSION: 2.17.4
12-
LIBTILEDB_SHA: a1f648ebef5e520f72e51a4f23d5b37921105d96
11+
LIBTILEDB_VERSION: 2.18.0
12+
LIBTILEDB_SHA: 71ca8b4ddc615e2ad124fb76427df65667850e5e
1313
LIBTILEDB_REPO: https://github.com/TileDB-Inc/TileDB
1414
TILEDB_SRC: "$(Build.Repository.Localpath)/tiledb_src"
1515
TILEDB_BUILD: "$(Build.Repository.Localpath)/tiledb_build"
@@ -26,12 +26,12 @@ stages:
2626
strategy:
2727
matrix:
2828
macOS_libtiledb:
29-
imageName: "macOS-11"
29+
imageName: "macOS-12"
3030
CMAKE_OSX_ARCHITECTURES: "x86_64"
31-
MACOSX_DEPLOYMENT_TARGET: 10.15
31+
MACOSX_DEPLOYMENT_TARGET: 11
3232
TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-x86_64"
3333
macOS_libtiledb_arm64:
34-
imageName: "macOS-11"
34+
imageName: "macOS-12"
3535
CMAKE_OSX_ARCHITECTURES: "arm64"
3636
MACOSX_DEPLOYMENT_TARGET: 11
3737
TILEDB_GCS: OFF
@@ -74,14 +74,15 @@ stages:
7474
CIBW_BUILD_VERBOSITY: 3
7575
CIBW_ARCHS_MACOS: ""
7676
macOS_py:
77-
imageName: "macOS-11"
78-
MACOSX_DEPLOYMENT_TARGET: 10.15
77+
imageName: "macOS-12"
78+
MACOSX_DEPLOYMENT_TARGET: 11
7979
TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-x86_64"
8080
CIBW_ARCHS_MACOS: "x86_64"
8181
CIBW_SKIP: "cp27-* cp35-* cp36-* pp*"
82+
CIBW_TEST_SKIP: "cp37-*"
8283
CIBW_BUILD_VERBOSITY: 3
8384
macOS_arm64_py:
84-
imageName: "macOS-11"
85+
imageName: "macOS-12"
8586
MACOSX_DEPLOYMENT_TARGET: 11
8687
TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-arm64"
8788
CIBW_BUILD_VERBOSITY: 3
@@ -152,7 +153,7 @@ stages:
152153
python -c "import os; print(os.environ.get('CIBW_ENVIRONMENT', None))"
153154
git rev-parse HEAD
154155
python3 -m pip install --upgrade -r requirements_dev.txt
155-
python3 setup.py sdist --dist-dir wheelhouse
156+
python3 -m build . --sdist --outdir wheelhouse
156157
pip3 install cibuildwheel==${USE_CIBW_VERSION}
157158
cibuildwheel --output-dir wheelhouse .
158159
displayName: "Build and test wheels (Linux)"
@@ -162,6 +163,7 @@ stages:
162163
set -xe pipefail
163164
164165
export TILEDB_WHEEL_BUILD=1
166+
export SYSTEM_VERSION_COMPAT=0 # https://github.com/numba/numba-benchmark/issues/14#issuecomment-1167675905
165167
# use the requirements_wheel.txt with numpy pins to ensure ABI compatibility
166168
if [[ "$CIBW_ARCHS" != "arm64" ]]; then
167169
export CIBW_TEST_COMMAND="python -c 'import tiledb'"
@@ -171,7 +173,7 @@ stages:
171173
python -c "import os; print(os.environ.get('CIBW_ENVIRONMENT', None))"
172174
git rev-parse HEAD
173175
python3 -m pip install --upgrade -r requirements_dev.txt
174-
python3 setup.py sdist --dist-dir wheelhouse
176+
python3 -m build . --sdist --outdir wheelhouse
175177
pip3 install cibuildwheel==${USE_CIBW_VERSION}
176178
cibuildwheel --output-dir wheelhouse .
177179
displayName: "Build and test wheels (macOS)"
@@ -192,7 +194,7 @@ stages:
192194
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
193195
git rev-parse HEAD
194196
python -m pip install --upgrade -r requirements_dev.txt
195-
python setup.py sdist --dist-dir wheelhouse
197+
python3 -m build . --sdist --outdir wheelhouse
196198
pip install cibuildwheel==%USE_CIBW_VERSION%
197199
cibuildwheel --output-dir wheelhouse .
198200
:: delete the sdist because we *do not* want files with CRLF endings

‎misc/requirements_wheel.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ numpy>=1.23.2 ; python_version >= "3.11"
1313
cmake >= 3.23
1414
cython < 3.0
1515
pybind11 < 2.11.0
16-
setuptools>=42,<=58.3.0
16+
setuptools>=42
1717
setuptools_scm >= 1.5.4
1818
wheel >= 0.30
1919

‎pyproject.toml

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
[build-system]
22

3-
# https://askubuntu.com/a/1407138
4-
# we encounter the above error in versions of setuptools<=58.3.0
5-
requires = ["setuptools>=42,<=58.3.0", "wheel", "pybind11<2.11.0", "Cython<3.0"]
3+
requires = ["setuptools>=42", "wheel", "pybind11<2.11.0", "Cython<3.0"]
64
build-backend = "setuptools.build_meta"
75

86
[tool.pytest.ini_options]

‎requirements_dev.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ numpy >= 1.16.5
33
# ------------------------------------------------
44
# ** MUST sync with misc/requirements_wheel.txt **
55
# ------------------------------------------------
6+
build
67
cmake >= 3.23
78
cython < 3.0
89
pybind11 < 2.11.0
9-
setuptools>=42,<=58.3.0
10+
setuptools>=42
1011
setuptools_scm >= 1.5.4
1112
wheel >= 0.30
1213
contextvars ;python_version<"3.7"

‎setup.py

Lines changed: 36 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
import glob
2-
import io
32
import multiprocessing
43
import os
54
import shutil
65
import subprocess
76
import sys
8-
import zipfile
97
from ctypes import CDLL, POINTER, Structure, byref, c_char_p, c_int, c_void_p
10-
from urllib.error import URLError
11-
from urllib.request import urlopen
128

139
from pkg_resources import resource_filename
1410
from pybind11.setup_helpers import Pybind11Extension
@@ -24,7 +20,7 @@
2420
# - this is for builds-from-source
2521
# - release builds are controlled by `misc/azure-release.yml`
2622
# - this should be set to the current core release, not `dev`
27-
TILEDB_VERSION = "2.17.4"
23+
TILEDB_VERSION = "2.18.0"
2824

2925
# allow overriding w/ environment variable
3026
TILEDB_VERSION = os.environ.get("TILEDB_VERSION") or TILEDB_VERSION
@@ -60,7 +56,7 @@
6056
# set MACOSX_DEPLOYMENT_TARGET before calling setup.py
6157
if sys.platform == "darwin":
6258
if "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
63-
os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.15"
59+
os.environ["MACOSX_DEPLOYMENT_TARGET"] = "11"
6460

6561
# Is this process building a wheel?
6662
WHEEL_BUILD = ("bdist_wheel" in sys.argv) or ("TILEDB_WHEEL_BUILD" in os.environ)
@@ -150,22 +146,42 @@ def download_libtiledb():
150146
:return: Path to extracted source directory.
151147
"""
152148
dest_name = "TileDB-{}".format(TILEDB_VERSION)
153-
dest = os.path.join(BUILD_DIR, dest_name)
149+
os.path.join(BUILD_DIR, dest_name)
150+
os.path.join(BUILD_DIR, f"TileDB-{TILEDB_VERSION[:8]}")
151+
152+
os.path.join(BUILD_DIR, dest_name)
154153
build_dir = os.path.join(BUILD_DIR, f"TileDB-{TILEDB_VERSION[:8]}")
154+
# note that this will only run once locally
155155
if not os.path.exists(build_dir):
156-
url = "https://github.com/TileDB-Inc/TileDB/archive/{}.zip".format(
157-
TILEDB_VERSION
158-
)
159-
print("Downloading TileDB package from {}...".format(TILEDB_VERSION))
160-
try:
161-
with zipfile.ZipFile(io.BytesIO(urlopen(url).read())) as z:
162-
z.extractall(BUILD_DIR)
163-
except URLError:
164-
# try falling back to wget, maybe SSL is broken
165-
subprocess.check_call(["wget", url], shell=True)
166-
with zipfile.ZipFile("{}.zip".format(TILEDB_VERSION)) as z:
167-
z.extractall(BUILD_DIR)
168-
shutil.move(dest, build_dir)
156+
157+
# ----
158+
# restore for 2.19: https://github.com/TileDB-Inc/TileDB/pull/4484
159+
# url = "https://github.com/TileDB-Inc/TileDB/archive/{}.zip".format(
160+
# TILEDB_VERSION
161+
# )
162+
# print("Downloading TileDB package from {}...".format(TILEDB_VERSION))
163+
# try:
164+
# with zipfile.ZipFile(io.BytesIO(urlopen(url).read())) as z:
165+
# z.extractall(BUILD_DIR)
166+
# except URLError:
167+
# # try falling back to wget, maybe SSL is broken
168+
# subprocess.check_call(["wget", url], shell=True)
169+
# with zipfile.ZipFile("{}.zip".format(TILEDB_VERSION)) as z:
170+
# z.extractall(BUILD_DIR)
171+
# shutil.move(dest, build_dir)
172+
# ----
173+
174+
# NOTE: build_dir is where we want the source
175+
args = [
176+
"git",
177+
"clone",
178+
"--depth=1",
179+
"--branch",
180+
TILEDB_VERSION,
181+
"https://github.com/TileDB-Inc/TileDB",
182+
build_dir,
183+
]
184+
subprocess.check_output(args)
169185

170186
return build_dir
171187

0 commit comments

Comments
 (0)