diff --git a/.github/workflows/docker-gdal.yml b/.github/workflows/docker-gdal.yml index de600454..c3d50bf6 100644 --- a/.github/workflows/docker-gdal.yml +++ b/.github/workflows/docker-gdal.yml @@ -26,8 +26,6 @@ jobs: - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" # python 3.10.12 - "ghcr.io/osgeo/gdal:ubuntu-small-3.7.3" # python 3.10.12 - "ghcr.io/osgeo/gdal:ubuntu-small-3.6.4" # python 3.10.6 - - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.9.20 (installed manually) - - "osgeo/gdal:ubuntu-small-3.4.3" # python 3.9.20 (installed manually) container: image: ${{ matrix.container }} @@ -37,15 +35,6 @@ jobs: run: | apt-get update && apt-get install -y build-essential git python3-dev - - name: Install Python - # the GDAL 3.4 and 3.5 images do have Python 3.8 installed, so have to - # install a more recent Python version manually - if: matrix.container == 'osgeo/gdal:ubuntu-small-3.5.3' || matrix.container == 'osgeo/gdal:ubuntu-small-3.4.3' - run: | - apt-get update && apt-get install -y software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update && apt-get install -y python3.9-dev - - uses: actions/checkout@v5 - name: Install uv @@ -65,8 +54,6 @@ jobs: uv pip install -e .[dev,test,geopandas] - name: Install pyarrow - # GDAL>=3.6 required to use Arrow API - if: matrix.container != 'osgeo/gdal:ubuntu-small-3.5.3' && matrix.container != 'osgeo/gdal:ubuntu-small-3.4.3' run: | uv pip install pyarrow diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml index d74a2c5d..6fa3a1d8 100644 --- a/.github/workflows/tests-conda.yml +++ b/.github/workflows/tests-conda.yml @@ -35,11 +35,6 @@ jobs: - os: "ubuntu-latest" python: "3.9" env: "minimal" - # environment for older Windows libgdal to make sure gdal_i.lib is - # properly detected - - os: "windows-2022" - python: "3.10" - env: "libgdal3.5.1" # environment with nightly wheels - os: "ubuntu-latest" python: "3.11" diff --git a/CHANGES.md b/CHANGES.md index e6702e2a..eccf613e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ - Return JSON fields (as identified by GDAL) as dicts/lists in `read_dataframe`; these were previously returned as strings (#556). +- Drop support for GDAL 3.4 and 3.5 (#584). ### Improvements diff --git a/README.md b/README.md index 9e5e9708..25d102ec 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Read the documentation for more information: ## Requirements -Supports Python 3.9 - 3.13 and GDAL 3.4.x - 3.9.x. +Supports Python 3.9 - 3.13 and GDAL 3.6.x - 3.9.x. Reading to GeoDataFrames requires `geopandas>=0.12` with `shapely>=2`. diff --git a/ci/envs/libgdal3.5.1.yml b/ci/envs/libgdal3.5.1.yml deleted file mode 100644 index a37aa2f1..00000000 --- a/ci/envs/libgdal3.5.1.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: test -channels: - - conda-forge -dependencies: - - numpy - - libgdal==3.5.1 - - pytest diff --git a/docs/source/install.md b/docs/source/install.md index 94513d11..8838e6e2 100644 --- a/docs/source/install.md +++ b/docs/source/install.md @@ -2,7 +2,7 @@ ## Requirements -Supports Python 3.9 - 3.13 and GDAL 3.4.x - 3.9.x +Supports Python 3.9 - 3.13 and GDAL 3.6.x - 3.9.x Reading to GeoDataFrames requires `geopandas>=0.12` with `shapely>=2`. @@ -132,20 +132,20 @@ To build on Windows, you need to provide additional environment variables or command-line parameters because the location of the GDAL binaries and headers cannot be automatically determined. -Assuming GDAL 3.4.1 is installed to `c:\GDAL`, you can set the `GDAL_INCLUDE_PATH`, +Assuming GDAL 3.8.3 is installed to `c:\GDAL`, you can set the `GDAL_INCLUDE_PATH`, `GDAL_LIBRARY_PATH` and `GDAL_VERSION` environment variables and build as follows: ```bash set GDAL_INCLUDE_PATH=C:\GDAL\include set GDAL_LIBRARY_PATH=C:\GDAL\lib -set GDAL_VERSION=3.4.1 +set GDAL_VERSION=3.8.3 python -m pip install --no-deps --force-reinstall --no-use-pep517 -e . -v ``` Alternatively, you can pass those options also as command-line parameters: ```bash -python -m pip install --install-option=build_ext --install-option="-IC:\GDAL\include" --install-option="-lgdal_i" --install-option="-LC:\GDAL\lib" --install-option="--gdalversion=3.4.1" --no-deps --force-reinstall --no-use-pep517 -e . -v +python -m pip install --install-option=build_ext --install-option="-IC:\GDAL\include" --install-option="-lgdal_i" --install-option="-LC:\GDAL\lib" --install-option="--gdalversion=3.8.3" --no-deps --force-reinstall --no-use-pep517 -e . -v ``` The location of the GDAL DLLs must be on your system `PATH`. diff --git a/pyogrio/_compat.py b/pyogrio/_compat.py index 78638f61..8275d99e 100644 --- a/pyogrio/_compat.py +++ b/pyogrio/_compat.py @@ -29,7 +29,6 @@ pandas = None -HAS_ARROW_API = __gdal_version__ >= (3, 6, 0) HAS_ARROW_WRITE_API = __gdal_version__ >= (3, 8, 0) HAS_PYARROW = pyarrow is not None HAS_PYPROJ = pyproj is not None @@ -44,8 +43,6 @@ PANDAS_GE_22 = pandas is not None and Version(pandas.__version__) >= Version("2.2.0") PANDAS_GE_30 = pandas is not None and Version(pandas.__version__) >= Version("3.0.0dev") -GDAL_GE_350 = __gdal_version__ >= (3, 5, 0) -GDAL_GE_352 = __gdal_version__ >= (3, 5, 2) GDAL_GE_37 = __gdal_version__ >= (3, 7, 0) GDAL_GE_38 = __gdal_version__ >= (3, 8, 0) GDAL_GE_311 = __gdal_version__ >= (3, 11, 0) diff --git a/pyogrio/_io.pyx b/pyogrio/_io.pyx index a4aed843..39882b3a 100644 --- a/pyogrio/_io.pyx +++ b/pyogrio/_io.pyx @@ -1620,9 +1620,6 @@ def ogr_open_arrow( cdef ArrowArrayStream* stream cdef ArrowSchema schema - IF CTE_GDAL_VERSION < (3, 6, 0): - raise RuntimeError("Need GDAL>=3.6 for Arrow support") - if force_2d: raise ValueError("forcing 2D is not supported for Arrow") diff --git a/pyogrio/_ogr.pxd b/pyogrio/_ogr.pxd index 681190ca..af8072ca 100644 --- a/pyogrio/_ogr.pxd +++ b/pyogrio/_ogr.pxd @@ -418,14 +418,10 @@ cdef extern from "ogr_api.h": const char* OLCFastGetExtent const char* OLCTransactions - -IF CTE_GDAL_VERSION >= (3, 6, 0): - - cdef extern from "ogr_api.h": - bint OGR_L_GetArrowStream( - OGRLayerH hLayer, ArrowArrayStream *out_stream, char** papszOptions - ) - +cdef extern from "ogr_api.h": + bint OGR_L_GetArrowStream( + OGRLayerH hLayer, ArrowArrayStream *out_stream, char** papszOptions + ) IF CTE_GDAL_VERSION >= (3, 8, 0): diff --git a/pyogrio/_ogr.pyx b/pyogrio/_ogr.pyx index 16d5305a..2109125b 100644 --- a/pyogrio/_ogr.pyx +++ b/pyogrio/_ogr.pyx @@ -1,6 +1,5 @@ import os import sys -from uuid import uuid4 import warnings from pyogrio._err cimport check_pointer @@ -44,21 +43,16 @@ def get_gdal_version_string(): return get_string(version) -IF CTE_GDAL_VERSION >= (3, 4, 0): - - cdef extern from "ogr_api.h": - bint OGRGetGEOSVersion(int *pnMajor, int *pnMinor, int *pnPatch) +cdef extern from "ogr_api.h": + bint OGRGetGEOSVersion(int *pnMajor, int *pnMinor, int *pnPatch) def get_gdal_geos_version(): cdef int major, minor, revision - IF CTE_GDAL_VERSION >= (3, 4, 0): - if not OGRGetGEOSVersion(&major, &minor, &revision): - return None - return (major, minor, revision) - ELSE: + if not OGRGetGEOSVersion(&major, &minor, &revision): return None + return (major, minor, revision) def set_gdal_config_options(dict options): @@ -167,7 +161,7 @@ def get_gdal_data_path(): """ cdef const char *path_c = CPLFindFile("gdal", "header.dxf") if path_c != NULL: - return get_string(path_c).rstrip("header.dxf") + return get_string(path_c).replace("header.dxf", "") return None diff --git a/pyogrio/raw.py b/pyogrio/raw.py index 098a9da5..22328b16 100644 --- a/pyogrio/raw.py +++ b/pyogrio/raw.py @@ -4,7 +4,7 @@ from io import BytesIO from pathlib import Path -from pyogrio._compat import HAS_ARROW_API, HAS_ARROW_WRITE_API, HAS_PYARROW +from pyogrio._compat import HAS_ARROW_WRITE_API, HAS_PYARROW from pyogrio._env import GDALEnv from pyogrio.core import detect_write_driver from pyogrio.errors import DataSourceError @@ -435,9 +435,6 @@ def open_arrow( } """ - if not HAS_ARROW_API: - raise RuntimeError("GDAL>= 3.6 required to read using arrow") - dataset_kwargs = _preprocess_options_key_value(kwargs) if kwargs else {} return ogr_open_arrow( @@ -581,12 +578,6 @@ def _get_write_path_driver(path, driver, append=False): f"{get_gdal_version_string()}" ) - # prevent segfault from: https://github.com/OSGeo/gdal/issues/5739 - if append and driver == "FlatGeobuf" and get_gdal_version() <= (3, 5, 0): - raise RuntimeError( - "append to FlatGeobuf is not supported for GDAL <= 3.5.0 due to segfault" - ) - return path, driver diff --git a/pyogrio/tests/conftest.py b/pyogrio/tests/conftest.py index dcb7028e..72017fc3 100644 --- a/pyogrio/tests/conftest.py +++ b/pyogrio/tests/conftest.py @@ -11,7 +11,6 @@ ) from pyogrio._compat import ( GDAL_GE_37, - HAS_ARROW_API, HAS_ARROW_WRITE_API, HAS_GDAL_GEOS, HAS_PYARROW, @@ -66,10 +65,7 @@ def pytest_report_header(config): # marks to skip tests if optional dependecies are not present -requires_arrow_api = pytest.mark.skipif(not HAS_ARROW_API, reason="GDAL>=3.6 required") -requires_pyarrow_api = pytest.mark.skipif( - not HAS_ARROW_API or not HAS_PYARROW, reason="GDAL>=3.6 and pyarrow required" -) +requires_pyarrow_api = pytest.mark.skipif(not HAS_PYARROW, reason="pyarrow required") requires_pyproj = pytest.mark.skipif(not HAS_PYPROJ, reason="pyproj required") diff --git a/pyogrio/tests/test_arrow.py b/pyogrio/tests/test_arrow.py index 4cb491a8..2916ae3a 100644 --- a/pyogrio/tests/test_arrow.py +++ b/pyogrio/tests/test_arrow.py @@ -476,10 +476,6 @@ def test_write_geojson(tmp_path, naturalearth_lowres): @requires_arrow_write_api -@pytest.mark.skipif( - __gdal_version__ < (3, 6, 0), - reason="OpenFileGDB write support only available for GDAL >= 3.6.0", -) @pytest.mark.parametrize( "write_int64", [ @@ -643,7 +639,7 @@ def test_write_append(request, tmp_path, naturalearth_lowres, ext): assert read_info(filename)["features"] == 354 -@pytest.mark.parametrize("driver,ext", [("GML", ".gml"), ("GeoJSONSeq", ".geojsons")]) +@pytest.mark.parametrize("driver,ext", [("GML", ".gml")]) @requires_arrow_write_api def test_write_append_unsupported(tmp_path, naturalearth_lowres, driver, ext): meta, table = read_arrow(naturalearth_lowres) @@ -961,9 +957,6 @@ def test_write_memory_driver_required(naturalearth_lowres): @requires_arrow_write_api @pytest.mark.parametrize("driver", ["ESRI Shapefile", "OpenFileGDB"]) def test_write_memory_unsupported_driver(naturalearth_lowres, driver): - if driver == "OpenFileGDB" and __gdal_version__ < (3, 6, 0): - pytest.skip("OpenFileGDB write support only available for GDAL >= 3.6.0") - meta, table = read_arrow(naturalearth_lowres, max_features=1) buffer = BytesIO() diff --git a/pyogrio/tests/test_core.py b/pyogrio/tests/test_core.py index 3ac23e36..634cbc1c 100644 --- a/pyogrio/tests/test_core.py +++ b/pyogrio/tests/test_core.py @@ -18,7 +18,7 @@ vsi_rmtree, vsi_unlink, ) -from pyogrio._compat import GDAL_GE_38, GDAL_GE_350 +from pyogrio._compat import GDAL_GE_38 from pyogrio._env import GDALEnv from pyogrio.errors import DataLayerError, DataSourceError from pyogrio.raw import read, write @@ -140,11 +140,7 @@ def test_list_drivers(): # verify that the core drivers are present for name in ("ESRI Shapefile", "GeoJSON", "GeoJSONSeq", "GPKG", "OpenFileGDB"): assert name in all_drivers - expected_capability = "rw" - if name == "OpenFileGDB" and __gdal_version__ < (3, 6, 0): - expected_capability = "r" - assert all_drivers[name] == expected_capability drivers = list_drivers(read=True) @@ -396,10 +392,6 @@ def test_read_bounds_mask(naturalearth_lowres_all_ext, mask, expected): assert array_equal(fids, fids_expected) -@pytest.mark.skipif( - __gdal_version__ < (3, 4, 0), - reason="Cannot determine if GEOS is present or absent for GDAL < 3.4", -) def test_read_bounds_bbox_intersects_vs_envelope_overlaps(naturalearth_lowres_all_ext): # If GEOS is present and used by GDAL, bbox filter will be based on intersection # of bbox and actual geometries; if GEOS is absent or not used by GDAL, it @@ -587,11 +579,7 @@ def test_read_info_jsonfield(nested_geojson_file): """Test if JSON fields types are returned correctly.""" meta = read_info(nested_geojson_file) assert meta["ogr_types"] == ["OFTString", "OFTString"] - if GDAL_GE_350: - # OFSTJSON is only supported for GDAL >= 3.5 - assert meta["ogr_subtypes"] == ["OFSTNone", "OFSTJSON"] - else: - assert meta["ogr_subtypes"] == ["OFSTNone", "OFSTNone"] + assert meta["ogr_subtypes"] == ["OFSTNone", "OFSTJSON"] def test_read_info_unspecified_layer_warning(data_dir): diff --git a/pyogrio/tests/test_geopandas_io.py b/pyogrio/tests/test_geopandas_io.py index e9a71aac..e4cad146 100644 --- a/pyogrio/tests/test_geopandas_io.py +++ b/pyogrio/tests/test_geopandas_io.py @@ -20,8 +20,6 @@ from pyogrio._compat import ( GDAL_GE_37, GDAL_GE_311, - GDAL_GE_350, - GDAL_GE_352, HAS_ARROW_WRITE_API, HAS_PYPROJ, PANDAS_GE_15, @@ -272,10 +270,6 @@ def test_read_force_2d(tmp_path, use_arrow): assert not df.iloc[0].geometry.has_z -@pytest.mark.skipif( - not GDAL_GE_352, - reason="gdal >= 3.5.2 needed to use OGR_GEOJSON_MAX_OBJ_SIZE with a float value", -) def test_read_geojson_error(naturalearth_lowres_geojson, use_arrow): try: set_gdal_config_options({"OGR_GEOJSON_MAX_OBJ_SIZE": 0.01}) @@ -387,10 +381,6 @@ def test_read_datetime_tz(datetime_tz_file, tmp_path, use_arrow): assert_series_equal(df_read.datetime_col, expected) -@pytest.mark.skipif( - not GDAL_GE_350, - reason="OFSTJSON subtype + some list type situations need GDAL >= 3.5", -) def test_read_list_types(list_field_values_file, use_arrow): """Test reading a geojson file containing fields with lists.""" info = read_info(list_field_values_file) @@ -1439,12 +1429,6 @@ def test_write_dataframe_gpkg_multiple_layers(tmp_path, naturalearth_lowres, use @pytest.mark.parametrize("ext", ALL_EXTS) @pytest.mark.requires_arrow_write_api def test_write_dataframe_append(request, tmp_path, naturalearth_lowres, ext, use_arrow): - if ext == ".fgb" and __gdal_version__ <= (3, 5, 0): - pytest.skip("Append to FlatGeobuf fails for GDAL <= 3.5.0") - - if ext in (".geojsonl", ".geojsons") and __gdal_version__ <= (3, 6, 0): - pytest.skip("Append to GeoJSONSeq only available for GDAL >= 3.6.0") - if use_arrow and ext.startswith(".geojson"): # Bug in GDAL when appending int64 to GeoJSON # (https://github.com/OSGeo/gdal/issues/9792) @@ -2088,9 +2072,6 @@ def test_read_multisurface(multisurface_file, use_arrow): assert df.geometry.type.tolist() == ["MultiPolygon"] -@pytest.mark.skipif( - not GDAL_GE_350, reason="OFSTJSON subtype only supported for GDAL >= 3.5" -) def test_read_dataset_kwargs(nested_geojson_file, use_arrow): # by default, nested data are not flattened df = read_dataframe(nested_geojson_file, use_arrow=use_arrow) @@ -2358,9 +2339,6 @@ def test_write_memory_driver_required(naturalearth_lowres): @pytest.mark.parametrize("driver", ["ESRI Shapefile", "OpenFileGDB"]) def test_write_memory_unsupported_driver(naturalearth_lowres, driver): - if driver == "OpenFileGDB" and __gdal_version__ < (3, 6, 0): - pytest.skip("OpenFileGDB write support only available for GDAL >= 3.6.0") - df = read_dataframe(naturalearth_lowres) buffer = BytesIO() diff --git a/pyogrio/tests/test_raw_io.py b/pyogrio/tests/test_raw_io.py index 12e85e76..8dff4ba1 100644 --- a/pyogrio/tests/test_raw_io.py +++ b/pyogrio/tests/test_raw_io.py @@ -24,7 +24,6 @@ DRIVER_EXT, DRIVERS, prepare_testfile, - requires_arrow_api, requires_pyarrow_api, requires_shapely, ) @@ -616,10 +615,6 @@ def test_write_no_geom_no_fields(): write("test.gpkg", geometry=None, field_data=None, fields=None) -@pytest.mark.skipif( - __gdal_version__ < (3, 6, 0), - reason="OpenFileGDB write support only available for GDAL >= 3.6.0", -) @pytest.mark.parametrize( "write_int64", [ @@ -698,12 +693,6 @@ def test_write_openfilegdb(tmp_path, write_int64): @pytest.mark.parametrize("ext", DRIVERS) def test_write_append(tmp_path, naturalearth_lowres, ext): - if ext == ".fgb" and __gdal_version__ <= (3, 5, 0): - pytest.skip("Append to FlatGeobuf fails for GDAL <= 3.5.0") - - if ext in (".geojsonl", ".geojsons") and __gdal_version__ < (3, 6, 0): - pytest.skip("Append to GeoJSONSeq only available for GDAL >= 3.6.0") - if ext == ".gpkg.zip": pytest.skip("Append to .gpkg.zip is not supported") @@ -725,11 +714,8 @@ def test_write_append(tmp_path, naturalearth_lowres, ext): assert read_info(filename)["features"] == 354 -@pytest.mark.parametrize("driver,ext", [("GML", ".gml"), ("GeoJSONSeq", ".geojsons")]) +@pytest.mark.parametrize("driver,ext", [("GML", ".gml")]) def test_write_append_unsupported(tmp_path, naturalearth_lowres, driver, ext): - if ext == ".geojsons" and __gdal_version__ >= (3, 6, 0): - pytest.skip("Append to GeoJSONSeq supported for GDAL >= 3.6.0") - meta, _, geometry, field_data = read(naturalearth_lowres) # GML does not support append functionality @@ -744,27 +730,6 @@ def test_write_append_unsupported(tmp_path, naturalearth_lowres, driver, ext): write(filename, geometry, field_data, driver=driver, append=True, **meta) -@pytest.mark.skipif( - __gdal_version__ > (3, 5, 0), - reason="segfaults on FlatGeobuf limited to GDAL <= 3.5.0", -) -def test_write_append_prevent_gdal_segfault(tmp_path, naturalearth_lowres): - """GDAL <= 3.5.0 segfaults when appending to FlatGeobuf; this test - verifies that we catch that before segfault""" - meta, _, geometry, field_data = read(naturalearth_lowres) - meta["geometry_type"] = "MultiPolygon" - - filename = tmp_path / "test.fgb" - write(filename, geometry, field_data, **meta) - - assert filename.exists() - - with pytest.raises( - RuntimeError, # match="append to FlatGeobuf is not supported for GDAL <= 3.5.0" - ): - write(filename, geometry, field_data, append=True, **meta) - - @pytest.mark.parametrize( "driver", { @@ -794,18 +759,6 @@ def test_write_supported(tmp_path, naturalearth_lowres, driver): assert filename.exists() -@pytest.mark.skipif( - __gdal_version__ >= (3, 6, 0), reason="OpenFileGDB supports write for GDAL >= 3.6.0" -) -def test_write_unsupported(tmp_path, naturalearth_lowres): - meta, _, geometry, field_data = read(naturalearth_lowres) - - filename = tmp_path / "test.gdb" - - with pytest.raises(DataSourceError, match="does not support write functionality"): - write(filename, geometry, field_data, driver="OpenFileGDB", **meta) - - def test_write_gdalclose_error(naturalearth_lowres): meta, _, geometry, field_data = read(naturalearth_lowres) @@ -1178,9 +1131,6 @@ def test_write_memory_driver_required(naturalearth_lowres): @pytest.mark.parametrize("driver", ["ESRI Shapefile", "OpenFileGDB"]) def test_write_memory_unsupported_driver(naturalearth_lowres, driver): - if driver == "OpenFileGDB" and __gdal_version__ < (3, 6, 0): - pytest.skip("OpenFileGDB write support only available for GDAL >= 3.6.0") - meta, _, geometry, field_data = read(naturalearth_lowres) buffer = BytesIO() @@ -1482,7 +1432,6 @@ def test_write_with_mask(tmp_path): write(filename, geometry, field_data, fields, field_mask, **meta) -@requires_arrow_api def test_open_arrow_capsule_protocol_without_pyarrow(naturalearth_lowres): # this test is included here instead of test_arrow.py to ensure we also run # it when pyarrow is not installed @@ -1500,7 +1449,6 @@ def test_open_arrow_capsule_protocol_without_pyarrow(naturalearth_lowres): @pytest.mark.skipif(HAS_PYARROW, reason="pyarrow is installed") -@requires_arrow_api def test_open_arrow_error_no_pyarrow(naturalearth_lowres): # this test is included here instead of test_arrow.py to ensure we run # it when pyarrow is not installed