GH-36744: [Python][Packaging] Add upper pin for cython<3 to pyarrow build dependencies (#36743)

### Rationale for this change

Although we already fixed some cython 3 build issues (#34726), some new have been introduced, which we are seeing now cython 3 is released (#36730)

Adding an upper pin (<3) for the release, so we have more time (the full 14.0 release cycle) to iron out issues.
* Closes: #36744

Authored-by: Joris Van den Bossche <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>

Author: jorisvandenbossche

.github/workflows/dev.yml

@@ -103,7 +103,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install cython setuptools six pytest jira
+          pip install "cython<3" setuptools six pytest jira
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

ci/conda_env_python.txt

@@ -18,7 +18,7 @@
 # don't add pandas here, because it is not a mandatory test dependency
 boto3  # not a direct dependency of s3fs, but needed for our s3fs fixture
 cffi
-cython
+cython<3
 cloudpickle
 fsspec
 hypothesis

python/pyproject.toml

@@ -17,7 +17,7 @@
 
 [build-system]
 requires = [
-    "cython >= 0.29.31",
+    "cython >= 0.29.31,<3",
     "oldest-supported-numpy>=0.14",
     "setuptools_scm",
     "setuptools >= 40.1.0",

python/requirements-build.txt

@@ -1,4 +1,4 @@
-cython>=0.29.31
+cython>=0.29.31,<3
 oldest-supported-numpy>=0.14
 setuptools_scm
 setuptools>=38.6.0

python/requirements-wheel-build.txt

@@ -1,4 +1,4 @@
-cython>=0.29.31
+cython>=0.29.31,<3
 oldest-supported-numpy>=0.14
 setuptools_scm
 setuptools>=58

python/setup.py

@@ -40,8 +40,9 @@
 # Check if we're running 64-bit Python
 is_64_bit = sys.maxsize > 2**32
 
-if Cython.__version__ < '0.29.31':
-    raise Exception('Please upgrade to Cython 0.29.31 or newer')
+if Cython.__version__ < '0.29.31' or Cython.__version__ >= '3.0':
+    raise Exception(
+        'Please update your Cython version. Supported Cython >= 0.29.31, < 3.0')
 
 setup_dir = os.path.abspath(os.path.dirname(__file__))
 
@@ -491,7 +492,7 @@ def has_ext_modules(foo):
                                  'pyarrow/_generated_version.py'),
         'version_scheme': guess_next_dev_version
     },
-    setup_requires=['setuptools_scm', 'cython >= 0.29.31'] + setup_requires,
+    setup_requires=['setuptools_scm', 'cython >= 0.29.31,<3'] + setup_requires,
     install_requires=install_requires,
     tests_require=['pytest', 'pandas', 'hypothesis'],
     python_requires='>=3.8',