Moving typechecks under 'Execute Docker Build' step

Author: rok

.github/workflows/python.yml

@@ -127,12 +127,6 @@ jobs:
           source ci/scripts/util_enable_core_dumps.sh
           archery docker run ${{ matrix.image }}
 
-      - name: Run Archery typechecking
-        run: archery docker run ${{ matrix.image }} /arrow/dev/run_typechecking.sh
-
-      - name: Test that PyArrow is present
-        run: archery docker run ${{ matrix.image }} /arrow/ci/scripts/python_test.sh /arrow
-
       - name: Docker Push
         if: >-
           success() &&
@@ -145,17 +139,6 @@ jobs:
         continue-on-error: true
         run: archery docker push ${{ matrix.image }}
 
-#      - name: Type check with mypy and pyright
-#        run: |-
-#            python -m pip install mypy pyright ty griffe libcst pytest hypothesis fsspec scipy-stubs pandas-stubs types-python-dateutil types-psutil types-requests griffe libcst sphinx types-cffi
-#            pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple pyarrow
-#            cd python
-#            mypy
-#            pyright
-#            ty check
-#            cd ..
-#            python ./dev/update_stub_docstrings.py -f ./python/pyarrow-stubs
-
   macos:
     name: ${{ matrix.architecture }} macOS ${{ matrix.macos-version }} Python 3
     runs-on: macos-${{ matrix.macos-version }}

dev/run_typechecking.sh …/scripts/python_test_type_annotations.shdev/run_typechecking.sh renamed to ci/scripts/python_test_type_annotations.sh dev/run_typechecking.sh renamed to ci/scripts/python_test_type_annotations.sh

@@ -19,18 +19,26 @@
 
 set -ex
 
-# Install library stubs
-pip install pandas-stubs scipy-stubs sphinx types-cffi types-psutil types-requests types-python-dateutil
+if [ "${PYARROW_TEST_ANNOTATIONS}" == "ON" ]; then
 
-# Install type checkers
-pip install mypy pyright ty
+  pyarrow_dir=${1}
 
-# TODO: pandas shouldn't be necessary, fix the stubs
-# Install other dependencies
-pip install pandas psutil
+  # Install library stubs
+  pip install pandas-stubs scipy-stubs sphinx types-cffi types-psutil types-requests types-python-dateutil
 
-# Run type checkers
-cd /arrow/python
-mypy
-pyright
-ty check
+  # Install type checkers
+  pip install mypy pyright ty
+
+  # TODO: pandas shouldn't be necessary, fix the stubs
+  # Install other dependencies
+  pip install pandas psutil
+
+  # Run type checkers
+  pushd "${pyarrow_dir}"
+  mypy
+  pyright
+  ty check;
+
+else
+  echo "Skipping type annotation tests";
+fi

compose.yaml

@@ -935,12 +935,14 @@ services:
     environment:
       <<: [*common, *ccache, *sccache]
       PYTEST_ARGS:  # inherit
+      PYARROW_TEST_ANNOTATIONS: "ON"
     volumes: *conda-volumes
     command: &python-conda-command
       ["
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
-        /arrow/ci/scripts/python_test.sh /arrow"]
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"]
 
   conda-python-emscripten:
     # Usage:
@@ -1015,14 +1017,16 @@ services:
       ARROW_S3: "OFF"
       ARROW_SUBSTRAIT: "OFF"
       ARROW_WITH_OPENTELEMETRY: "OFF"
+      PYARROW_TEST_ANNOTATIONS: "ON"
       SETUPTOOLS_SCM_PRETEND_VERSION:
     volumes: *ubuntu-volumes
     deploy: *cuda-deploy
     command: &python-command >
       /bin/bash -c "
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
-        /arrow/ci/scripts/python_test.sh /arrow"
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"
 
   debian-python:
     # Usage:
@@ -1510,6 +1514,7 @@ services:
         python: ${PYTHON}
     shm_size: *shm-size
     environment:
+      PYARROW_TEST_ANNOTATIONS: "ON"
       <<: [*common, *ccache, *sccache]
       PARQUET_REQUIRE_ENCRYPTION:  # inherit
       HYPOTHESIS_PROFILE:  # inherit
@@ -1520,7 +1525,8 @@ services:
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         mamba uninstall -y numpy &&
-        /arrow/ci/scripts/python_test.sh /arrow"]
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"]
 
   conda-python-docs:
     # Usage:
@@ -1540,13 +1546,15 @@ services:
       BUILD_DOCS_CPP: "ON"
       BUILD_DOCS_PYTHON: "ON"
       PYTEST_ARGS: "--doctest-modules --doctest-cython"
+      PYARROW_TEST_ANNOTATIONS: "ON"
     volumes: *conda-volumes
     command:
       ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         pip install -e /arrow/dev/archery[numpydoc] &&
         archery numpydoc --allow-rule GL10,PR01,PR03,PR04,PR05,PR10,RT03,YD01 &&
-        /arrow/ci/scripts/python_test.sh /arrow"]
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"]
 
   conda-python-dask:
     # Possible $DASK parameters:

python/pyarrow/tests/util.py

@@ -170,8 +170,9 @@ def get_modified_env_with_pythonpath():
     env = os.environ.copy()
     existing_pythonpath = env.get('PYTHONPATH', '')
 
-    module_path = os.path.abspath(
-        os.path.dirname(os.path.dirname(pa.__file__)))
+    module_path = os.path.abspath(os.path.dirname(os.path.dirname(
+        pa.__file__  # type: ignore[no-matching-overload]
+    )))
 
     if existing_pythonpath:
         new_pythonpath = os.pathsep.join((module_path, existing_pythonpath))