fix(python/adbc_driver_manager): load manifests from venv path (apache#3490)

Closes apache#3479.

Author: lidavidm

.github/workflows/native-unix.yml

@@ -648,6 +648,13 @@ jobs:
       - name: Typecheck Python
         run: |
           ./ci/scripts/python_typecheck.sh "$(pwd)"
+      - name: Run Python Docker-based integration tests
+        if: runner.os == 'Linux'
+        run: |
+          # Self-contained tests using docker-compose
+
+          # Test that the driver manager can load manifests installed into a venv
+          docker compose run python-venv
 
   python-docs:
     name: "Documentation ${{ matrix.python }} (Conda/${{ matrix.os }})"

ci/scripts/python_venv_test.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+main() {
+    local -r source_root="${1}"
+    local -r scratch="${2}"
+    local -r sqlite_driver="${3}"
+
+    mkdir -p "${scratch}"
+    python -m venv "${scratch}/.venv"
+    source "${scratch}/.venv/bin/activate"
+
+    "${scratch}"/.venv/bin/python -m pip install "${source_root}"/python/adbc_driver_manager
+    "${scratch}"/.venv/bin/python -m pip install pyarrow
+
+    mkdir -p "${scratch}/.venv/etc/adbc/drivers/"
+    cat >"${scratch}/.venv/etc/adbc/drivers/sqlite.toml" <<EOF
+name = "SQLite"
+[Driver]
+shared = "${sqlite_driver}"
+EOF
+
+    cat >"${scratch}/test.py" <<EOF
+import adbc_driver_manager.dbapi
+
+with adbc_driver_manager.dbapi.connect(driver="sqlite") as con:
+    with con.cursor() as cur:
+        cur.execute("SELECT 1")
+        assert cur.fetchall() == [(1,)]
+EOF
+
+    "${scratch}"/.venv/bin/python "${scratch}/test.py"
+    echo "PASSED: find manifest"
+
+    cat >"${scratch}/test2.py" <<EOF
+import adbc_driver_manager.dbapi
+
+try:
+    with adbc_driver_manager.dbapi.connect(driver="notfound") as con:
+        pass
+except adbc_driver_manager.dbapi.Error as e:
+    print(e)
+    assert ".venv/etc/adbc/drivers" in str(e)
+else:
+    assert False, "Expected exception"
+EOF
+
+    "${scratch}"/.venv/bin/python "${scratch}/test2.py"
+    echo "PASSED: failed manifest contains the proper path in the exception"
+}
+
+main "$@"

compose.yaml

@@ -102,6 +102,14 @@ services:
       - .:/adbc:delegated
     command: "/bin/bash -c 'env CMAKE_BUILD_TYPE=Release /adbc/ci/scripts/java_jni_build.sh /adbc /adbc/dist /adbc/local'"
 
+  ############################### Python #####################################
+
+  python-venv:
+    image: ${ARCH}/python:${PYTHON}
+    volumes:
+      - .:/adbc:delegated
+    command: "/bin/bash -c 'apt update && apt install -y libsqlite3-dev build-essential cmake git && git config --global --add safe.directory /adbc && env BUILD_ALL=0 BUILD_DRIVER_SQLITE=1 ADBC_BUILD_TESTS=OFF ADBC_USE_ASAN=OFF ADBC_USE_UBSAN=OFF /adbc/ci/scripts/cpp_build.sh /adbc /adbc/build/venv && /adbc/ci/scripts/python_venv_test.sh /adbc /adbc/build/venv-test /adbc/build/venv/driver/sqlite/libadbc_driver_sqlite.so'"
+
   ############################ Python conda ##################################
 
   # Must be run as docker compose run -e HOST_USER_ID=$(id -u) python-conda

python/adbc_driver_manager/adbc_driver_manager/_lib.pyx

@@ -572,7 +572,7 @@ cdef class AdbcDatabase(_AdbcHandle):
         if sys.prefix != sys.base_prefix:
             # if we're in a venv, add the venv prefix to the search path list
             status = AdbcDriverManagerDatabaseSetAdditionalSearchPathList(
-                &self.database, _to_bytes(os.path.join(sys.prefix, 'etc/adbc'),
+                &self.database, _to_bytes(os.path.join(sys.prefix, "etc/adbc/drivers"),
                                           "sys.prefix"),
                 &c_error)
             check_error(status, &c_error)