Merge branch 'main' of https://github.com/davidhcoe/arrow-adbc

Author: David Coe

.env

@@ -16,7 +16,7 @@
 # under the License.
 
 # All of the following environment variables are required to set default values
-# for the parameters in docker-compose.yml.
+# for the parameters in compose.yaml.
 
 # Default repository to pull and push images from
 REPO=apache/arrow-dev
@@ -37,14 +37,14 @@ GO=1.22.9
 ARROW_MAJOR_VERSION=14
 DOTNET=8.0
 
-# Used through docker-compose.yml and serves as the default version for the
+# Used through compose.yaml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
 VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01"
 
 # These are used to tell tests where to find services for integration testing.
-# They are valid if the services are started with the docker-compose config.
+# They are valid if the services are started with the compose config.
 ADBC_DREMIO_FLIGHTSQL_PASS=dremio123
 ADBC_DREMIO_FLIGHTSQL_URI=grpc+tcp://localhost:32010
 ADBC_DREMIO_FLIGHTSQL_USER=dremio

.github/workflows/dev_adbc.yml

@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Dev ADBC
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "dev/**"
+      - ".github/workflows/dev_adbc.yml"
+  push:
+    paths:
+      - "dev/**"
+      - ".github/workflows/dev_adbc.yml"
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash -l -eo pipefail {0}
+
+jobs:
+  pre-commit:
+    name: "pre-commit"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Cache Conda
+        uses: actions/cache@v4
+        with:
+          path: ~/conda_pkgs_dir
+          key: conda-${{ runner.os }}-${{ steps.get-date.outputs.today }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/**') }}
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          use-only-tar-bz2: false
+          use-mamba: true
+
+      - name: Install Dependencies
+        run: |
+          mamba install -c conda-forge \
+            --file ci/conda_env_dev.txt \
+            pytest
+
+      - name: Test
+        run: |
+          pytest -vv dev/adbc_dev/

.github/workflows/dev_pr.yml

@@ -63,7 +63,7 @@ jobs:
         env:
           PR_TITLE: ${{ github.event.pull_request.title }}
         run: |
-          python .github/workflows/dev_pr/title_check.py $(pwd)/pr_checkout "$PR_TITLE"
+          python dev/adbc_dev/title_check.py $(pwd)/pr_checkout "$PR_TITLE"
 
       # Pings make it into the commit message where they annoy the user every
       # time the commit gets pushed somewhere

c/driver/postgresql/postgresql_test.cc

@@ -1696,6 +1696,29 @@ TEST_F(PostgresStatementTest, SetUseCopyFalse) {
   ASSERT_EQ(reader.array->release, nullptr);
 }
 
+TEST_F(PostgresStatementTest, UnknownOid) {
+  // Regression test for https://github.com/apache/arrow-adbc/issues/2448
+  ASSERT_THAT(AdbcStatementNew(&connection, &statement, &error), IsOkStatus(&error));
+  ASSERT_THAT(AdbcStatementSetSqlQuery(
+                  &statement, "SELECT typacl FROM pg_type WHERE oid <= 6157", &error),
+              IsOkStatus(&error));
+  adbc_validation::StreamReader reader;
+  ASSERT_THAT(AdbcStatementExecuteQuery(&statement, &reader.stream.value,
+                                        &reader.rows_affected, &error),
+              IsOkStatus(&error));
+  ASSERT_NO_FATAL_FAILURE(reader.GetSchema());
+  ASSERT_EQ(1, reader.fields.size());
+  ASSERT_EQ(NANOARROW_TYPE_BINARY, reader.fields[0].type);
+  struct ArrowStringView extension_name = reader.fields[0].extension_name;
+  ASSERT_EQ("arrow.opaque",
+            std::string_view(extension_name.data,
+                             static_cast<size_t>(extension_name.size_bytes)));
+  struct ArrowStringView extension_metadata = reader.fields[0].extension_metadata;
+  ASSERT_EQ(R"({"type_name": "unnamed<oid:1034>", "vendor_name": "PostgreSQL"})",
+            std::string_view(extension_metadata.data,
+                             static_cast<size_t>(extension_metadata.size_bytes)));
+}
+
 struct TypeTestCase {
   std::string name;
   std::string sql_type;

c/driver/postgresql/result_helper.cc

@@ -167,11 +167,10 @@ Status PqResultHelper::ResolveOutputTypes(PostgresTypeResolver& type_resolver,
     const Oid pg_oid = PQftype(result_, i);
     PostgresType pg_type;
     if (type_resolver.Find(pg_oid, &pg_type, &na_error) != NANOARROW_OK) {
-      Status status =
-          Status::NotImplemented("[libpq] Column #", i + 1, " (\"", PQfname(result_, i),
-                                 "\") has unknown type code ", pg_oid);
-      ClearResult();
-      return status;
+      // We couldn't look up the OID.
+      // TODO(apache/arrow-adbc#1243): issue a warning (maybe reloading the
+      // connection will load the OIDs if it was a newly created type)
+      pg_type = PostgresType::Unnamed(pg_oid);
     }
 
     root_type.AppendChild(PQfname(result_, i), pg_type);

ci/conda_env_dev.txt

@@ -15,8 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-commitizen
 gh>=2.32.0
 jq
 pre-commit
+pygit2
+python
+python-dotenv
 twine

ci/scripts/docs_build.sh

@@ -26,7 +26,7 @@ main() {
     popd
 
     pushd "$source_dir/java"
-    mvn site
+    mvn --no-transfer-progress site
     popd
 
     pushd "$source_dir/docs"

docker-compose.yml compose.yamldocker-compose.yml renamed to compose.yaml

@@ -77,7 +77,7 @@ services:
 
   ############################ Python conda ##################################
 
-  # Must be run as docker-compose run -e HOST_USER_ID=$(id -u) python-conda
+  # Must be run as docker compose run -e HOST_USER_ID=$(id -u) python-conda
   python-conda:
     image: condaforge/linux-anvil-cos7-x86_64
     volumes:

csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj

@@ -6,7 +6,7 @@
     <PackageReadmeFile>readme.md</PackageReadmeFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Apache.Arrow" Version="18.1.0" />
+    <PackageReference Include="Apache.Arrow" Version="19.0.0" />
     <PackageReference Include="System.Text.Json" Version="8.0.5" />
   </ItemGroup>
   <ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible($(TargetFramework), 'net6.0'))">

csharp/src/Drivers/FlightSql/Apache.Arrow.Adbc.Drivers.FlightSql.csproj

@@ -4,7 +4,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="System.Net.Http.WinHttpHandler" Version="8.0.2" Condition="'$(TargetFrameworkIdentifier)' == '.NETStandard'" />
-    <PackageReference Include="Apache.Arrow.Flight" Version="18.1.0" />
+    <PackageReference Include="Apache.Arrow.Flight" Version="19.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\Apache.Arrow.Adbc\Apache.Arrow.Adbc.csproj" />