[WIP] Upgrade to V1.2.1 (#462)

* fix: pin duckdb and libduckdb-sys versions

Signed-off-by: luong-hq <[email protected]>

* Bump arrow to the version compatible with chrono 0.4.40

* Update only arrow

* Fix minor typo

* Do not pass stdlib

* Fix #380, bug in code that reads Enums

In turning a ValueRef to a Value, the code was indexing with the row index directly into the dictionary's string array, so iterating any query's results was effectively just listing the Enum variants in order until row index > number of variants and it panicked. Fix by getting the row value (key) and using that to index into the dictionary's string array.

* Fix incorrect Enum test

The test tests for correct value extraction, based on built-in examples in DuckDB. The old expected value was flatly incorrect, as can be seen in duckdb directly with `SELECT medium_enum FROM test_all_types();`.

* Fix writing Arrow Decimal128 with zero scale

* Fix CI install of rustfmt and clippy

Extra space breaks hecrj/setup-rust-action. Arguments should be comma-separated.

* update duckdb to v1.2.1

* replace rust toolchain action

* fix warnings

* add back sanitizer

* move logicaltypeid

* add correct component for asan

* add back extensions to asan build

* bundle ICU as part of extensions-full

---------

Signed-off-by: luong-hq <[email protected]>
Co-authored-by: luong-hq <[email protected]>
Co-authored-by: Hiroaki Yutani <[email protected]>
Co-authored-by: Shritesh Bhattarai <[email protected]>
Co-authored-by: Nickolas Fotopoulos <[email protected]>
Co-authored-by: sgrebnov <[email protected]>

Author: 6 people

.github/workflows/release.yaml

@@ -11,12 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      # This has a matcher for test panics, so we use it even though elsewhere
-      # we use actions-rs/toolchain.
-      - uses: hecrj/setup-rust-action@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          rust-version: stable${{ matrix.host }}
-          targets: ${{ matrix.target }}
+          toolchain: stable
+          target: ${{ matrix.target }}
           components: 'rustfmt, clippy'
 
       # cargo publish

.github/workflows/rust.yaml

@@ -26,20 +26,18 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
-      # This has a matcher for test panics, so we use it even though elsewhere
-      # we use actions-rs/toolchain.
-      - uses: hecrj/setup-rust-action@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          rust-version: stable${{ matrix.host }}
-          targets: ${{ matrix.target }}
+          toolchain: stable
+          target: ${{ matrix.target }}
           components: 'rustfmt, clippy'
 
       # download libduckdb
       - uses: robinraju/[email protected]
         name: Download duckdb
         with:
           repository: "duckdb/duckdb"
-          tag: "v1.2.0"
+          tag: "v1.2.1"
           fileName: ${{ matrix.duckdb }}
           out-file-path: .
 
@@ -112,9 +110,41 @@ jobs:
           key: index-${{ runner.os }}-${{ github.run_number }}
           restore-keys: |
             index-${{ runner.os }}-
-      - uses: hecrj/setup-rust-action@v1
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          rust-version: stable
-          targets: x86_64-pc-windows-msvc
+          toolchain: stable
+          target: x86_64-pc-windows-msvc
 
-      - run: cargo install cargo-examples
+      - run: cargo install cargo-examples
+
+  Sanitizer:
+    name: Address Sanitizer
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # Need nightly rust.
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly
+          components: 'rust-src'
+
+      - name: Tests with asan
+        env:
+          RUSTFLAGS: -Zsanitizer=address -C debuginfo=0
+          RUSTDOCFLAGS: -Zsanitizer=address
+          ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_leaks=1"
+          # Work around https://github.com/rust-lang/rust/issues/59125 by
+          # disabling backtraces. In an ideal world we'd probably suppress the
+          # leak sanitization, but we don't care about backtraces here, so long
+          # as the other tests have them.
+          RUST_BACKTRACE: "0"
+        run: |
+          cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu --package duckdb
+      - name: publish crates --dry-run
+        uses: katyo/publish-crates@v2
+        with:
+          path: './'
+          args: --allow-dirty --all-features
+          dry-run: true
+          ignore-unpublished-changes: true

Cargo.toml

@@ -7,7 +7,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.2.0"
+version = "1.2.1"
 authors = ["wangfenjin <[email protected]>"]
 edition = "2021"
 repository = "https://github.com/duckdb/duckdb-rs"
@@ -19,9 +19,9 @@ license = "MIT"
 categories = ["database"]
 
 [workspace.dependencies]
-duckdb = { version = "1.2.0", path = "crates/duckdb" }
-libduckdb-sys = { version = "1.2.0", path = "crates/libduckdb-sys" }
-duckdb-loadable-macros = { version = "0.1.4", path = "crates/duckdb-loadable-macros" }
+duckdb = { version = "=1.2.1", path = "crates/duckdb" }
+libduckdb-sys = { version = "=1.2.1", path = "crates/libduckdb-sys" }
+duckdb-loadable-macros = { version = "=0.1.5", path = "crates/duckdb-loadable-macros" }
 autocfg = "1.0"
 bindgen = { version = "0.71.1", default-features = false }
 byteorder = "1.3"
@@ -63,4 +63,4 @@ unicase = "2.6.0"
 url = "2.1"
 uuid = "1.0"
 vcpkg = "0.2"
-arrow = { version = "54", default-features = false }
+arrow = { version = "54.2.1", default-features = false }

crates/duckdb-loadable-macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb-loadable-macros"
-version = "0.1.4"
+version = "0.1.5"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true

crates/duckdb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb"
-version = "1.2.0"
+version = "1.2.1"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
@@ -22,6 +22,7 @@ default = []
 bundled = ["libduckdb-sys/bundled"]
 json = ["libduckdb-sys/json", "bundled"]
 parquet = ["libduckdb-sys/parquet", "bundled"]
+icu = ["libduckdb-sys/icu", "bundled"]
 vscalar = []
 vscalar-arrow = []
 vtab = []
@@ -30,7 +31,7 @@ vtab-excel = ["vtab", "calamine"]
 vtab-arrow = ["vtab", "num"]
 appender-arrow = ["vtab-arrow"]
 vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
-extensions-full = ["json", "parquet", "vtab-full"]
+extensions-full = ["json", "parquet", "icu", "vtab-full"]
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
 modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
 polars = ["dep:polars", "dep:polars-arrow"]

crates/duckdb/src/lib.rs

@@ -126,7 +126,7 @@ pub mod types;
 #[cfg(feature = "vtab")]
 pub mod vtab;
 
-/// The duckdb table function interface
+/// The duckdb scalar function interface
 #[cfg(feature = "vscalar")]
 pub mod vscalar;
 

crates/duckdb/src/test_all_types.rs

@@ -651,7 +651,7 @@ fn test_single(idx: &mut i32, column: String, value: ValueRef) {
         },
         "medium_enum" => match idx {
             0 => assert_eq!(value.to_owned(), Value::Enum("enum_0".to_string())),
-            1 => assert_eq!(value.to_owned(), Value::Enum("enum_1".to_string())),
+            1 => assert_eq!(value.to_owned(), Value::Enum("enum_299".to_string())),
             _ => assert_eq!(value, ValueRef::Null),
         },
         "large_enum" => match idx {

crates/duckdb/src/types/string.rs

@@ -6,6 +6,7 @@ pub struct DuckString<'a> {
 }
 
 impl<'a> DuckString<'a> {
+    #[allow(dead_code)]
     pub(crate) fn new(ptr: &'a mut duckdb_string_t) -> Self {
         DuckString { ptr }
     }

crates/duckdb/src/types/value_ref.rs

@@ -6,8 +6,8 @@ use rust_decimal::prelude::*;
 
 use arrow::{
     array::{
-        Array, ArrayRef, DictionaryArray, FixedSizeListArray, LargeListArray, ListArray, MapArray, StructArray,
-        UnionArray,
+        Array, ArrayRef, DictionaryArray, FixedSizeListArray, LargeListArray, ListArray, MapArray, StringArray,
+        StructArray, UnionArray,
     },
     datatypes::{UInt16Type, UInt32Type, UInt8Type},
 };
@@ -239,22 +239,21 @@ impl From<ValueRef<'_>> for Value {
                 }
             },
             ValueRef::Enum(items, idx) => {
-                let value = Row::value_ref_internal(
-                    idx,
-                    0,
-                    match items {
-                        EnumType::UInt8(res) => res.values(),
-                        EnumType::UInt16(res) => res.values(),
-                        EnumType::UInt32(res) => res.values(),
-                    },
-                )
-                .to_owned();
-
-                if let Value::Text(s) = value {
-                    Value::Enum(s)
-                } else {
-                    panic!("Enum value is not a string")
+                let dict_values = match items {
+                    EnumType::UInt8(res) => res.values(),
+                    EnumType::UInt16(res) => res.values(),
+                    EnumType::UInt32(res) => res.values(),
+                }
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .expect("Enum value is not a string");
+                let dict_key = match items {
+                    EnumType::UInt8(res) => res.key(idx),
+                    EnumType::UInt16(res) => res.key(idx),
+                    EnumType::UInt32(res) => res.key(idx),
                 }
+                .unwrap();
+                Value::Enum(dict_values.value(dict_key).to_string())
             }
             ValueRef::Struct(items, idx) => {
                 let value: Vec<(String, Value)> = items

crates/duckdb/src/vscalar/mod.rs

@@ -14,7 +14,7 @@ use crate::{
 };
 mod function;
 
-/// The duckdb Arrow table function interface
+/// The duckdb Arrow scalar function interface
 #[cfg(feature = "vscalar-arrow")]
 pub mod arrow;
 
@@ -32,7 +32,7 @@ pub trait VScalar: Sized {
     ///
     /// This function is unsafe because it:
     ///
-    /// - Dereferences multiple raw pointers (`func``).
+    /// - Dereferences multiple raw pointers (`func`).
     ///
     unsafe fn invoke(
         state: &Self::State,