Switch from xz2 to liblzma to reduce duplicate dependencies (#17509)

## Which issue does this PR close?

Closes #15342

## Rationale for this change

Reduces the duplicate dependencies.

We currently depend on bzip2 in two different ways. In attempting to
reduce this, I needed to update `async-compression` which caused two
different libraries to link to the system lzma library. This is not
allowed in rust.

This PR updates avro-rs, but we cannot merge this PR until that crate
merges apache/avro-rs#284 and we remove the
crates.io patch this PR contains.

## What changes are included in this PR?

Update avro-rs and switch from the unmaintained xz2 crate to liblzma.

## Are these changes tested?

Unit tests. 

## Are there any user-facing changes?

None. This is simply a dependency update to a more recent crate.

Author: timsaucer

Cargo.lock

@@ -90,7 +90,7 @@ version = "51.0.0"
 ahash = { version = "0.8", default-features = false, features = [
     "runtime-rng",
 ] }
-apache-avro = { version = "0.20", default-features = false }
+apache-avro = { version = "0.21", default-features = false }
 arrow = { version = "57.0.0", features = [
     "prettyprint",
     "chrono-tz",
@@ -158,6 +158,7 @@ hex = { version = "0.4.3" }
 indexmap = "2.12.0"
 insta = { version = "1.43.2", features = ["glob", "filters"] }
 itertools = "0.14"
+liblzma = { version = "0.4.4", features = ["static"] }
 log = "^0.4"
 num-traits = { version = "0.2" }
 object_store = { version = "0.12.4", default-features = false }

Cargo.toml

@@ -56,7 +56,7 @@ sql = ["sqlparser"]
 
 [dependencies]
 ahash = { workspace = true }
-apache-avro = { version = "0.20", default-features = false, features = [
+apache-avro = { workspace = true, features = [
     "bzip",
     "snappy",
     "xz",

datafusion/common/Cargo.toml

@@ -46,7 +46,7 @@ array_expressions = ["nested_expressions"]
 avro = ["datafusion-common/avro", "datafusion-datasource-avro"]
 backtrace = ["datafusion-common/backtrace"]
 compression = [
-    "xz2",
+    "liblzma",
     "bzip2",
     "flate2",
     "zstd",
@@ -146,6 +146,7 @@ datafusion-sql = { workspace = true, optional = true }
 flate2 = { version = "1.1.4", optional = true }
 futures = { workspace = true }
 itertools = { workspace = true }
+liblzma = { workspace = true, optional = true }
 log = { workspace = true }
 object_store = { workspace = true }
 parking_lot = { workspace = true }
@@ -159,7 +160,6 @@ tempfile = { workspace = true }
 tokio = { workspace = true }
 url = { workspace = true }
 uuid = { version = "1.18", features = ["v4", "js"] }
-xz2 = { version = "0.1", optional = true, features = ["static"] }
 zstd = { version = "0.13", optional = true, default-features = false }
 
 [dev-dependencies]

datafusion/core/Cargo.toml

@@ -56,9 +56,9 @@ use datafusion_datasource_csv::partitioned_csv_config;
 use flate2::write::GzEncoder;
 #[cfg(feature = "compression")]
 use flate2::Compression as GzCompression;
-use object_store::local_unpartitioned_file;
 #[cfg(feature = "compression")]
-use xz2::write::XzEncoder;
+use liblzma::write::XzEncoder;
+use object_store::local_unpartitioned_file;
 #[cfg(feature = "compression")]
 use zstd::Encoder as ZstdEncoder;
 

datafusion/core/src/test/mod.rs

@@ -31,12 +31,12 @@ version.workspace = true
 all-features = true
 
 [features]
-compression = ["async-compression", "xz2", "bzip2", "flate2", "zstd", "tokio-util"]
+compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
 default = ["compression"]
 
 [dependencies]
 arrow = { workspace = true }
-async-compression = { version = "0.4.19", features = [
+async-compression = { version = "0.4.30", features = [
     "bzip2",
     "gzip",
     "xz",
@@ -60,14 +60,14 @@ flate2 = { version = "1.1.4", optional = true }
 futures = { workspace = true }
 glob = "0.3.0"
 itertools = { workspace = true }
+liblzma = { workspace = true, optional = true }
 log = { workspace = true }
 object_store = { workspace = true }
 rand = { workspace = true }
 tempfile = { workspace = true, optional = true }
 tokio = { workspace = true }
 tokio-util = { version = "0.7.16", features = ["io"], optional = true }
 url = { workspace = true }
-xz2 = { version = "0.1", optional = true, features = ["static"] }
 zstd = { version = "0.13", optional = true, default-features = false }
 
 [dev-dependencies]

datafusion/datasource/Cargo.toml

@@ -43,13 +43,13 @@ use futures::stream::BoxStream;
 use futures::StreamExt;
 #[cfg(feature = "compression")]
 use futures::TryStreamExt;
+#[cfg(feature = "compression")]
+use liblzma::read::XzDecoder;
 use object_store::buffered::BufWriter;
 use tokio::io::AsyncWrite;
 #[cfg(feature = "compression")]
 use tokio_util::io::{ReaderStream, StreamReader};
 #[cfg(feature = "compression")]
-use xz2::read::XzDecoder;
-#[cfg(feature = "compression")]
 use zstd::Decoder as ZstdDecoder;
 
 /// Readable file compression type