VER: Release 0.27.0

Author: threecgreen

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## 0.27.0 - 2025-06-10
+
+### Enhancements
+- Made the buffer size used by the live client when reading from the TCP socket
+  configurable through the `LiveBuilder::buffer_size()` method
+- Added support for using `rustls` without pulling in OpenSSL. `reqwest` with OpenSSL is
+  still the default
+- Upgraded DBN version to 0.36.0:
+  - Added support for width, fill, and padding when formatting `pretty::Ts`
+  - Added support for sign, precision, width, fill, and padding when formatting
+    `pretty::Px`
+  - Optimized pretty formatting of prices and timestamps
+
+### Breaking changes
+- Changed type of `split_duration` to `Option<SplitDuration>` to support setting no
+  split duration
+- Breaking changes from DBN:
+  - Moved core async decoding and encoding functionality to new traits to
+    match the sync interface and present a standardized interface
+    - Decoding: `AsyncDecodeRecordRef` and `AsyncDecodeRecord`
+    - Encoding: `AsyncEncodeRecord`, `AsyncEncodeRecordRef`, and
+      `AsyncEncodeRecordTextExt`
+
+### Deprecations
+- Deprecated `LiveClient::connect` and `LiveClient::connect_with_addr` methods in favor
+  of using the builder so additional optional parameters can be added without a breaking
+  change
+
+### Bug fixes
+- Fixed bug with deserializing `null` `split_duration` in historical
+  `batch().list_jobs()`
+
 ## 0.26.2 - 2025-06-03
 
 ### Enhancements

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "databento"
 authors = ["Databento <support@databento.com>"]
-version = "0.26.2"
+version = "0.27.0"
 edition = "2021"
 repository = "https://github.com/databento/databento-rs"
 description = "Official Databento client library"
@@ -18,7 +18,7 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [features]
-default = ["historical", "live"]
+default = ["historical", "live", "reqwest/default-tls"]
 historical = [
   "dep:async-compression",
   "dep:futures",
@@ -31,14 +31,14 @@ historical = [
 live = ["dep:hex", "dep:sha2", "tokio/net"]
 
 [dependencies]
-dbn = { version = "0.35.1", features = ["async", "serde"] }
+dbn = { version = "0.36.0", features = ["async", "serde"] }
 
 async-compression = { version = "0.4", features = ["tokio", "zstd"], optional = true }
 # Async stream trait
 futures = { version = "0.3", optional = true }
 # Used for Live authentication
 hex = { version = "0.4", optional = true }
-reqwest = { version = "0.12", optional = true, features = ["json", "stream"] }
+reqwest = { version = "0.12", optional = true, features = ["json", "stream"], default-features = false }
 serde = { version = "1.0", optional = true, features = ["derive"] }
 serde_json = { version = "1.0", optional = true }
 # Used for Live authentication

README.md

@@ -9,6 +9,11 @@
 The official Rust client library for [Databento](https://databento.com).
 The clients support fast and safe streaming of both real-time and historical market data
 through similar interfaces.
+The library is built on top of the tokio asynchronous runtime and
+[Databento's efficient binary encoding](https://databento.com/docs/standards-and-conventions/databento-binary-encoding).
+
+You can find getting started tutorials, full API method documentation, examples with output on the
+[Databento docs site](https://databento.com/docs/?historical=rust&live=rust).
 
 ## Installation
 
@@ -17,14 +22,70 @@ To add the crate to an existing project, run the following command:
 cargo add databento
 ```
 
+### Feature flags
+
+- `historical`: enables the historical client for data older than 24 hours
+- `live`: enables the live client for real-time and intraday historical data
+
+By default both features are enabled and the historical client uses OpenSSL for TLS.
+To use `rustls`, disable default features for both the databento crate and [reqwest](https://github.com/seanmonstar/reqwest).
+```toml
+databento = { features = ["historical"], default-features = false }
+reqwest = { features = ["rustls-tls"], default-features = false }
+```
+
 ## Usage
 
+### Historical
+
+Here is a simple program that fetches 10 minutes worth of historical trades for E-mini S&P 500 futures from CME Globex:
+```rust no_run
+use std::error::Error;
+
+use databento::{
+    dbn::{decode::DbnMetadata, Dataset, SType, Schema, TradeMsg},
+    historical::timeseries::GetRangeParams,
+    HistoricalClient,
+};
+use time::macros::{date, datetime};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+    let mut client = HistoricalClient::builder().key_from_env()?.build()?;
+    let mut decoder = client
+        .timeseries()
+        .get_range(
+            &GetRangeParams::builder()
+                .dataset(Dataset::GlbxMdp3)
+                .date_time_range((
+                    datetime!(2022-06-10 14:30 UTC),
+                    datetime!(2022-06-10 14:40 UTC),
+                ))
+                .symbols("ES.FUT")
+                .stype_in(SType::Parent)
+                .schema(Schema::Trades)
+                .build(),
+        )
+        .await?;
+    let symbol_map = decoder
+        .metadata()
+        .symbol_map_for_date(date!(2022 - 06 - 10))?;
+    while let Some(trade) = decoder.decode_record::<TradeMsg>().await? {
+        let symbol = &symbol_map[trade];
+        println!("Received trade for {symbol}: {trade:?}");
+    }
+    Ok(())
+}
+```
+
+To run this program, set the `DATABENTO_API_KEY` environment variable with an API key and run `cargo bin --example historical`.
+
 ### Live
 
 Real-time and intraday replay is provided through the Live clients.
-Here is a simple program that fetches the next ES mini futures trade:
+Here is a simple program that fetches the next E-mini S&P 500 futures trade:
 
-```rust
+```rust no_run
 use std::error::Error;
 
 use databento::{
@@ -65,54 +126,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 ```
-To run this program, set the `DATABENTO_API_KEY` environment variable with an API key and run `cargo run --example historical`
-
-### Historical
-
-Here is a simple program that fetches 10 minutes worth of historical trades for the entire CME Globex market:
-```rust
-use std::error::Error;
-
-use databento::{
-    dbn::{Schema, TradeMsg},
-    historical::timeseries::GetRangeParams,
-    HistoricalClient, Symbols,
-};
-use time::macros::{date, datetime};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
-    let mut client = HistoricalClient::builder().key_from_env()?.build()?;
-    let mut decoder = client
-        .timeseries()
-        .get_range(
-            &GetRangeParams::builder()
-                .dataset("GLBX.MDP3")
-                .date_time_range((
-                    datetime!(2022-06-10 14:30 UTC),
-                    datetime!(2022-06-10 14:40 UTC),
-                ))
-                .symbols(Symbols::All)
-                .schema(Schema::Trades)
-                .build(),
-        )
-        .await?;
-    let symbol_map = decoder
-        .metadata()
-        .symbol_map_for_date(date!(2022 - 06 - 10))?;
-    while let Some(trade) = decoder.decode_record::<TradeMsg>().await? {
-        let symbol = &symbol_map[trade];
-        println!("Received trade for {symbol}: {trade:?}");
-    }
-    Ok(())
-}
-```
-
-To run this program, set the `DATABENTO_API_KEY` environment variable with an API key and run `cargo bin --example live`.
-
-## Documentation
 
-You can find more detailed examples and the full API documentation on the [Databento docs site](https://databento.com/docs/quickstart?historical=rust&live=rust).
+To run this program, set the `DATABENTO_API_KEY` environment variable with an API key and run `cargo run --example live`
 
 ## License
 

examples/historical.rs

@@ -2,9 +2,9 @@
 use std::error::Error;
 
 use databento::{
-    dbn::{Schema, TradeMsg},
+    dbn::{decode::DbnMetadata, Dataset, SType, Schema, TradeMsg},
     historical::timeseries::GetRangeParams,
-    HistoricalClient, Symbols,
+    HistoricalClient,
 };
 use time::macros::{date, datetime};
 
@@ -15,12 +15,13 @@ async fn main() -> Result<(), Box<dyn Error>> {
         .timeseries()
         .get_range(
             &GetRangeParams::builder()
-                .dataset("GLBX.MDP3")
+                .dataset(Dataset::GlbxMdp3)
                 .date_time_range((
                     datetime!(2022-06-10 14:30 UTC),
                     datetime!(2022-06-10 14:40 UTC),
                 ))
-                .symbols(Symbols::All)
+                .symbols("ES.FUT")
+                .stype_in(SType::Parent)
                 .schema(Schema::Trades)
                 .build(),
         )

examples/split_symbols.rs

@@ -6,8 +6,9 @@ use anyhow::Context;
 use async_compression::tokio::write::ZstdEncoder;
 use databento::{
     dbn::{
-        decode::AsyncDbnDecoder, encode::AsyncDbnEncoder, InstrumentDefMsg, Metadata, Schema,
-        SymbolIndex,
+        decode::{AsyncDbnDecoder, DbnMetadata},
+        encode::{AsyncDbnEncoder, AsyncEncodeRecord, AsyncEncodeRecordRef},
+        InstrumentDefMsg, Metadata, Schema, SymbolIndex,
     },
     historical::timeseries::GetRangeParams,
     HistoricalClient,
@@ -55,7 +56,7 @@ async fn main() -> anyhow::Result<()> {
             encoders.insert(parent.clone(), encoder);
         };
     }
-    for (parent, encoder) in encoders {
+    for (parent, mut encoder) in encoders {
         if let Err(e) = encoder.shutdown().await {
             eprintln!("Failed to shutdown encoder for {parent}: {e:?}");
         }

src/historical/batch.rs

@@ -51,7 +51,6 @@ impl BatchClient<'_> {
             ("pretty_ts", params.pretty_ts.to_string()),
             ("map_symbols", params.map_symbols.to_string()),
             ("split_symbols", params.split_symbols.to_string()),
-            ("split_duration", params.split_duration.to_string()),
             ("delivery", params.delivery.to_string()),
             ("stype_in", params.stype_in.to_string()),
             ("stype_out", params.stype_out.to_string()),
@@ -64,6 +63,9 @@ impl BatchClient<'_> {
         if let Some(limit) = params.limit {
             form.push(("limit", limit.to_string()));
         }
+        if let Some(split_duration) = params.split_duration {
+            form.push(("split_duration", split_duration.to_string()));
+        }
         let builder = self.post("submit_job")?.form(&form);
         let resp = builder.send().await?;
         handle_response(resp).await
@@ -270,10 +272,11 @@ pub struct SubmitJobParams {
     /// If `true`, files will be split by raw symbol. Cannot be requested with [`Symbols::All`].
     #[builder(default)]
     pub split_symbols: bool,
-    /// The maximum time duration before batched data is split into multiple files.
-    /// Defaults to [`Day`](SplitDuration::Day).
-    #[builder(default)]
-    pub split_duration: SplitDuration,
+    /// The maximum time duration before batched data is split into multiple
+    /// files. If `None` the data will not be split by time. Defaults to
+    /// [`Day`](SplitDuration::Day).
+    #[builder(default = Some(SplitDuration::default()))]
+    pub split_duration: Option<SplitDuration>,
     /// The optional maximum size (in bytes) of each batched data file before being split.
     /// Must be an integer between 1e9 and 10e9 inclusive (1GB - 10GB). Defaults to `None`.
     #[builder(default, setter(strip_option))]
@@ -339,7 +342,7 @@ pub struct BatchJob {
     pub split_symbols: bool,
     /// The maximum time interval for an individual file before splitting into multiple
     /// files.
-    pub split_duration: SplitDuration,
+    pub split_duration: Option<SplitDuration>,
     /// The maximum size for an individual file before splitting into multiple files.
     pub split_size: Option<NonZeroU64>,
     /// The delivery mechanism of the batch data.
@@ -689,6 +692,41 @@ mod tests {
                      "ts_process_start": "2023-07-19 23:01:04.000000+00:00",
                      "ts_process_done": null,
                      "ts_expiration": null
+                },
+                {
+                    "id": "XNAS-20250602-5KM3HL5BUW",
+                    "user_id": "AA89XSlBV",
+                    "bill_id": null,
+                    "cost_usd": 0.0,
+                    "dataset": "XNAS.ITCH",
+                    "symbols": "MSFT",
+                    "stype_in": "raw_symbol",
+                    "stype_out": "instrument_id",
+                    "schema": "trades",
+                    "start": "2022-06-10T12:30:00.000000000Z",
+                    "end": "2022-06-10T14:00:00.000000000Z",
+                    "limit": 1000,
+                    "encoding": "csv",
+                    "compression": null,
+                    "pretty_px": false,
+                    "pretty_ts": false,
+                    "map_symbols": true,
+                    "split_symbols": false,
+                    "split_duration": null,
+                    "split_size": null,
+                    "packaging": null,
+                    "delivery": "download",
+                    "record_count": 1000,
+                    "billed_size": 48000,
+                    "actual_size": 94000,
+                    "package_size": 97690,
+                    "state": "done",
+                    "ts_received": "2025-06-02T15:51:19.251582000Z",
+                    "ts_queued": "2025-06-02T15:51:20.997673000Z",
+                    "ts_process_start": "2025-06-02T15:51:45.312317000Z",
+                    "ts_process_done": "2025-06-02T15:51:46.324860000Z",
+                    "ts_expiration": "2025-07-02T16:00:00.000000000Z",
+                    "progress": 100
                 }])),
             )
             .mount(&mock_server)
@@ -699,8 +737,8 @@ mod tests {
             HistoricalGateway::Bo1,
         )?;
         let job_descs = target.batch().list_jobs(&ListJobsParams::default()).await?;
-        assert_eq!(job_descs.len(), 1);
-        let job_desc = &job_descs[0];
+        assert_eq!(job_descs.len(), 2);
+        let mut job_desc = &job_descs[0];
         assert_eq!(
             job_desc.ts_queued.unwrap(),
             datetime!(2023-07-19 23:00:08.095538123 UTC)
@@ -713,6 +751,26 @@ mod tests {
         assert!(job_desc.pretty_px);
         assert!(!job_desc.pretty_ts);
         assert!(job_desc.map_symbols);
+        assert_eq!(job_desc.split_duration, Some(SplitDuration::Day));
+
+        job_desc = &job_descs[1];
+        assert_eq!(
+            job_desc.ts_queued.unwrap(),
+            datetime!(2025-06-02 15:51:20.997673000 UTC)
+        );
+        assert_eq!(
+            job_desc.ts_process_start.unwrap(),
+            datetime!(2025-06-02 15:51:45.312317000 UTC)
+        );
+        assert_eq!(job_desc.start, datetime!(2022-06-10 12:30:00.000000000 UTC));
+        assert_eq!(job_desc.end, datetime!(2022-06-10 14:00:00.000000000 UTC));
+        assert_eq!(job_desc.encoding, Encoding::Csv);
+        assert!(!job_desc.pretty_px);
+        assert!(!job_desc.pretty_ts);
+        assert!(job_desc.map_symbols);
+        assert!(!job_desc.split_symbols);
+        assert_eq!(job_desc.split_duration, None);
+
         Ok(())
     }