Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,6 @@ jobs:
- uses: actions/checkout@v5
- name: Install cargo-msrv
run: cargo install cargo-msrv
- name: Downgrade object_store dependencies
# Necessary because tokio 1.30.0 updates MSRV to 1.63
# and url 2.5.1, updates to 1.67
run: |
cargo update -p tokio --precise 1.29.1
cargo update -p url --precise 2.5.0
cargo update -p once_cell --precise 1.20.3
cargo update -p tracing-core --precise 0.1.33
cargo update -p tracing-attributes --precise 0.1.28
cargo update -p parking_lot --precise 0.12.4
cargo update -p parking_lot_core --precise 0.9.11
cargo update -p lock_api --precise 0.4.13
cargo update -p syn --precise 2.0.106
- name: Check
run: |
# run `cargo msrv verify` to see problems
Expand Down
4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
[package]
name = "object_store"
version = "0.12.4"
edition = "2021"
edition = "2024"
license = "MIT/Apache-2.0"
readme = "README.md"
description = "A generic object store interface for uniformly interacting with AWS S3, Google Cloud Storage, Azure Blob Storage and local files."
keywords = ["object", "storage", "cloud"]
repository = "https://github.com/apache/arrow-rs-object-store"
rust-version = "1.64.0"
rust-version = "1.85"
include = ["src/**/*.rs", "README.md", "LICENSE.txt", "NOTICE.txt", "Cargo.toml"]

[package.metadata.docs.rs]
Expand Down
9 changes: 6 additions & 3 deletions src/aws/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ use crate::aws::{
AmazonS3, AwsCredential, AwsCredentialProvider, Checksum, S3ConditionalPut, S3CopyIfNotExists,
STORE,
};
use crate::client::{http_connector, HttpConnector, TokenCredentialProvider};
use crate::client::{HttpConnector, TokenCredentialProvider, http_connector};
use crate::config::ConfigValue;
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use base64::prelude::BASE64_STANDARD;
use itertools::Itertools;
use md5::{Digest, Md5};
use reqwest::header::{HeaderMap, HeaderValue};
Expand Down Expand Up @@ -75,7 +75,10 @@ enum Error {
#[error("Invalid Zone suffix for bucket '{bucket}'")]
ZoneSuffix { bucket: String },

#[error("Invalid encryption type: {}. Valid values are \"AES256\", \"sse:kms\", \"sse:kms:dsse\" and \"sse-c\".", passed)]
#[error(
"Invalid encryption type: {}. Valid values are \"AES256\", \"sse:kms\", \"sse:kms:dsse\" and \"sse-c\".",
passed
)]
InvalidEncryptionType { passed: String },

#[error(
Expand Down
12 changes: 6 additions & 6 deletions src/aws/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ use crate::aws::builder::S3EncryptionHeaders;
use crate::aws::checksum::Checksum;
use crate::aws::credential::{AwsCredential, CredentialExt};
use crate::aws::{
AwsAuthorizer, AwsCredentialProvider, S3ConditionalPut, S3CopyIfNotExists, COPY_SOURCE_HEADER,
AwsAuthorizer, AwsCredentialProvider, COPY_SOURCE_HEADER, S3ConditionalPut, S3CopyIfNotExists,
STORE, STRICT_PATH_ENCODE_SET, TAGS_HEADER,
};
use crate::client::builder::{HttpRequestBuilder, RequestBuilderError};
use crate::client::get::GetClient;
use crate::client::header::{get_etag, HeaderConfig};
use crate::client::header::{HeaderConfig, get_etag};
use crate::client::header::{get_put_result, get_version};
use crate::client::list::ListClient;
use crate::client::retry::{RetryContext, RetryExt};
Expand All @@ -40,8 +40,8 @@ use crate::{
PutPayload, PutResult, Result, RetryConfig, TagSet,
};
use async_trait::async_trait;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use base64::prelude::BASE64_STANDARD;
use bytes::{Buf, Bytes};
use http::header::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH,
Expand All @@ -50,7 +50,7 @@ use http::header::{
use http::{HeaderMap, HeaderName, Method};
use itertools::Itertools;
use md5::{Digest, Md5};
use percent_encoding::{utf8_percent_encode, PercentEncode};
use percent_encoding::{PercentEncode, utf8_percent_encode};
use quick_xml::events::{self as xml_events};
use ring::digest;
use ring::digest::Context;
Expand Down Expand Up @@ -953,10 +953,10 @@ fn encode_path(path: &Path) -> PercentEncode<'_> {
#[cfg(test)]
mod tests {
use super::*;
use crate::client::mock_server::MockServer;
use crate::client::HttpClient;
use http::header::CONTENT_LENGTH;
use crate::client::mock_server::MockServer;
use http::Response;
use http::header::CONTENT_LENGTH;

#[tokio::test]
async fn test_create_multipart_has_content_length() {
Expand Down
29 changes: 22 additions & 7 deletions src/aws/credential.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use crate::{CredentialProvider, Result, RetryConfig};
use async_trait::async_trait;
use bytes::Buf;
use chrono::{DateTime, Utc};
use http::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION};
use http::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue};
use http::{Method, StatusCode};
use percent_encoding::utf8_percent_encode;
use serde::Deserialize;
Expand Down Expand Up @@ -863,8 +863,8 @@ struct CreateSessionOutput {
mod tests {
use super::*;
use crate::aws::{AmazonS3Builder, AmazonS3ConfigKey};
use crate::client::mock_server::MockServer;
use crate::client::HttpClient;
use crate::client::mock_server::MockServer;
use http::Response;
use reqwest::{Client, Method};
use std::env;
Expand Down Expand Up @@ -908,7 +908,10 @@ mod tests {
};

signer.authorize(&mut request, None);
assert_eq!(request.headers().get(&AUTHORIZATION).unwrap(), "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a3c787a7ed37f7fdfbfd2d7056a3d7c9d85e6d52a2bfbec73793c0be6e7862d4")
assert_eq!(
request.headers().get(&AUTHORIZATION).unwrap(),
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a3c787a7ed37f7fdfbfd2d7056a3d7c9d85e6d52a2bfbec73793c0be6e7862d4"
)
}

#[test]
Expand Down Expand Up @@ -949,7 +952,10 @@ mod tests {
};

signer.authorize(&mut request, None);
assert_eq!(request.headers().get(&AUTHORIZATION).unwrap(), "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-request-payer, Signature=7030625a9e9b57ed2a40e63d749f4a4b7714b6e15004cab026152f870dd8565d")
assert_eq!(
request.headers().get(&AUTHORIZATION).unwrap(),
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-request-payer, Signature=7030625a9e9b57ed2a40e63d749f4a4b7714b6e15004cab026152f870dd8565d"
)
}

#[test]
Expand Down Expand Up @@ -990,7 +996,10 @@ mod tests {
};

authorizer.authorize(&mut request, None);
assert_eq!(request.headers().get(&AUTHORIZATION).unwrap(), "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=653c3d8ea261fd826207df58bc2bb69fbb5003e9eb3c0ef06e4a51f2a81d8699");
assert_eq!(
request.headers().get(&AUTHORIZATION).unwrap(),
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=653c3d8ea261fd826207df58bc2bb69fbb5003e9eb3c0ef06e4a51f2a81d8699"
);
}

#[test]
Expand Down Expand Up @@ -1113,7 +1122,10 @@ mod tests {
};

authorizer.authorize(&mut request, None);
assert_eq!(request.headers().get(&AUTHORIZATION).unwrap(), "AWS4-HMAC-SHA256 Credential=H20ABqCkLZID4rLe/20220809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9ebf2f92872066c99ac94e573b4e1b80f4dbb8a32b1e8e23178318746e7d1b4d")
assert_eq!(
request.headers().get(&AUTHORIZATION).unwrap(),
"AWS4-HMAC-SHA256 Credential=H20ABqCkLZID4rLe/20220809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9ebf2f92872066c99ac94e573b4e1b80f4dbb8a32b1e8e23178318746e7d1b4d"
)
}

#[tokio::test]
Expand Down Expand Up @@ -1320,7 +1332,10 @@ mod tests {
(" a b c ", "a b c"),
("a \t b c ", "a b c"),
("\"a \t b c ", "\"a b c"),
(" \t\n\u{000b}\r\u{000c}a \t\n\u{000b}\r\u{000c} b \t\n\u{000b}\r\u{000c} c \t\n\u{000b}\r\u{000c}", "a b c"),
(
" \t\n\u{000b}\r\u{000c}a \t\n\u{000b}\r\u{000c} b \t\n\u{000b}\r\u{000c} c \t\n\u{000b}\r\u{000c}",
"a b c",
),
];

for (input, expected) in test_cases {
Expand Down
10 changes: 5 additions & 5 deletions src/aws/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ use std::{sync::Arc, time::Duration};
use url::Url;

use crate::aws::client::{CompleteMultipartMode, PutPartPayload, RequestError, S3Client};
use crate::client::CredentialProvider;
use crate::client::get::GetClientExt;
use crate::client::list::{ListClient, ListClientExt};
use crate::client::CredentialProvider;
use crate::multipart::{MultipartStore, PartId};
use crate::signer::Signer;
use crate::util::STRICT_ENCODE_SET;
Expand Down Expand Up @@ -358,7 +358,7 @@ impl ObjectStore for AmazonS3 {
None => {
return Err(Error::NotSupported {
source: "S3 does not support copy-if-not-exists".to_string().into(),
})
});
}
};

Expand Down Expand Up @@ -493,14 +493,14 @@ impl PaginatedListStore for AmazonS3 {
#[cfg(test)]
mod tests {
use super::*;
use crate::ClientOptions;
use crate::client::SpawnedReqwestConnector;
use crate::client::get::GetClient;
use crate::client::retry::RetryContext;
use crate::client::SpawnedReqwestConnector;
use crate::integration::*;
use crate::tests::*;
use crate::ClientOptions;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use base64::prelude::BASE64_STANDARD;
use http::HeaderMap;

const NON_EXISTENT_NAME: &str = "nonexistentname";
Expand Down
2 changes: 1 addition & 1 deletion src/azure/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use crate::azure::credential::{
ImdsManagedIdentityProvider, WorkloadIdentityOAuthProvider,
};
use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE};
use crate::client::{http_connector, HttpConnector, TokenCredentialProvider};
use crate::client::{HttpConnector, TokenCredentialProvider, http_connector};
use crate::config::ConfigValue;
use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider};
use percent_encoding::percent_decode_str;
Expand Down
10 changes: 5 additions & 5 deletions src/azure/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,25 @@ use crate::azure::credential::*;
use crate::azure::{AzureCredentialProvider, STORE};
use crate::client::builder::HttpRequestBuilder;
use crate::client::get::GetClient;
use crate::client::header::{get_put_result, HeaderConfig};
use crate::client::header::{HeaderConfig, get_put_result};
use crate::client::list::ListClient;
use crate::client::retry::{RetryContext, RetryExt};
use crate::client::{GetOptionsExt, HttpClient, HttpError, HttpRequest, HttpResponse};
use crate::list::{PaginatedListOptions, PaginatedListResult};
use crate::multipart::PartId;
use crate::util::{deserialize_rfc1123, GetRange};
use crate::util::{GetRange, deserialize_rfc1123};
use crate::{
Attribute, Attributes, ClientOptions, GetOptions, ListResult, ObjectMeta, Path, PutMode,
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, RetryConfig, TagSet,
};
use async_trait::async_trait;
use base64::prelude::{BASE64_STANDARD, BASE64_STANDARD_NO_PAD};
use base64::Engine;
use base64::prelude::{BASE64_STANDARD, BASE64_STANDARD_NO_PAD};
use bytes::{Buf, Bytes};
use chrono::{DateTime, Utc};
use http::{
header::{HeaderMap, HeaderValue, CONTENT_LENGTH, CONTENT_TYPE, IF_MATCH, IF_NONE_MATCH},
HeaderName, Method,
header::{CONTENT_LENGTH, CONTENT_TYPE, HeaderMap, HeaderValue, IF_MATCH, IF_NONE_MATCH},
};
use rand::Rng as _;
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -497,7 +497,7 @@ async fn parse_blob_batch_delete_body(
code: code.to_string(),
reason: part_response.reason.unwrap_or_default().to_string(),
}
.into())
.into());
}
_ => return Err(invalid_response("missing part response status code").into()),
}
Expand Down
14 changes: 7 additions & 7 deletions src/azure/credential.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,23 @@
// under the License.

use super::client::UserDelegationKey;
use crate::RetryConfig;
use crate::azure::STORE;
use crate::client::builder::{add_query_pairs, HttpRequestBuilder};
use crate::client::builder::{HttpRequestBuilder, add_query_pairs};
use crate::client::retry::RetryExt;
use crate::client::token::{TemporaryToken, TokenCache};
use crate::client::{CredentialProvider, HttpClient, HttpError, HttpRequest, TokenProvider};
use crate::util::hmac_sha256;
use crate::RetryConfig;
use async_trait::async_trait;
use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE_NO_PAD};
use base64::Engine;
use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE_NO_PAD};
use chrono::{DateTime, SecondsFormat, Utc};
use http::Method;
use http::header::{
HeaderMap, HeaderName, HeaderValue, ACCEPT, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LANGUAGE,
CONTENT_LENGTH, CONTENT_TYPE, DATE, IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH,
ACCEPT, AUTHORIZATION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, CONTENT_TYPE, DATE,
HeaderMap, HeaderName, HeaderValue, IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH,
IF_UNMODIFIED_SINCE, RANGE,
};
use http::Method;
use serde::Deserialize;
use std::borrow::Cow;
use std::collections::HashMap;
Expand Down Expand Up @@ -1079,7 +1079,7 @@ mod tests {
async fn test_managed_identity() {
let server = MockServer::new().await;

std::env::set_var(MSI_SECRET_ENV_KEY, "env-secret");
unsafe { std::env::set_var(MSI_SECRET_ENV_KEY, "env-secret") };

let endpoint = server.url();
let client = HttpClient::new(Client::new());
Expand Down
8 changes: 4 additions & 4 deletions src/azure/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@
//!
//! Unused blocks will automatically be dropped after 7 days.
use crate::{
GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore,
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart,
multipart::{MultipartStore, PartId},
path::Path,
signer::Signer,
GetOptions, GetResult, ListResult, MultipartId, MultipartUpload, ObjectMeta, ObjectStore,
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart,
};
use async_trait::async_trait;
use futures::stream::{BoxStream, StreamExt, TryStreamExt};
Expand All @@ -37,10 +37,10 @@ use std::sync::Arc;
use std::time::Duration;
use url::Url;

use crate::client::CredentialProvider;
use crate::client::get::GetClientExt;
use crate::client::list::{ListClient, ListClientExt};
use crate::client::CredentialProvider;
pub use credential::{authority_hosts, AzureAccessKey, AzureAuthorizer};
pub use credential::{AzureAccessKey, AzureAuthorizer, authority_hosts};

mod builder;
mod client;
Expand Down
4 changes: 2 additions & 2 deletions src/chunked.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ use std::sync::Arc;

use async_trait::async_trait;
use bytes::{BufMut, Bytes, BytesMut};
use futures::stream::BoxStream;
use futures::StreamExt;
use futures::stream::BoxStream;

use crate::path::Path;
use crate::{
Expand Down Expand Up @@ -120,7 +120,7 @@ impl ObjectStore for ChunkedStore {
source: Box::new(e),
}),
(stream, buffer, exhausted, chunk_size),
))
));
}
};
}
Expand Down
8 changes: 4 additions & 4 deletions src/client/get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

use crate::client::header::{get_etag, header_meta, HeaderConfig};
use crate::client::header::{HeaderConfig, get_etag, header_meta};
use crate::client::retry::RetryContext;
use crate::client::{HttpResponse, HttpResponseBody};
use crate::path::Path;
Expand All @@ -25,13 +25,13 @@ use crate::{
};
use async_trait::async_trait;
use bytes::Bytes;
use futures::stream::BoxStream;
use futures::StreamExt;
use futures::stream::BoxStream;
use http::StatusCode;
use http::header::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_RANGE,
CONTENT_TYPE,
};
use http::StatusCode;
use http_body_util::BodyExt;
use reqwest::header::ToStrError;
use std::ops::Range;
Expand Down Expand Up @@ -494,7 +494,7 @@ mod http_tests {
use http::{Response, StatusCode};
use hyper::body::Frame;
use std::pin::Pin;
use std::task::{ready, Context, Poll};
use std::task::{Context, Poll, ready};
use std::time::Duration;

#[derive(Debug, thiserror::Error)]
Expand Down
Loading