feat(config): add configuration values for C4GH S3 and Url storage

Author: mmalenic

htsget-actix/README.md

@@ -50,7 +50,7 @@ are exposed in the public API.
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features like `C4GHStorage`.
 
 ## Benchmarks
 Benchmarks for this crate written using [Criterion.rs][criterion-rs], and aim to compare the performance of this crate with the

htsget-axum/README.md

@@ -171,7 +171,7 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features like `C4GHStorage`.
 
 ## License
 

htsget-config/README.md

@@ -193,6 +193,7 @@ For example, a `resolvers` value of:
 [[resolvers]]
 regex = '^(example_bucket)/(?P<key>.*)$'
 substitution_string = '$key'
+
 [resolvers.storage]
 type = 'S3'
 # Uses the first capture group in the regex as the bucket.
@@ -275,6 +276,7 @@ regex = '.*'
 substitution_string = '$0'
 
 [resolvers.storage]
+type = 'S3'
 bucket = 'bucket'
 
 [resolvers.allow_guard]
@@ -452,6 +454,7 @@ export HTSGET_RESOLVERS="[{
     regex=regex,
     substitution_string=substitution_string,
     storage={
+        type=S3,
         bucket=bucket
     },
     allow_guard={
@@ -483,6 +486,7 @@ regex = '.*'
 substitution_string = '$0'
 
 [resolvers.storage]
+type = 'S3'
 bucket = 'bucket'
 endpoint = 'http://127.0.0.1:9000'
 path_style = true
@@ -504,8 +508,7 @@ serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and r
 them. When the client receives byte ranges from htsget-rs and concatenates them, the output bytes will be Crypt4GH encrypted,
 and will need to be decrypted before they can be read. All file formats (BAM, CRAM, VCF, and BCF) are supported using Crypt4GH.
 
-To use this feature, an additional config option called `object_type` under `resolvers.storage` is required,
-which allows specifying the private and public keys:
+To use this feature, additional config under `resolvers.storage` is required to specify the private and public keys:
 
 | Option                 | Description                                                                                                                                                              | Type              | Default |
 |------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------|
@@ -516,17 +519,17 @@ For example:
 
 ```toml
 [[resolvers]]
-regex = ".*"
-substitution_string = "$0"
+regex = '.*'
+substitution_string = '$0'
 
 [resolvers.storage]
-object_type = { private_key = "data/c4gh/keys/bob.sec", recipient_public_key = "data/c4gh/keys/alice.pub" } # pragma: allowlist secret
+type = 'Local'
+private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret
+recipient_public_key = 'data/c4gh/keys/alice.pub'
 ```
 
 The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure.
-
-> [!NOTE]  
-> This option is currently only supported for `LocalStorage`. The `object_type` will not have an effect if using `S3Storage` or `UrlStorage`.
+Any of the storage types are supported, i.e. `Local`, `S3`, or `Url`.
 
 ### As a library
 
@@ -541,7 +544,7 @@ regex, and changing it by using a substitution string.
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features like `C4GHStorage`.
 
 ## License
 

htsget-config/src/resolver.rs

@@ -499,7 +499,7 @@ mod tests {
   #[cfg(feature = "s3-storage")]
   #[tokio::test]
   async fn resolver_resolve_s3_request_tagged() {
-    let s3_storage = S3Storage::new("id".to_string(), None, false);
+    let s3_storage = S3Storage::new("id".to_string(), None, false, Default::default());
     let resolver = Resolver::new(
       Storage::S3(s3_storage),
       "(id)-1",
@@ -538,6 +538,7 @@ mod tests {
       }),
       true,
       vec![],
+      Default::default(),
       client,
     );
 
@@ -692,7 +693,8 @@ mod tests {
           Interval::new(Some(100), Some(1000)),
         );
         let resolver = config.resolvers().first().unwrap();
-        let expected_storage = S3Storage::new("bucket".to_string(), None, false);
+        let expected_storage =
+          S3Storage::new("bucket".to_string(), None, false, Default::default());
 
         assert_eq!(resolver.regex().to_string(), "regex");
         assert_eq!(resolver.substitution_string(), "substitution_string");

htsget-config/src/storage/object/c4gh.rs

@@ -64,13 +64,16 @@ impl From<Crypt4GHError> for Error {
 #[cfg(test)]
 mod tests {
   use crate::config::tests::test_config_from_file;
+  use crate::config::Config;
   use crate::storage::Storage;
   use std::fs::copy;
   use std::path::PathBuf;
   use tempfile::TempDir;
 
-  #[test]
-  fn config_storage_c4gh() {
+  fn test_c4gh_storage_config<F>(storage_config: &str, test_fn: F)
+  where
+    F: Fn(Config),
+  {
     let tmp = TempDir::new().unwrap();
     let private_key = tmp.path().join("bob.sec");
     let recipient_public_key = tmp.path().join("alice.pub");
@@ -94,18 +97,62 @@ mod tests {
         regex = "regex"
 
         [resolvers.storage]
-        type = "Local"
+        {}
         private_key = "{}"
         recipient_public_key = "{}"
         "#,
+        storage_config,
         private_key.to_string_lossy(),
         recipient_public_key.to_string_lossy()
       ),
       |config| {
         println!("{:?}", config.resolvers().first().unwrap().storage());
-        assert!(matches!(
+        test_fn(config);
+      },
+    );
+  }
+
+  #[test]
+  fn config_local_storage_c4gh() {
+    test_c4gh_storage_config(r#"type = "Local""#, |config| {
+      assert!(matches!(
             config.resolvers().first().unwrap().storage(),
             Storage::Local(local_storage) if local_storage.object_type().keys().is_some()
+      ));
+    });
+  }
+
+  #[cfg(feature = "s3-storage")]
+  #[test]
+  fn config_s3_storage_c4gh() {
+    test_c4gh_storage_config(
+      r#"
+        type = "S3"
+        bucket = "bucket"
+        "#,
+      |config| {
+        assert!(matches!(
+              config.resolvers().first().unwrap().storage(),
+              Storage::S3(s3_storage) if s3_storage.object_type().keys().is_some()
+        ));
+      },
+    );
+  }
+
+  #[cfg(feature = "url-storage")]
+  #[test]
+  fn config_url_storage_c4gh() {
+    test_c4gh_storage_config(
+      r#"
+        type = "Url"
+        url = "https://example.com/"
+        response_url = "https://example.com/"
+        forward_headers = false
+        "#,
+      |config| {
+        assert!(matches!(
+              config.resolvers().first().unwrap().storage(),
+              Storage::Url(url_storage) if url_storage.object_type().keys().is_some()
         ));
       },
     );

htsget-config/src/storage/s3.rs

@@ -1,3 +1,4 @@
+use crate::storage::object::ObjectType;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)]
@@ -6,15 +7,23 @@ pub struct S3Storage {
   pub(crate) bucket: String,
   pub(crate) endpoint: Option<String>,
   pub(crate) path_style: bool,
+  #[serde(flatten)]
+  pub(crate) object_type: ObjectType,
 }
 
 impl S3Storage {
   /// Create a new S3 storage.
-  pub fn new(bucket: String, endpoint: Option<String>, path_style: bool) -> Self {
+  pub fn new(
+    bucket: String,
+    endpoint: Option<String>,
+    path_style: bool,
+    object_type: ObjectType,
+  ) -> Self {
     Self {
       bucket,
       endpoint,
       path_style,
+      object_type,
     }
   }
 
@@ -32,6 +41,11 @@ impl S3Storage {
   pub fn path_style(self) -> bool {
     self.path_style
   }
+
+  /// Get the object type.
+  pub fn object_type(&self) -> &ObjectType {
+    &self.object_type
+  }
 }
 
 #[cfg(test)]

htsget-config/src/storage/url.rs

@@ -8,6 +8,7 @@ use serde_with::with_prefix;
 use crate::error::Error::ParseError;
 use crate::error::{Error, Result};
 use crate::storage::local::default_authority;
+use crate::storage::object::ObjectType;
 use crate::tls::client::TlsClientConfig;
 
 fn default_url() -> ValidatedUrl {
@@ -28,6 +29,8 @@ pub struct UrlStorage {
   header_blacklist: Vec<String>,
   #[serde(skip_serializing)]
   tls: TlsClientConfig,
+  #[serde(flatten)]
+  object_type: ObjectType,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -37,6 +40,7 @@ pub struct UrlStorageClient {
   response_url: ValidatedUrl,
   forward_headers: bool,
   header_blacklist: Vec<String>,
+  object_type: ObjectType,
   client: Client,
 }
 
@@ -66,6 +70,7 @@ impl TryFrom<UrlStorage> for UrlStorageClient {
       storage.response_url,
       storage.forward_headers,
       storage.header_blacklist,
+      storage.object_type,
       client,
     ))
   }
@@ -78,13 +83,15 @@ impl UrlStorageClient {
     response_url: ValidatedUrl,
     forward_headers: bool,
     header_blacklist: Vec<String>,
+    object_type: ObjectType,
     client: Client,
   ) -> Self {
     Self {
       url,
       response_url,
       forward_headers,
       header_blacklist,
+      object_type,
       client,
     }
   }
@@ -113,6 +120,11 @@ impl UrlStorageClient {
   pub fn client_cloned(&self) -> Client {
     self.client.clone()
   }
+
+  /// Get the object type.
+  pub fn object_type(&self) -> &ObjectType {
+    &self.object_type
+  }
 }
 
 /// A wrapper around `http::Uri` type which implements serialize and deserialize.
@@ -154,6 +166,7 @@ impl UrlStorage {
     forward_headers: bool,
     header_blacklist: Vec<String>,
     tls: TlsClientConfig,
+    object_type: ObjectType,
   ) -> Self {
     Self {
       url: ValidatedUrl(Url { inner: url }),
@@ -163,6 +176,7 @@ impl UrlStorage {
       forward_headers,
       header_blacklist,
       tls,
+      object_type,
     }
   }
 
@@ -186,6 +200,11 @@ impl UrlStorage {
   pub fn tls(&self) -> &TlsClientConfig {
     &self.tls
   }
+
+  /// Get the object type.
+  pub fn object_type(&self) -> &ObjectType {
+    &self.object_type
+  }
 }
 
 impl Default for UrlStorage {
@@ -196,6 +215,7 @@ impl Default for UrlStorage {
       forward_headers: true,
       header_blacklist: vec![],
       tls: TlsClientConfig::default(),
+      object_type: Default::default(),
     }
   }
 }
@@ -221,6 +241,7 @@ mod tests {
         true,
         vec![],
         client_config,
+        Default::default(),
       ));
 
       assert!(url_storage.is_ok());

htsget-http/README.md

@@ -38,7 +38,7 @@ These functions take query and endpoint information, and process it using [htsge
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features like `C4GHStorage`.
 
 [warp]: https://github.com/seanmonstar/warp
 [htsget-search]: ../htsget-search

htsget-lambda/README.md

@@ -46,7 +46,7 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features like `C4GHStorage`.
 
 ## License
 

htsget-search/README.md

@@ -59,7 +59,7 @@ used to process requests.
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features like `C4GHStorage`.
 
 ## Minimising Byte Ranges