feat: add C4GHStorage on Url and S3 storage backends

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - name: Cache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].5
       - name: Check out
         uses: actions/checkout@v4
       - name: Install Rust

.github/workflows/release.yml

@@ -32,7 +32,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Docker GitHub release
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: deploy/Dockerfile

.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - name: Cache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].5
       - name: Check out
         uses: actions/checkout@v4
       - name: Install Rust

deploy/config/dev_umccr.toml

@@ -26,39 +26,39 @@ environment = "dev"
 [[resolvers]]
 regex = '^(org.umccr.dev.htsget-rs-test-data)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-10c-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-10f-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-10g-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-agha-test-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-research-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-primary-data-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-validation-prod)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'

deploy/config/example_deploy.toml

@@ -17,4 +17,4 @@ environment = "dev"
 [[resolvers]]
 regex = '^(?P<bucket>.*?)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'

deploy/config/prod_umccr.toml

@@ -22,14 +22,14 @@ environment = "prod"
 [[resolvers]]
 regex = '^(umccr-research-dev)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-validation-prod)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(umccr-primary-data-prod)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'

deploy/config/public_umccr.toml

@@ -11,9 +11,9 @@ environment = 'public'
 [[resolvers]]
 regex = '^(org.umccr.demo.sbeacon-data)/CINECA_UK1/(?P<key>.*)$'
 substitution_string = 'CINECA_UK1/$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 [[resolvers]]
 regex = '^(org.umccr.demo.htsget-rs-data)/(?P<type>bam|cram|vcf|bcf|crypt4gh|mixed)/(?P<key>.*)$'
 substitution_string = '$type/$key'
-storage.type = 'S3'
+storage.backend = 'S3'

htsget-actix/README.md

@@ -50,7 +50,7 @@ are exposed in the public API.
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`.
 
 ## Benchmarks
 Benchmarks for this crate written using [Criterion.rs][criterion-rs], and aim to compare the performance of this crate with the

htsget-axum/README.md

@@ -171,7 +171,7 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`.
 
 ## License
 

htsget-config/README.md

@@ -151,7 +151,7 @@ For more information about regex options see the [regex crate](https://docs.rs/r
 Each resolver also maps to a certain storage backend. This storage backend can be used to set query IDs which are served from local storage, from S3-style bucket storage, or from HTTP URLs.
 To set the storage backend for a resolver, add a `[resolvers.storage]` table. Some storage backends require feature flags to be set when compiling htsget-rs.
 
-To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and specify any additional options from below:
+To use `LocalStorage`, set `backend = 'Local'` under `[resolvers.storage]`, and specify any additional options from below:
 
 | Option                   | Description                                                                                                                         | Type                         | Default            |
 |--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------|
@@ -161,7 +161,7 @@ To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and spe
 | `path_prefix`            | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path.                          | URL path                     | `''`               |
 | `use_data_server_config` | Whether to use the data server config to fill in the above values. This overrides any other options specified from this table.      | Boolean                      | `false`            |
 
-To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `type = 'S3'` under `[resolvers.storage]`, and specify any additional options from below:
+To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `backend = 'S3'` under `[resolvers.storage]`, and specify any additional options from below:
 
 | Option       | Description                                                                                                                                                                   | Type    | Default                                                                                                                   |
 |--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------|
@@ -170,7 +170,7 @@ To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `
 | `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used.  | Boolean | `false`                                                                                                                   |
 
 `UrlStorage` is another storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. 
-To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `type = 'Url'` under `[resolvers.storage]`, and specify any additional options from below:
+To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `backend = 'Url'` under `[resolvers.storage]`, and specify any additional options from below:
 
 | Option                               | Description                                                                                                                 | Type                     | Default                                                                                                         |
 |--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------|
@@ -193,8 +193,9 @@ For example, a `resolvers` value of:
 [[resolvers]]
 regex = '^(example_bucket)/(?P<key>.*)$'
 substitution_string = '$key'
+
 [resolvers.storage]
-type = 'S3'
+backend = 'S3'
 # Uses the first capture group in the regex as the bucket.
 ```
 Will use "example_bucket" as the S3 bucket if that resolver matches, because this is the first capture group in the `regex`.
@@ -209,7 +210,7 @@ regex = '.*'
 substitution_string = '$0'
 
 [resolvers.storage]
-type = 'Local'
+backend = 'Local'
 scheme = 'Http'
 authority = '127.0.0.1:8081'
 local_path = './'
@@ -224,7 +225,7 @@ regex = '.*'
 substitution_string = '$0'
 
 [resolvers.storage]
-type = 'S3'
+backend = 'S3'
 bucket = 'bucket'
 ```
 
@@ -236,7 +237,7 @@ regex = ".*"
 substitution_string = "$0"
 
 [resolvers.storage]
-type = 'Url'
+backend = 'Url'
 url = "http://localhost:8080"
 response_url = "https://example.com"
 forward_headers = true
@@ -275,6 +276,7 @@ regex = '.*'
 substitution_string = '$0'
 
 [resolvers.storage]
+backend = 'S3'
 bucket = 'bucket'
 
 [resolvers.allow_guard]
@@ -452,6 +454,7 @@ export HTSGET_RESOLVERS="[{
     regex=regex,
     substitution_string=substitution_string,
     storage={
+        type=S3,
         bucket=bucket
     },
     allow_guard={
@@ -483,6 +486,7 @@ regex = '.*'
 substitution_string = '$0'
 
 [resolvers.storage]
+backend = 'S3'
 bucket = 'bucket'
 endpoint = 'http://127.0.0.1:9000'
 path_style = true
@@ -504,8 +508,7 @@ serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and r
 them. When the client receives byte ranges from htsget-rs and concatenates them, the output bytes will be Crypt4GH encrypted,
 and will need to be decrypted before they can be read. All file formats (BAM, CRAM, VCF, and BCF) are supported using Crypt4GH.
 
-To use this feature, an additional config option called `object_type` under `resolvers.storage` is required,
-which allows specifying the private and public keys:
+To use this feature, additional config under `resolvers.storage` is required to specify the private and public keys:
 
 | Option                 | Description                                                                                                                                                              | Type              | Default |
 |------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------|
@@ -516,17 +519,17 @@ For example:
 
 ```toml
 [[resolvers]]
-regex = ".*"
-substitution_string = "$0"
+regex = '.*'
+substitution_string = '$0'
 
 [resolvers.storage]
-object_type = { private_key = "data/c4gh/keys/bob.sec", recipient_public_key = "data/c4gh/keys/alice.pub" } # pragma: allowlist secret
+backend = 'Local'
+private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret
+recipient_public_key = 'data/c4gh/keys/alice.pub'
 ```
 
 The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure.
-
-> [!NOTE]  
-> This option is currently only supported for `LocalStorage`. The `object_type` will not have an effect if using `S3Storage` or `UrlStorage`.
+Any of the storage types are supported, i.e. `Local`, `S3`, or `Url`.
 
 ### As a library
 
@@ -541,7 +544,7 @@ regex, and changing it by using a substitution string.
 This crate has the following features:
 * `s3-storage`: used to enable `S3Storage` functionality.
 * `url-storage`: used to enable `UrlStorage` functionality.
-* `experimental`: used to enable `C4GHStorage` functionality.
+* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`.
 
 ## License
 

htsget-config/examples/config-files/c4gh.toml

@@ -9,7 +9,7 @@ regex = ".*"
 substitution_string = "$0"
 
 [resolvers.storage]
-type = 'Local'
+backend = 'Local'
 
 private_key = "data/c4gh/keys/bob.sec" # pragma: allowlist secret
 recipient_public_key = "data/c4gh/keys/alice.pub"

htsget-config/examples/config-files/s3_storage.toml

@@ -11,9 +11,9 @@ data_server_enabled = false
 [[resolvers]]
 regex = '^(bucket)/(?P<key>.*)$'
 substitution_string = '$key'
-storage.type = 'S3'
+storage.backend = 'S3'
 
 # Or, set the bucket manually
 #[resolvers.storage]
-#type = 'S3'
+#backend = 'S3'
 #bucket = 'bucket'

htsget-config/examples/config-files/tls_data_server.toml

@@ -12,5 +12,5 @@ regex = ".*"
 substitution_string = "$0"
 
 [resolvers.storage]
-type = 'Local'
+backend = 'Local'
 use_data_server_config = true

htsget-config/examples/config-files/tls_ticket_server.toml

@@ -12,5 +12,5 @@ regex = ".*"
 substitution_string = "$0"
 
 [resolvers.storage]
-type = 'S3'
+backend = 'S3'
 bucket = "bucket"

htsget-config/examples/config-files/url_storage.toml

@@ -16,7 +16,7 @@ regex = ".*"
 substitution_string = "$0"
 
 [resolvers.storage]
-type = 'Url'
+backend = 'Url'
 url = "http://127.0.0.1:8081"
 response_url = "https://127.0.0.1:8081"
 forward_headers = true

htsget-config/src/config/mod.rs

@@ -775,7 +775,7 @@ pub(crate) mod tests {
 
     [[resolvers]]
     [resolvers.storage]
-    type = "Local"
+    backend = "Local"
     use_data_server_config = true
     "#,
       |config| {

htsget-config/src/resolver.rs

@@ -499,7 +499,7 @@ mod tests {
   #[cfg(feature = "s3-storage")]
   #[tokio::test]
   async fn resolver_resolve_s3_request_tagged() {
-    let s3_storage = S3Storage::new("id".to_string(), None, false);
+    let s3_storage = S3Storage::new("id".to_string(), None, false, Default::default());
     let resolver = Resolver::new(
       Storage::S3(s3_storage),
       "(id)-1",
@@ -538,6 +538,7 @@ mod tests {
       }),
       true,
       vec![],
+      Default::default(),
       client,
     );
 
@@ -677,7 +678,7 @@ mod tests {
       vec![(
         "HTSGET_RESOLVERS",
         "[{ regex=regex, substitution_string=substitution_string, \
-        storage={ type=S3, bucket=bucket }, \
+        storage={ backend=S3, bucket=bucket }, \
         allow_guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \
         allow_formats=[BAM], allow_classes=[body], allow_interval_start=100, \
         allow_interval_end=1000 } }]",
@@ -692,7 +693,8 @@ mod tests {
           Interval::new(Some(100), Some(1000)),
         );
         let resolver = config.resolvers().first().unwrap();
-        let expected_storage = S3Storage::new("bucket".to_string(), None, false);
+        let expected_storage =
+          S3Storage::new("bucket".to_string(), None, false, Default::default());
 
         assert_eq!(resolver.regex().to_string(), "regex");
         assert_eq!(resolver.substitution_string(), "substitution_string");

htsget-config/src/storage/local.rs

@@ -126,7 +126,7 @@ mod tests {
         regex = "regex"
 
         [resolvers.storage]
-        type = "Local"
+        backend = "Local"
         local_path = "path"
         scheme = "HTTPS"
         path_prefix = "path"