Skip to content

Commit a34e332

Browse files
mmalenicmmalenic
authored
Merge pull request #267 from umccr/refactor/c4gh-config
refactor: c4gh config
2 parents ded4d9d + 1210175 commit a34e332

File tree

27 files changed

+271
-225
lines changed

27 files changed

+271
-225
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deploy/config/dev_umccr.toml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,39 +26,39 @@ environment = "dev"
2626
[[resolvers]]
2727
regex = '^(org.umccr.dev.htsget-rs-test-data)/(?P<key>.*)$'
2828
substitution_string = '$key'
29-
storage = 'S3'
29+
storage.type = 'S3'
3030

3131
[[resolvers]]
3232
regex = '^(umccr-10c-data-dev)/(?P<key>.*)$'
3333
substitution_string = '$key'
34-
storage = 'S3'
34+
storage.type = 'S3'
3535

3636
[[resolvers]]
3737
regex = '^(umccr-10f-data-dev)/(?P<key>.*)$'
3838
substitution_string = '$key'
39-
storage = 'S3'
39+
storage.type = 'S3'
4040

4141
[[resolvers]]
4242
regex = '^(umccr-10g-data-dev)/(?P<key>.*)$'
4343
substitution_string = '$key'
44-
storage = 'S3'
44+
storage.type = 'S3'
4545

4646
[[resolvers]]
4747
regex = '^(umccr-agha-test-dev)/(?P<key>.*)$'
4848
substitution_string = '$key'
49-
storage = 'S3'
49+
storage.type = 'S3'
5050

5151
[[resolvers]]
5252
regex = '^(umccr-research-dev)/(?P<key>.*)$'
5353
substitution_string = '$key'
54-
storage = 'S3'
54+
storage.type = 'S3'
5555

5656
[[resolvers]]
5757
regex = '^(umccr-primary-data-dev)/(?P<key>.*)$'
5858
substitution_string = '$key'
59-
storage = 'S3'
59+
storage.type = 'S3'
6060

6161
[[resolvers]]
6262
regex = '^(umccr-validation-prod)/(?P<key>.*)$'
6363
substitution_string = '$key'
64-
storage = 'S3'
64+
storage.type = 'S3'

‎deploy/config/example_deploy.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@ environment = "dev"
1717
[[resolvers]]
1818
regex = '^(?P<bucket>.*?)/(?P<key>.*)$'
1919
substitution_string = '$key'
20-
storage = 'S3'
20+
storage.type = 'S3'

‎deploy/config/prod_umccr.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ environment = "prod"
2222
[[resolvers]]
2323
regex = '^(umccr-research-dev)/(?P<key>.*)$'
2424
substitution_string = '$key'
25-
storage = 'S3'
25+
storage.type = 'S3'
2626

2727
[[resolvers]]
2828
regex = '^(umccr-validation-prod)/(?P<key>.*)$'
2929
substitution_string = '$key'
30-
storage = 'S3'
30+
storage.type = 'S3'
3131

3232
[[resolvers]]
3333
regex = '^(umccr-primary-data-prod)/(?P<key>.*)$'
3434
substitution_string = '$key'
35-
storage = 'S3'
35+
storage.type = 'S3'

‎deploy/config/public_umccr.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ environment = 'public'
1111
[[resolvers]]
1212
regex = '^(org.umccr.demo.sbeacon-data)/CINECA_UK1/(?P<key>.*)$'
1313
substitution_string = 'CINECA_UK1/$key'
14-
storage = 'S3'
14+
storage.type = 'S3'
1515

1616
[[resolvers]]
1717
regex = '^(org.umccr.demo.htsget-rs-data)/(?P<type>bam|cram|vcf|bcf|crypt4gh|mixed)/(?P<key>.*)$'
1818
substitution_string = '$type/$key'
19-
storage = 'S3'
19+
storage.type = 'S3'

‎htsget-config/README.md‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -151,33 +151,34 @@ For more information about regex options see the [regex crate](https://docs.rs/r
151151
Each resolver also maps to a certain storage backend. This storage backend can be used to set query IDs which are served from local storage, from S3-style bucket storage, or from HTTP URLs.
152152
To set the storage backend for a resolver, add a `[resolvers.storage]` table. Some storage backends require feature flags to be set when compiling htsget-rs.
153153

154-
To use `LocalStorage`, set `storage = 'Local'`. This will derive the values for the fields below from the `data_server` config:
154+
To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and specify any additional options from below:
155155

156-
| Option | Description | When `storage = 'Local'` | Type | Default |
157-
|---------------------|-------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------|
158-
| `scheme` | The scheme present on URL tickets. | Derived from `data_server_key` and `data_server_cert`. If no key and cert are present, then uses `Http`, otherwise uses `Https`. | Either `'Http'` or `'Https'` | `'Http'` |
159-
| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | Same as `data_server_addr`. | URL authority | `'127.0.0.1:8081'` |
160-
| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Same as `data_server_local_path`. | Filesystem path | `'./'` |
161-
| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | Same as `data_server_serve_at`. | URL path | `''` |
156+
| Option | Description | Type | Default |
157+
|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------|
158+
| `scheme` | The scheme present on URL tickets. | Either `'Http'` or `'Https'` | `'Http'` |
159+
| `authority` | The authority present on URL tickets. This should likely match the `data_server_addr`. | URL authority | `'127.0.0.1:8081'` |
160+
| `local_path` | The local filesystem path which the data server uses to respond to tickets. This should likely match the `data_server_local_path`. | Filesystem path | `'./'` |
161+
| `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `''` |
162+
| `use_data_server_config` | Whether to use the data server config to fill in the above values. This overrides any other options specified from this table. | Boolean | `false` |
162163

163-
To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, and set `storage = 'S3'`. This will derive the value for `bucket` from the `regex` component of the `resolvers`:
164+
To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `type = 'S3'` under `[resolvers.storage]`, and specify any additional options from below:
164165

165-
| Option | Description | When `storage = 'S3'` | Type | Default |
166-
|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|---------|----------------------------------------|
167-
| `bucket` | The AWS S3 bucket where resources can be retrieved from. | Derived from the `resolvers` `regex` property. This uses the first capture group in the `regex` as the `bucket`. | String | `''` |
168-
| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | Not set, uses regular AWS S3 services. | String | Not set, uses regular AWS S3 services. |
169-
| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | `false` | Boolean | `false` |
166+
| Option | Description | Type | Default |
167+
|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------|
168+
| `bucket` | The AWS S3 bucket where resources can be retrieved from. | String | Derived from the `resolvers` `regex` property if empty. This uses the first capture group in the `regex` as the `bucket`. |
169+
| `endpoint` | A custom endpoint to override the default S3 service address. This is useful for using S3 locally or with storage backends such as MinIO. See [MinIO](#minio). | String | Not set, uses regular AWS S3 services. |
170+
| `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` |
170171

171172
`UrlStorage` is another storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication.
172-
To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, and set the following options under `[resolvers.storage]`:
173+
To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `type = 'Url'` under `[resolvers.storage]`, and specify any additional options from below:
173174

174-
| Option | Description | Type | Default |
175-
|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------|
176-
| <span id="url">`url`</span> | The URL to fetch data from. | HTTP URL | `"https://127.0.0.1:8081/"` |
177-
| <span id="url">`response_url`</span> | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` |
178-
| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. | Boolean | `true` |
179-
| `header_blacklist` | List of headers that should not be forwarded | Array of headers | `[]` |
180-
| `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [TLS](#tls) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. |
175+
| Option | Description | Type | Default |
176+
|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------|
177+
| <span id="url">`url`</span> | The URL to fetch data from. | HTTP URL | `"https://127.0.0.1:8081/"` |
178+
| <span id="url">`response_url`</span> | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` |
179+
| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. | Boolean | `true` |
180+
| `header_blacklist` | List of headers that should not be forwarded. | Array of headers | `[]` |
181+
| `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [TLS](#tls) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. |
181182

182183
When using `UrlStorage`, the following requests will be made to the `url`.
183184
* `GET` request to fetch only the headers of the data file (e.g. `GET /data.bam`, with `Range: bytes=0-<end_of_bam_header>`).
@@ -192,7 +193,9 @@ For example, a `resolvers` value of:
192193
[[resolvers]]
193194
regex = '^(example_bucket)/(?P<key>.*)$'
194195
substitution_string = '$key'
195-
storage = 'S3'
196+
[resolvers.storage]
197+
type = 'S3'
198+
# Uses the first capture group in the regex as the bucket.
196199
```
197200
Will use "example_bucket" as the S3 bucket if that resolver matches, because this is the first capture group in the `regex`.
198201
Note, to use this feature, at least one capture group must be defined in the `regex`.
@@ -206,6 +209,7 @@ regex = '.*'
206209
substitution_string = '$0'
207210

208211
[resolvers.storage]
212+
type = 'Local'
209213
scheme = 'Http'
210214
authority = '127.0.0.1:8081'
211215
local_path = './'
@@ -220,6 +224,7 @@ regex = '.*'
220224
substitution_string = '$0'
221225

222226
[resolvers.storage]
227+
type = 'S3'
223228
bucket = 'bucket'
224229
```
225230

@@ -231,6 +236,7 @@ regex = ".*"
231236
substitution_string = "$0"
232237

233238
[resolvers.storage]
239+
type = 'Url'
234240
url = "http://localhost:8080"
235241
response_url = "https://example.com"
236242
forward_headers = true

‎htsget-config/examples/config-files/c4gh.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,7 @@ regex = ".*"
99
substitution_string = "$0"
1010

1111
[resolvers.storage]
12-
object_type = { private_key = "data/c4gh/keys/bob.sec", recipient_public_key = "data/c4gh/keys/alice.pub" } # pragma: allowlist secret
12+
type = 'Local'
13+
14+
private_key = "data/c4gh/keys/bob.sec" # pragma: allowlist secret
15+
recipient_public_key = "data/c4gh/keys/alice.pub"

‎htsget-config/examples/config-files/s3_storage.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ data_server_enabled = false
1111
[[resolvers]]
1212
regex = '^(bucket)/(?P<key>.*)$'
1313
substitution_string = '$key'
14-
storage = 'S3'
14+
storage.type = 'S3'
1515

1616
# Or, set the bucket manually
1717
#[resolvers.storage]
18+
#type = 'S3'
1819
#bucket = 'bucket'

‎htsget-config/examples/config-files/tls_data_server.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,7 @@ data_server_tls.key = "key.pem"
1010
[[resolvers]]
1111
regex = ".*"
1212
substitution_string = "$0"
13+
14+
[resolvers.storage]
15+
type = 'Local'
16+
use_data_server_config = true

‎htsget-config/examples/config-files/tls_ticket_server.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ regex = ".*"
1212
substitution_string = "$0"
1313

1414
[resolvers.storage]
15+
type = 'S3'
1516
bucket = "bucket"

0 commit comments

Comments
 (0)