Merge pull request #334 from umccr/feat/cache-http

feat: add HTTP caching

Author: mmalenic

Cargo.lock

@@ -71,7 +71,7 @@ impl DataServer {
 }
 
 impl From<DataServerConfig> for BindServer {
-  /// Returns a data server with TLS enabled if the tls config is not None or without TLS enabled
+  /// Returns a data server with TLS enabled if the http config is not None or without TLS enabled
   /// if it is None.
   fn from(config: DataServerConfig) -> Self {
     let addr = config.addr();
@@ -114,7 +114,7 @@ mod tests {
   use tokio::io::AsyncWriteExt;
 
   use htsget_config::config::Config;
-  use htsget_config::tls::TlsServerConfig;
+  use htsget_config::http::TlsServerConfig;
   use htsget_config::types::Scheme;
   use htsget_test::http::cors::{test_cors_preflight_request_uri, test_cors_simple_request_uri};
   use htsget_test::http::{

htsget-axum/src/server/data.rs

@@ -17,7 +17,7 @@ use axum::extract::Request;
 use htsget_config::config::advanced::auth::AuthConfig;
 use htsget_config::config::advanced::cors::CorsConfig;
 use htsget_config::config::service_info::ServiceInfo;
-use htsget_config::tls::TlsServerConfig;
+use htsget_config::http::TlsServerConfig;
 use htsget_config::types::Scheme;
 use htsget_http::middleware::auth::Auth;
 use htsget_search::HtsGet;
@@ -245,7 +245,7 @@ impl Server {
 
           tokio::spawn(async move {
             let Ok(stream) = tls_acceptor.accept(cnx).await else {
-              error!("error during tls handshake connection from {}", addr);
+              error!("error during http handshake connection from {}", addr);
               return;
             };
 

htsget-axum/src/server/mod.rs

@@ -22,7 +22,7 @@ use tower_http::trace::TraceLayer;
 use tracing::info;
 
 impl From<TicketServerConfig> for BindServer {
-  /// Returns a ticket server with TLS enabled if the tls config is not None or without TLS enabled
+  /// Returns a ticket server with TLS enabled if the http config is not None or without TLS enabled
   /// if it is None.
   fn from(config: TicketServerConfig) -> Self {
     let addr = config.addr();

htsget-axum/src/server/ticket.rs

@@ -37,6 +37,8 @@ rustls = "0.23"
 rustls-pki-types = "1"
 chrono = { version = "0.4", features = ["now"], default-features = false }
 reqwest = { version = "0.12", features = ["rustls-tls"], default-features = false }
+http-cache-reqwest = "0.16"
+reqwest-middleware = "0.4"
 schemars = "1"
 
 # Crypt4GH

htsget-config/Cargo.toml

@@ -253,13 +253,13 @@ backend.path_style = true
 
 To manually configure `Url` locations, set `backend.kind = "Url"`, specify any additional options from below under the `backend` table:
 
-| Option                               | Description                                                                                                                                                   | Type                     | Default                                                                                                         |
-|--------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------|
-| <span id="url">`url`</span>          | The URL to fetch data from.                                                                                                                                   | HTTP URL                 | `"https://127.0.0.1:8081/"`                                                                                     |
-| <span id="url">`response_url`</span> | The URL to return to the client for fetching tickets.                                                                                                         | HTTP URL                 | `"https://127.0.0.1:8081/"`                                                                                     |
-| `forward_headers`                    | When constructing the URL tickets, copy HTTP headers received in the initial query.                                                                           | Boolean                  | `true`                                                                                                          |
-| `header_blacklist`                   | List of headers that should not be forwarded.                                                                                                                 | Array of headers         | `[]`                                                                                                            |
-| `tls`                                | Additionally enables client authentication, or sets non-native root certificates for TLS. See [server configuration](#server-configuration) for more details. | TOML table               | TLS is always allowed, however the default performs no client authentication and uses native root certificates. |
+| Option                               | Description                                                                                                                                                                                    | Type                     | Default                                                                                                         |
+|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------|
+| <span id="url">`url`</span>          | The URL to fetch data from.                                                                                                                                                                    | HTTP URL                 | `"https://127.0.0.1:8081/"`                                                                                     |
+| <span id="url">`response_url`</span> | The URL to return to the client for fetching tickets.                                                                                                                                          | HTTP URL                 | `"https://127.0.0.1:8081/"`                                                                                     |
+| `forward_headers`                    | When constructing the URL tickets, copy HTTP headers received in the initial query.                                                                                                            | Boolean                  | `true`                                                                                                          |
+| `header_blacklist`                   | List of headers that should not be forwarded.                                                                                                                                                  | Array of headers         | `[]`                                                                                                            |
+| `http`                               | Additionally enables client authentication, or sets non-native root certificates for TLS, or disables HTTP header caching. See [server configuration](#server-configuration) for more details. | TOML table               | TLS is always allowed, however the default performs no client authentication and uses native root certificates. |
 
 For example, the following forwards all headers to response tickets except `Host`, and constructs tickets using `https://example.com` instead of `http://localhost:8080`:
 
@@ -275,6 +275,12 @@ backend.forward_headers = true
 backend.header_blacklist = ["Host"]
 ```
 
+> [!NOTE]  
+> Calls to the url endpoint use HTTP caching from headers like cache-control and expires. Requests to the url endpoint
+> only involve the beginning of a file to obtain the header. These requests are cached in a temporary file called
+> `htsget_rs_client_cache` in the system temp directory. To disable this functionality set
+> `http.use_cache = false`.
+
 Regex-based locations also support multiple locations:
 
 ```toml
@@ -340,7 +346,9 @@ substitution_string = "$0"
 
 backend.kind = "Url"
 backend.url = "https://example.com"
-backend.tls.root_store = "root.crt"
+backend.http.root_store = "root.crt"
+## Disable HTTP caching.
+#backend.http.use_cache = false
 ```
 
 This project uses [rustls] for all TLS logic, and it does not depend on OpenSSL. The rustls library can be more
@@ -500,6 +508,11 @@ JWT is invalid or the client is not allowed to view the requested `<id>` path. P
 after authorizing the request, so invalid requests may return an unauthorized or forbidden response instead of a bad
 request if the user lacks authorization.
 
+> [!NOTE]  
+> Calls to both the authorization service and jwks endpoint correctly handle and support HTTP cache headers like
+> cache-control and expires. Requests are cached in a temporary file called `htsget_rs_client_cache` in the system
+> temp directory. To disable this functionality set `http.use_cache = false`.
+
 [auth-json]: docs/schemas/auth.schema.json
 [auth-example]: docs/examples/auth.toml
 

htsget-config/README.md

@@ -23,12 +23,14 @@ validate_issuer = ["https://www.example.com"]
 validate_subject = "htsget"
 
 ## Set client authentication
-#tls.key = "key.pem"
-#tls.cert = "cert.pem"
+#http.key = "key.pem"
+#http.cert = "cert.pem"
 #
 ## Set root certificates
-#tls.root_store = "root.crt"
+#http.root_store = "root.crt"
+## Disable HTTP caching
+#http.use_cache = false
 
-## It's also possible to set auth individually for the ticket and auth servers:
+## It's also possible to set auth individually for the ticket and data servers:
 #[ticket_server.auth]
 #[data_server.auth]

htsget-config/docs/examples/auth.toml

@@ -24,8 +24,11 @@ locations = "https://127.0.0.1:8081"
 #backend.forward_headers = true
 #
 ## Set client authentication
-#backend.tls.key = "key.pem"
-#backend.tls.cert = "cert.pem"
+#backend.http.key = "key.pem"
+#backend.http.cert = "cert.pem"
 #
 ## Set root certificates
-#backend.tls.root_store = "root.crt"
+#backend.http.root_store = "root.crt"
+#
+## Disable HTTP caching
+#backend.http.use_cache = false

htsget-config/docs/examples/url_storage.toml

@@ -8,7 +8,9 @@ use crate::config::advanced::{Bytes, HttpClient};
 use crate::config::{deserialize_vec_from_str, serialize_array_display};
 use crate::error::Error::{BuilderError, ParseError};
 use crate::error::{Error, Result};
+use crate::http::client::HttpClientConfig;
 use http::Uri;
+use reqwest_middleware::ClientWithMiddleware;
 pub use response::{AuthorizationRestrictions, AuthorizationRule, ReferenceNameRestriction};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
@@ -127,7 +129,7 @@ impl AuthConfig {
   }
 
   /// Get the http client.
-  pub fn http_client(&self) -> &reqwest::Client {
+  pub fn http_client(&self) -> &ClientWithMiddleware {
     &self.http_client.0
   }
 
@@ -152,7 +154,7 @@ pub struct AuthConfigBuilder {
   )]
   trusted_authorization_urls: Vec<Uri>,
   authorization_path: Option<String>,
-  #[serde(rename = "tls", skip_serializing)]
+  #[serde(rename = "http", alias = "tls", skip_serializing)]
   http_client: Option<HttpClient>,
   authentication_only: bool,
   #[cfg(feature = "experimental")]
@@ -259,7 +261,9 @@ impl AuthConfigBuilder {
       validate_subject: self.validate_subject,
       trusted_authorization_urls: self.trusted_authorization_urls,
       authorization_path: self.authorization_path,
-      http_client: HttpClient::default(),
+      http_client: self
+        .http_client
+        .unwrap_or(HttpClient::try_from(HttpClientConfig::default())?),
       authentication_only: self.authentication_only,
       #[cfg(feature = "experimental")]
       suppress_errors: self.suppress_errors,
@@ -302,7 +306,7 @@ impl TryFrom<AuthConfigBuilder> for AuthConfig {
 #[cfg(test)]
 mod tests {
   use super::*;
-  use crate::tls::tests::with_test_certificates;
+  use crate::http::tests::with_test_certificates;
 
   #[test]
   fn auth_config() {

htsget-config/src/config/advanced/auth/mod.rs

@@ -3,9 +3,12 @@
 
 use crate::error::Error::ParseError;
 use crate::error::{Error, Result};
-use crate::tls::client::TlsClientConfig;
+use crate::http::client::HttpClientConfig;
+use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache, HttpCacheOptions};
 use reqwest::Client;
+use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
 use serde::{Deserialize, Serialize};
+use std::env::temp_dir;
 use std::fs::File;
 use std::io::Read;
 use std::path::PathBuf;
@@ -29,29 +32,29 @@ pub enum FormattingStyle {
 }
 
 /// A wrapper around a reqwest client to support creating from config fields.
-#[derive(Debug, Clone, Deserialize, Default)]
-#[serde(deny_unknown_fields, try_from = "TlsClientConfig")]
-pub struct HttpClient(Client);
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields, try_from = "HttpClientConfig")]
+pub struct HttpClient(ClientWithMiddleware);
 
 impl HttpClient {
   /// Create a new client.
-  pub fn new(client: Client) -> Self {
+  pub fn new(client: ClientWithMiddleware) -> Self {
     Self(client)
   }
 
   /// Get the inner client value.
-  pub fn into_inner(self) -> Client {
+  pub fn into_inner(self) -> ClientWithMiddleware {
     self.0
   }
 }
 
-impl TryFrom<TlsClientConfig> for HttpClient {
+impl TryFrom<HttpClientConfig> for HttpClient {
   type Error = Error;
 
-  fn try_from(config: TlsClientConfig) -> Result<Self> {
+  fn try_from(config: HttpClientConfig) -> Result<Self> {
     let mut builder = Client::builder();
 
-    let (certs, identity) = config.into_inner();
+    let (certs, identity, use_cache) = config.into_inner();
 
     if let Some(certs) = certs {
       for cert in certs {
@@ -62,9 +65,24 @@ impl TryFrom<TlsClientConfig> for HttpClient {
       builder = builder.identity(identity);
     }
 
-    Ok(Self(builder.build().map_err(|err| {
-      ParseError(format!("building http client: {err}"))
-    })?))
+    let client = builder
+      .build()
+      .map_err(|err| ParseError(format!("building http client: {err}")))?;
+
+    let client = if use_cache {
+      let client_cache = temp_dir().join("htsget_rs_client_cache");
+      ClientBuilder::new(client)
+        .with(Cache(HttpCache {
+          mode: CacheMode::Default,
+          manager: CACacheManager::new(client_cache, false),
+          options: HttpCacheOptions::default(),
+        }))
+        .build()
+    } else {
+      ClientBuilder::new(client).build()
+    };
+
+    Ok(Self::new(client))
   }
 }