feat: Implement static killswitches (#239)

This implements a first version of killswitches. Switches are configured
through objectstore config, so in order to create a killswitch one has
to apply new config and restart the service.

Note that all scope values must be strings. Examples for configuration
look like this:

```yaml
killswitches:
  # Drop all requests for a specific usecase, regardless of scopes
  - usecase: broken_usecase

  # Drop requests from a specific org, all projects, all usecases
  - scopes:
      org: "42"

  # Drop requests from a specific project, all usecases
  - scopes:
      org: "42" # can also be omitted
      project: "4711"

  # Combination of usecase and org
  - usecase: attachments
    scopes:
      org: "42"
```

At a later stage, we can replace the static configuration with a runtime
system.

Author: jan-auer

objectstore-server/src/config.rs

@@ -45,6 +45,8 @@ use secrecy::{CloneableSecret, SecretBox, SerializableSecret, zeroize::Zeroize};
 use serde::{Deserialize, Serialize};
 use tracing::level_filters::LevelFilter;
 
+use crate::killswitches::Killswitches;
+
 /// Environment variable prefix for all configuration options.
 const ENV_PREFIX: &str = "OS__";
 
@@ -92,7 +94,7 @@ impl Zeroize for ConfigSecret {
 /// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
 ///
 /// Used in: [`Config::high_volume_storage`], [`Config::long_term_storage`]
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 #[serde(tag = "type", rename_all = "lowercase")]
 pub enum Storage {
     /// Local filesystem storage backend (type `"filesystem"`).
@@ -306,7 +308,7 @@ pub enum Storage {
 /// Controls the threading behavior of the server's async runtime.
 ///
 /// Used in: [`Config::runtime`]
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 #[serde(default)]
 pub struct Runtime {
     /// Number of worker threads for the server runtime.
@@ -355,7 +357,7 @@ impl Default for Runtime {
 /// tracing. Sentry is disabled by default and only enabled when a DSN is provided.
 ///
 /// Used in: [`Config::sentry`]
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct Sentry {
     /// Sentry DSN (Data Source Name).
     ///
@@ -607,7 +609,7 @@ mod display_fromstr {
 /// Controls the verbosity and format of log output. Logs are always written to stderr.
 ///
 /// Used in: [`Config::logging`]
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct Logging {
     /// Minimum log level to output.
     ///
@@ -664,7 +666,7 @@ impl Default for Logging {
 /// Metrics configuration.
 ///
 /// Configures submission of internal metrics to Datadog.
-#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize)]
 pub struct Metrics {
     /// Datadog [API key] for metrics.
     ///
@@ -710,7 +712,7 @@ pub struct Metrics {
 
 /// A key that may be used to verify a request's `Authorization` header and its
 /// associated permissions. May contain multiple key versions to facilitate rotation.
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct AuthZVerificationKey {
     /// Versions of this key's key material which may be used to verify signatures.
     ///
@@ -729,7 +731,7 @@ pub struct AuthZVerificationKey {
 }
 
 /// Configuration for content-based authorization.
-#[derive(Debug, Default, Clone, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize)]
 pub struct AuthZ {
     /// Whether to enforce content-based authorization or not.
     ///
@@ -758,7 +760,7 @@ pub struct AuthZ {
 ///
 /// See individual field documentation for details on each configuration option, including
 /// defaults and environment variables.
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct Config {
     /// HTTP server bind address.
     ///
@@ -864,6 +866,9 @@ pub struct Config {
     /// Controls the verification and enforcement of content-based access control based on the
     /// JWT in a request's `Authorization` header.
     pub auth: AuthZ,
+
+    /// A list of matchers for requests to discard without processing.
+    pub killswitches: Killswitches,
 }
 
 impl Default for Config {
@@ -883,6 +888,7 @@ impl Default for Config {
             sentry: Sentry::default(),
             metrics: Metrics::default(),
             auth: AuthZ::default(),
+            killswitches: Killswitches::default(),
         }
     }
 }
@@ -920,6 +926,8 @@ mod tests {
 
     use secrecy::ExposeSecret;
 
+    use crate::killswitches::Killswitch;
+
     use super::*;
 
     #[test]
@@ -1071,29 +1079,30 @@ mod tests {
             Ok(())
         });
     }
+
     #[test]
     fn configure_auth_with_yaml() {
         let mut tempfile = tempfile::NamedTempFile::new().unwrap();
         tempfile
             .write_all(
                 br#"
-            auth:
-                enforce: true
-                keys:
-                    kid1:
-                        key_versions:
-                            - "abcde"
-                            - "fghij"
-                            - |
-                              this is a test
-                                multiline string
-                              end of string
-                        max_permissions:
-                            - "object.read"
-                            - "object.write"
-                    kid2:
-                        key_versions:
-                            - "12345"
+                auth:
+                    enforce: true
+                    keys:
+                        kid1:
+                            key_versions:
+                                - "abcde"
+                                - "fghij"
+                                - |
+                                  this is a test
+                                    multiline string
+                                  end of string
+                            max_permissions:
+                                - "object.read"
+                                - "object.write"
+                        kid2:
+                            key_versions:
+                                - "12345"
             "#,
             )
             .unwrap();
@@ -1122,4 +1131,54 @@ mod tests {
             Ok(())
         });
     }
+
+    #[test]
+    fn configure_killswitches_with_yaml() {
+        let mut tempfile = tempfile::NamedTempFile::new().unwrap();
+        tempfile
+            .write_all(
+                br#"
+                killswitches:
+                  - usecase: broken_usecase
+                  - scopes:
+                      org: "42"
+                  - scopes:
+                      org: "42"
+                      project: "4711"
+                  - usecase: attachments
+                    scopes:
+                      org: "42"
+                "#,
+            )
+            .unwrap();
+
+        figment::Jail::expect_with(|_jail| {
+            let expected = [
+                Killswitch {
+                    usecase: Some("broken_usecase".into()),
+                    scopes: BTreeMap::new(),
+                },
+                Killswitch {
+                    usecase: None,
+                    scopes: BTreeMap::from([("org".into(), "42".into())]),
+                },
+                Killswitch {
+                    usecase: None,
+                    scopes: BTreeMap::from([
+                        ("org".into(), "42".into()),
+                        ("project".into(), "4711".into()),
+                    ]),
+                },
+                Killswitch {
+                    usecase: Some("attachments".into()),
+                    scopes: BTreeMap::from([("org".into(), "42".into())]),
+                },
+            ];
+
+            let config = Config::load(Some(tempfile.path())).unwrap();
+            assert_eq!(&config.killswitches.0, &expected,);
+
+            Ok(())
+        });
+    }
 }

objectstore-server/src/extractors/id.rs

@@ -3,15 +3,41 @@ use std::borrow::Cow;
 use axum::extract::rejection::PathRejection;
 use axum::extract::{FromRequestParts, Path};
 use axum::http::request::Parts;
+use axum::response::{IntoResponse, Response};
 use objectstore_service::id::{ObjectContext, ObjectId};
 use objectstore_types::scope::{EMPTY_SCOPES, Scope, Scopes};
 use serde::{Deserialize, de};
 
 use crate::extractors::Xt;
 use crate::state::ServiceState;
 
+#[derive(Debug)]
+pub enum ObjectRejection {
+    Path(PathRejection),
+    Killswitched,
+}
+
+impl IntoResponse for ObjectRejection {
+    fn into_response(self) -> Response {
+        match self {
+            ObjectRejection::Path(rejection) => rejection.into_response(),
+            ObjectRejection::Killswitched => (
+                axum::http::StatusCode::FORBIDDEN,
+                "Object access is disabled for this scope through killswitches",
+            )
+                .into_response(),
+        }
+    }
+}
+
+impl From<PathRejection> for ObjectRejection {
+    fn from(rejection: PathRejection) -> Self {
+        ObjectRejection::Path(rejection)
+    }
+}
+
 impl FromRequestParts<ServiceState> for Xt<ObjectId> {
-    type Rejection = PathRejection;
+    type Rejection = ObjectRejection;
 
     async fn from_request_parts(
         parts: &mut Parts,
@@ -23,6 +49,10 @@ impl FromRequestParts<ServiceState> for Xt<ObjectId> {
         populate_sentry_context(id.context());
         sentry::configure_scope(|s| s.set_extra("key", id.key().into()));
 
+        if state.config.killswitches.matches(id.context()) {
+            return Err(ObjectRejection::Killswitched);
+        }
+
         Ok(Xt(id))
     }
 }
@@ -66,7 +96,7 @@ where
 }
 
 impl FromRequestParts<ServiceState> for Xt<ObjectContext> {
-    type Rejection = PathRejection;
+    type Rejection = ObjectRejection;
 
     async fn from_request_parts(
         parts: &mut Parts,
@@ -80,6 +110,10 @@ impl FromRequestParts<ServiceState> for Xt<ObjectContext> {
 
         populate_sentry_context(&context);
 
+        if state.config.killswitches.matches(&context) {
+            return Err(ObjectRejection::Killswitched);
+        }
+
         Ok(Xt(context))
     }
 }