removes robot tag on ever request adds setting for custom response headers

Author: ChristianPavilonis

crates/common/src/settings.rs

@@ -5,6 +5,7 @@ use error_stack::{Report, ResultExt};
 use regex::Regex;
 use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize};
 use serde_json::Value as JsonValue;
+use std::collections::HashMap;
 use std::sync::OnceLock;
 use url::Url;
 use validator::{Validate, ValidationError};
@@ -134,6 +135,8 @@ pub struct Settings {
     #[serde(default, deserialize_with = "vec_from_seq_or_map")]
     #[validate(nested)]
     pub handlers: Vec<Handler>,
+    #[serde(default)]
+    pub response_headers: HashMap<String, String>,
 }
 
 #[allow(unused)]

crates/fastly/src/main.rs

@@ -84,8 +84,9 @@ async fn route_request(settings: Settings, req: Request) -> Result<Response, Err
     // Convert any errors to HTTP error responses
     let mut response = result.unwrap_or_else(to_error_response);
 
-    // Add X-Robots-Tag header to prevent crawlers and indexers
-    response.set_header("X-Robots-Tag", "noindex, nofollow");
+    for (key, value) in &settings.response_headers {
+        response.set_header(key, value);
+    }
 
     Ok(response)
 }