🥅 fix: improve ZeroKMS error handling and startup validation

- Add dedicated ZeroKMSError type with clearer authentication failure messages
- Validate ZeroKMS connection during proxy startup to fail fast on credential issues
- Improve error documentation with step-by-step troubleshooting guidance
- Enhanced logging with connection status and initialization feedback
- Better error categorization for credential vs system issues

This ensures users get immediate, actionable feedback when ZeroKMS authentication
fails rather than encountering cryptic errors during operation.

Author: tobyhede

docs/errors.md

@@ -22,6 +22,7 @@
   - [KeysetName could not be set](#encrypt-keyset-name-could-not-be-set)
   - [Plaintext could not be encoded](#encrypt-plaintext-could-not-be-encoded)
   - [Unknown column](#encrypt-unknown-column)
+  - [Unknown Keyset Identifier](#encrypt-unknown-keyset)
   - [Unknown table](#encrypt-unknown-table)
   - [Unknown index term](#encrypt-unknown-index-term)
   - [Column configuration mismatch](#encrypt-column-config-mismatch)
@@ -46,14 +47,14 @@ Authentication failed when connecting to the database.
 ### Error message
 
 ```
-Database authentication failed: check username and password
+Database authentication failed. Check username and password
 ```
 
 ### How to fix
 
-Check the configured username and password are correct and can connect to the database.
+1. Check the configured username and password are correct and can connect to the database.
 
-Check the database is using a supported authentication method.
+2. Check the database is using a supported authentication method.
 
 CipherStash Proxy supports several PostgreSQL password authentication methods:
 
@@ -76,21 +77,38 @@ Authentication failed when connecting a client to the proxy.
 ### Error message
 
 ```
-Client authentication failed: check username and password
+Client authentication failed. Check username and password.
 ```
 
 ### How to fix
 
-Check the configured username and password are correct.
+1. Check the configured username and password are correct.
 
 
 
 <!-- ---------------------------------------------------------------------------------------------------- -->
 
 
+## ZeroKMS  <a id='authentication-failed-zerokms'></a>
+
+Authentication failed when connecting to ZeroKMS.
+
+
+### Error message
+
+```
+ZeroKMS authentication failed. Check the configured credentials.
+```
+
+### How to Fix
+
+1. Check that the configured `client` credentials are correct.
+2. Check that the active `keyset_name` or `keyset_id` is associated with a keyset in the configured workspace.
+
 
 <!-- ---------------------------------------------------------------------------------------------------- -->
 
+
 # Mapping errors
 
 
@@ -354,7 +372,7 @@ Keyset Name could not be set using `SET CIPHERSTASH.KEYSET_NAME`
 <!-- ---------------------------------------------------------------------------------------------------- -->
 
 
-## Unknown Keyset Identifier <a id='encrypt-unknown-keyset'></a>
+## Unknown keyset identifier <a id='encrypt-unknown-keyset'></a>
 
 The specified keyset could not be loaded.
 
@@ -370,11 +388,13 @@ Unknown keyset name or id '{keyset}'
 1. Check that the active `keyset_name` or `keyset_id` is associated with a keyset in the configured workspace.
 2. Check that the configured `client` has been granted access to the keyset via the dashboard.
 3. Keyset names are case sensitive. If setting the active keyset by name, check that the `keyset_name` is an exact match.
+4. Check that the configured `client` credentials are correct.
 
 
 <!-- ---------------------------------------------------------------------------------------------------- -->
 
 
+
 ## Could not decrypt data for keyset <a id='encrypt-could-not-decrypt-data-for-keyset'></a>
 
 The data belonging to the active `keyset_id` could not be decrypted.

packages/cipherstash-proxy-integration/src/multitenant/set_keyset_id.rs

@@ -283,7 +283,7 @@ mod tests {
         if let Err(err) = result {
             let msg = err.to_string();
 
-            assert_eq!(msg, "db error: FATAL: Unknown keyset name or id '2cace9db-3a2a-4b46-a184-ba412b3e0730'. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-unknown-keyset");
+            assert_eq!(msg, "db error: FATAL: Unknown keyset name or id '2cace9db-3a2a-4b46-a184-ba412b3e0730'. Check the configured credentials. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-unknown-keyset");
         } else {
             unreachable!();
         }

packages/cipherstash-proxy-integration/src/multitenant/set_keyset_name.rs

@@ -341,7 +341,7 @@ mod tests {
         if let Err(err) = result {
             let msg = err.to_string();
 
-            assert_eq!(msg, "db error: FATAL: Unknown keyset name or id 'BLAHVTHA'. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-unknown-keyset");
+            assert_eq!(msg, "db error: FATAL: Unknown keyset name or id 'BLAHVTHA'. Check the configured credentials. For help visit https://github.com/cipherstash/proxy/blob/main/docs/errors.md#encrypt-unknown-keyset");
         } else {
             unreachable!();
         }

packages/cipherstash-proxy/src/error.rs

@@ -52,7 +52,7 @@ pub enum Error {
     Tls(#[from] rustls::Error),
 
     #[error(transparent)]
-    ZeroKMS(#[from] cipherstash_client::zerokms::Error),
+    ZeroKMS(#[from] ZeroKMSError),
 
     #[error("Unknown error")]
     Unknown,
@@ -67,6 +67,15 @@ pub enum ContextError {
     UnknownPortal,
 }
 
+#[derive(Error, Debug)]
+pub enum ZeroKMSError {
+    #[error("ZeroKMS authentication failed. Check the configured credentials. For help visit {}#zerokms-authentication-failed", ERROR_DOC_BASE_URL)]
+    AuthenticationFailed,
+
+    #[error(transparent)]
+    System(#[from] cipherstash_client::zerokms::Error),
+}
+
 #[derive(Error, Debug)]
 pub enum MappingError {
     #[error("Invalid parameter for column '{}' of type '{}' in table '{}' (OID {}). For help visit {}#mapping-invalid-parameter",
@@ -283,7 +292,7 @@ pub enum EncryptError {
     UnknownColumn { table: String, column: String },
 
     #[error(
-        "Unknown keyset name or id '{keyset}'. For help visit {}#encrypt-unknown-keyset",
+        "Unknown keyset name or id '{keyset}'. Check the configured credentials. For help visit {}#encrypt-unknown-keyset",
         ERROR_DOC_BASE_URL
     )]
     UnknownKeysetIdentifier { keyset: String },
@@ -300,10 +309,10 @@ pub enum EncryptError {
 
 #[derive(Error, Debug)]
 pub enum ProtocolError {
-    #[error("Database authentication failed: check username and password. For help visit {}#authentication-failed-database", ERROR_DOC_BASE_URL)]
+    #[error("Database authentication failed. Check username and password. For help visit {}#authentication-failed-database", ERROR_DOC_BASE_URL)]
     AuthenticationFailed,
 
-    #[error("Client authentication failed: check username and password. For help visit {}#authentication-failed-client", ERROR_DOC_BASE_URL)]
+    #[error("Client authentication failed. Check username and password. For help visit {}#authentication-failed-client", ERROR_DOC_BASE_URL)]
     ClientAuthenticationFailed,
 
     #[error("Expected {expected} parameter format codes, received {received}")]

packages/cipherstash-proxy/src/proxy/mod.rs

@@ -37,7 +37,11 @@ pub struct Proxy {
 
 impl Proxy {
     pub async fn init(config: TandemConfig) -> Result<Proxy, Error> {
-        let zerokms = ZeroKms::new(&config)?;
+        let zerokms = ZeroKms::init(&config)?;
+
+        // Attempt to connect to default keyset
+        // Ensures error on start if credential or network issue
+        zerokms.init_cipher(None).await?;
 
         let encrypt_config = EncryptConfigManager::init(&config.database).await?;
         // TODO: populate EqlTraitImpls based in config

packages/cipherstash-proxy/src/proxy/zerokms/zerokms.rs

@@ -1,8 +1,8 @@
 use crate::{
     config::TandemConfig,
     eql,
-    error::{EncryptError, Error},
-    log::ENCRYPT,
+    error::{EncryptError, Error, ZeroKMSError},
+    log::{ENCRYPT, PROXY},
     postgresql::{Column, KeysetIdentifier},
     prometheus::{KEYSET_CIPHER_CACHE_HITS_TOTAL, KEYSET_CIPHER_INIT_TOTAL},
 };
@@ -13,7 +13,7 @@ use cipherstash_client::{
 use metrics::counter;
 use moka::future::Cache;
 use std::{sync::Arc, time::Duration};
-use tracing::debug;
+use tracing::{debug, info, warn};
 
 use super::{
     init_zerokms_client, plaintext_type_name, to_eql_encrypted, to_eql_encrypted_from_index_term,
@@ -30,7 +30,7 @@ pub struct ZeroKms {
 }
 
 impl ZeroKms {
-    pub fn new(config: &TandemConfig) -> Result<Self, Error> {
+    pub fn init(config: &TandemConfig) -> Result<Self, Error> {
         let zerokms_client = init_zerokms_client(config)?;
 
         let cipher_cache = Cache::builder()
@@ -66,14 +66,14 @@ impl ZeroKms {
 
         // Check cache first
         if let Some(cached_cipher) = self.cipher_cache.get(&cache_key).await {
-            debug!(target: "proxy", msg = "Use cached ScopedCipher", ?keyset_id);
+            debug!(target: PROXY, msg = "Use cached ScopedCipher", ?keyset_id);
             counter!(KEYSET_CIPHER_CACHE_HITS_TOTAL).increment(1);
             return Ok(cached_cipher);
         }
 
         let zerokms_client = self.zerokms_client.clone();
 
-        debug!(target: "proxy", msg = "Initializing ZeroKMS ScopedCipher", ?keyset_id);
+        debug!(target: PROXY, msg = "Initializing ZeroKMS ScopedCipher", ?keyset_id);
 
         let identified_by = keyset_id.as_ref().map(|id| id.0.clone());
 
@@ -94,12 +94,14 @@ impl ZeroKms {
                 let entry_count = self.cipher_cache.entry_count();
                 let memory_usage_bytes = self.cipher_cache.weighted_size();
 
-                debug!(target: "proxy", msg = "ScopedCipher cached", ?keyset_id, entry_count, memory_usage_bytes);
+                info!(msg = "Connected to ZeroKMS");
+                debug!(target: PROXY, msg = "ScopedCipher cached", ?keyset_id, entry_count, memory_usage_bytes);
 
                 Ok(arc_cipher)
             }
             Err(err) => {
-                debug!(target: "proxy", msg = "Error initializing ZeroKMS ScopedCipher", error = err.to_string());
+                debug!(target: PROXY, msg = "Error initializing ZeroKMS ScopedCipher", error = err.to_string());
+                warn!(msg = "Error initializing ZeroKMS", error = err.to_string());
 
                 match err {
                     cipherstash_client::zerokms::Error::LoadKeyset(_) => {
@@ -108,7 +110,10 @@ impl ZeroKms {
                         }
                         .into())
                     }
-                    _ => Err(err.into()),
+                    cipherstash_client::zerokms::Error::Credentials(_) => {
+                        Err(ZeroKMSError::AuthenticationFailed.into())
+                    }
+                    _ => Err(Error::ZeroKMS(err.into())),
                 }
             }
         }
@@ -219,10 +224,10 @@ impl ZeroKms {
             .filter_map(|(idx, eql)| Some((idx, eql?.body.ciphertext.unwrap())))
             .collect::<_>();
 
-        let decrypted = cipher.decrypt(encrypted).await.map_err(|e| -> Error {
-            match &e {
+        let decrypted = cipher.decrypt(encrypted).await.map_err(|err| -> Error {
+            match &err {
                 cipherstash_client::zerokms::Error::Decrypt(_) => {
-                    let error_msg = e.to_string();
+                    let error_msg = err.to_string();
                     if error_msg.contains("Failed to retrieve key") {
                         EncryptError::CouldNotDecryptDataForKeyset {
                             keyset_id: keyset_id
@@ -231,10 +236,10 @@ impl ZeroKms {
                         }
                         .into()
                     } else {
-                        e.into()
+                        Error::ZeroKMS(err.into())
                     }
                 }
-                _ => e.into(),
+                _ => Error::ZeroKMS(err.into()),
             }
         })?;