[WIP] Integrate ejson loading prototype

Cargo.toml

@@ -1,32 +1,34 @@
 [package]
 name = "shadowenv"
 version = "3.0.3"
-authors = [
-    "Shopify Engineering <[email protected]>",
-]
+authors = ["Shopify Engineering <[email protected]>"]
 edition = "2021"
 
 [dependencies]
 blake2 = "0.10.6"
 clap = { version = "4.5.20", features = ["cargo", "derive"] }
-clap_complete = "4.5.35"
+clap_complete = "4.5.38"
 dirs = "5.0.1"
 exec = "0.3.1"
-anyhow = "1.0.89"
+anyhow = "1.0.94"
 thiserror = "1.0.64"
 hex = "0.4.3"
 ketos = "0.12"
 ketos_derive = "0.12"
-libc = "0.2.48"
-regex = "1.11.0"
-serde = "1.0.210"
-serde_derive = "1.0.210"
-serde_json = "1.0.128"
+libc = "0.2.168"
+regex = "1.11.1"
+serde = { version = "1.0.216", features = ["derive"] }
+serde_derive = "1.0.216"
+serde_json = "1.0.133"
+json_dotpath = "1.1.0"
 shell-escape = "0.1.4"
 shellexpand = "3.1.0"
 ed25519-dalek = { version = "2.1.1", features = ["rand_core"] }
 ed25519 = "2.2.3"
 rand = "0.8.5"
+nom = "7"
+crypto_box = "0.9.1"
+base64 = "0.22.1"
 
 [build-dependencies]
 clap = { version = "4.5.20", features = ["cargo", "derive"] }
@@ -35,4 +37,4 @@ clap_complete = "4.5.35"
 [dev-dependencies]
 quickcheck = "1.0.3"
 quickcheck_macros = "1.0.0"
-tempfile = "3.13.0"
+tempfile = "3.14.0"

src/ejson.rs

@@ -0,0 +1,198 @@
+//! File TODO: Too many `map_err`, can be beautified.
+//! EJSON TODO: Potential improvement: Cache decoded files for
+//!             multiple `env/ejson` on the same file.
+use base64::Engine;
+use crypto_box::{aead::Aead, Nonce, PublicKey, SalsaBox, SecretKey};
+use nom::{
+    bytes::complete::{tag, take_till},
+    character::complete::digit1,
+    combinator::{map, map_res},
+    IResult,
+};
+use serde::Deserialize;
+use serde_json::{Map, Value};
+use std::{
+    fs::{read, read_to_string},
+    io,
+    path::Path,
+    str::FromStr,
+};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum EJsonError {
+    #[error("Invalid EJSON: {}", .0)]
+    InvalidJson(#[from] serde_json::Error),
+
+    #[error(transparent)]
+    IoErrorr(#[from] io::Error),
+
+    /// Generic parsing error.
+    #[error("{}", .0)]
+    BoxParseError(String),
+}
+
+#[derive(Deserialize, Debug)]
+struct EJsonFile {
+    /// An EJSON file must have a public key associated, otherwise it's invalid.
+    #[serde(rename = "_public_key")]
+    pub public_key: String,
+
+    /// All other key-value pairs contained in the file.
+    #[serde(flatten)]
+    pub other: Map<String, Value>,
+}
+
+/// Attempts to load an ejson file from the given path. Decodes all values in the
+/// file using the public key specified in the file. Keys stay unchanged (no `_` removal).
+///
+/// Returns the entire parsed & decoded JSON file, minus the `_public_key` root field.
+pub fn load_ejson_file(path: &Path) -> Result<Map<String, Value>, EJsonError> {
+    let bytes = read(path)?;
+    let mut parsed_file: EJsonFile = serde_json::from_slice(&bytes)?;
+
+    let priv_key = find_private_key(&parsed_file.public_key)?;
+    decode_map(&mut parsed_file.other, &priv_key)?;
+
+    Ok(parsed_file.other)
+}
+
+fn decode_value(key: &str, value: &mut Value, private_key: &SecretKey) -> Result<(), EJsonError> {
+    match value {
+        Value::Object(obj) => decode_map(obj, private_key)?,
+        Value::String(s) if !key.starts_with("_") => {
+            if let Some(s) = decode_ejson_string(s, private_key)? {
+                *value = Value::String(s);
+            }
+        }
+        Value::Array(array) => {
+            for elem in array {
+                decode_value(key, elem, private_key)?;
+            }
+        }
+        _ => (),
+    };
+
+    Ok(())
+}
+
+fn decode_map(map: &mut Map<String, Value>, private_key: &SecretKey) -> Result<(), EJsonError> {
+    for (key, value) in map.iter_mut() {
+        decode_value(key, value, private_key)?;
+    }
+
+    Ok(())
+}
+
+fn decode_ejson_string(s: &str, private_key: &SecretKey) -> Result<Option<String>, EJsonError> {
+    let parsed = match parse_ejson_box(&s) {
+        Ok((_, parsed)) => parsed,
+
+        // Ignore value if we can't parse the box, for now.
+        // TODO: Do we want to assume that all non-underscore strings should be decodable? Then we can bubble the parse error.
+        Err(_) => return Ok(None),
+    };
+
+    let keybox = SalsaBox::new(&parsed.encrypter_public_key()?, &private_key);
+    let nonce = Nonce::from(parsed.nonce()?);
+    let decrypted_plaintext = keybox
+        .decrypt(&nonce, parsed.boxed_message()?.as_slice())
+        .map_err(|err| {
+            EJsonError::BoxParseError(format!("Unable to decrypt secret box `{s}`: {}", err))
+        })?;
+
+    String::from_utf8(decrypted_plaintext)
+        .map(Some)
+        .map_err(|err| {
+            EJsonError::BoxParseError(format!(
+                "Decrypted message value for secret box `{s}` contains invalid UTF-8: {err}."
+            ))
+        })
+}
+
+#[derive(Debug)]
+struct EJsonMessageBox<'input> {
+    _schema_version: u32,
+    /// Base64-encoded key used for encryption,
+    encrypter_key_b64: &'input str,
+    /// Base64-encoded nonce used for encryption,
+    nonce_b64: &'input str,
+    /// The encrypted message.
+    boxed_message_b64: &'input str,
+}
+
+impl<'input> EJsonMessageBox<'input> {
+    fn encrypter_public_key(&self) -> Result<PublicKey, EJsonError> {
+        let pk_bytes = base64::engine::general_purpose::STANDARD
+            .decode(self.encrypter_key_b64)
+            .map_err(|_err| {
+                EJsonError::BoxParseError("Encrypter public key is invalid base64".to_owned())
+            })?;
+
+        let pk_bytes: [u8; 32] = pk_bytes.try_into().map_err(|pk_bytes: Vec<u8>| {
+            EJsonError::BoxParseError(format!(
+                "Invalid nonce length: Found {}, must be 24",
+                pk_bytes.len()
+            ))
+        })?;
+
+        Ok(PublicKey::from_bytes(pk_bytes))
+    }
+
+    fn nonce(&self) -> Result<[u8; 24], EJsonError> {
+        let nonce_bytes = base64::engine::general_purpose::STANDARD
+            .decode(self.nonce_b64)
+            .map_err(|_err| EJsonError::BoxParseError("Nonce is invalid base64".to_owned()))?;
+
+        nonce_bytes.try_into().map_err(|nonce_bytes: Vec<u8>| {
+            EJsonError::BoxParseError(format!(
+                "Invalid nonce length: Found {}, must be 24",
+                nonce_bytes.len()
+            ))
+        })
+    }
+
+    fn boxed_message(&self) -> Result<Vec<u8>, EJsonError> {
+        base64::engine::general_purpose::STANDARD
+            .decode(self.boxed_message_b64)
+            .map_err(|_err| EJsonError::BoxParseError("Boxed message is invalid base64".to_owned()))
+    }
+}
+
+fn parse_ejson_box<'input>(input: &'input str) -> IResult<&str, EJsonMessageBox<'input>> {
+    let (input, _) = tag("EJ[")(input)?;
+    let (input, schema_version) =
+        map(take_till(|c| c == ':'), map_res(digit1, u32::from_str))(input)?;
+    let (_, schema_version) = schema_version?;
+
+    let (input, _) = tag(":")(input)?;
+    let (input, encrypter_key_b64) = take_till(|c| c == ':')(input)?;
+
+    let (input, _) = tag(":")(input)?;
+    let (input, nonce_b64) = take_till(|c| c == ':')(input)?;
+
+    let (input, _) = tag(":")(input)?;
+    let (_input, boxed_message_b64) = take_till(|c| c == ']')(input)?;
+
+    Ok((
+        input,
+        EJsonMessageBox {
+            _schema_version: schema_version,
+            encrypter_key_b64,
+            nonce_b64,
+            boxed_message_b64,
+        },
+    ))
+}
+
+fn find_private_key(hexed_key: &str) -> Result<SecretKey, EJsonError> {
+    let hexed_private_key_bytes = read_to_string(format!("/opt/ejson/keys/{hexed_key}"))?;
+    let decoded_bytes = hex::decode(hexed_private_key_bytes.trim_end_matches("\n"))
+        .map_err(|_err| EJsonError::BoxParseError("Key is invalid hex".to_owned()))?;
+
+    let key_bytes: [u8; 32] = decoded_bytes[..32].try_into().map_err(|_err| {
+        EJsonError::BoxParseError("Invalid key length, must be 32 bytes".to_owned())
+    })?;
+
+    Ok(SecretKey::from_bytes(key_bytes))
+}