locale_encoding.rs

// This file is part of the uutils awk package.
//
// For the full copyright and license information, please view the LICENSE
// files that was distributed with this source code.

//! Locale-aware encoding for `\u` escape sequences, matching gawk behavior.
//!
//! See: <https://www.gnu.org/software/gawk/manual/html_node/Escape-Sequences.html>

use encoding_rs::{EncoderResult, Encoding, UTF_8};

/// Character encoding derived from the process locale (`LC_*` / `LANG`).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct LocaleEncoding {
    encoding: &'static Encoding,
    /// `C` / `POSIX` locales only accept ASCII via `\u`.
    ascii_only: bool,
}

impl LocaleEncoding {
    pub fn utf8() -> Self {
        Self { encoding: UTF_8, ascii_only: false }
    }

    /// `C` / `POSIX` locale: `\u` only encodes code points in ASCII.
    pub fn ascii() -> Self {
        Self { encoding: UTF_8, ascii_only: true }
    }

    /// ISO-8859-1 (Latin-1).
    pub fn iso_8859_1() -> Self {
        Self {
            encoding: Encoding::for_label(b"iso-8859-1").unwrap_or(UTF_8),
            ascii_only: false,
        }
    }

    /// Detect encoding from `LC_ALL`, `LC_CTYPE`, or `LANG`.
    pub fn detect() -> Self {
        let name = std::env::var("LC_ALL")
            .or_else(|_| std::env::var("LC_CTYPE"))
            .or_else(|_| std::env::var("LANG"))
            .unwrap_or_else(|_| "C.UTF-8".to_string());
        from_locale_name(&name)
    }

    /// Encode a Unicode scalar value for a `\u` escape in the current locale.
    ///
    /// Invalid code points and characters that cannot be represented in the
    /// locale encoding become `?`, matching gawk.
    pub fn encode_unicode_escape(self, codepoint: u32) -> Vec<u8> {
        if codepoint > 0x0010_FFFF || (0xD800..=0xDFFF).contains(&codepoint) {
            return vec![b'?'];
        }

        let c = char::from_u32(codepoint).unwrap();

        if self.ascii_only && codepoint > 0x7F {
            return vec![b'?'];
        }

        if self.encoding == UTF_8 && !self.ascii_only {
            let mut buf = [0u8; 4];
            return c.encode_utf8(&mut buf).as_bytes().to_vec();
        }

        let mut encoder = self.encoding.new_encoder();
        let mut buf = [0u8; 8];
        let ch = c.to_string();
        match encoder.encode_from_utf8_without_replacement(&ch, &mut buf, true) {
            (EncoderResult::InputEmpty, _, written) if written > 0 => buf[..written].to_vec(),
            _ => vec![b'?'],
        }
    }
}

impl Default for LocaleEncoding {
    fn default() -> Self {
        Self::utf8()
    }
}

fn from_locale_name(name: &str) -> LocaleEncoding {
    let lower = name.to_ascii_lowercase();
    let extension = lower.rsplit_once('.').map(|(_, ext)| ext);
    if lower == "c" || lower == "posix" || extension == Some("c") || extension == Some("posix") {
        return LocaleEncoding::ascii();
    }

    let charset = name.rsplit('.').next().unwrap_or(name);
    let label = charset.to_ascii_lowercase().replace('_', "-");

    if label.contains("utf-8") || label == "utf8" {
        return LocaleEncoding::utf8();
    }

    if let Some(encoding) = Encoding::for_label(label.as_bytes()) {
        LocaleEncoding { encoding, ascii_only: false }
    } else {
        LocaleEncoding::utf8()
    }
}