fix(sort): Enable locale-aware collation for UTF-8 locales (#9176)

* fix(sort): Enable locale-aware collation for UTF-8
  locales

  Fixes #9148

  The sort implementation had locale support
  infrastructure (ICU collator)
  but it was never being used due to the
  fast_lexicographic optimization
  bypassing all locale-aware code.

Author: quantum-encoding

src/uu/sort/Cargo.toml

@@ -19,6 +19,9 @@ workspace = true
 [lib]
 path = "src/sort.rs"
 
+[features]
+i18n-collator = ["uucore/i18n-collator"]
+
 [dependencies]
 bigdecimal = { workspace = true }
 binary-heap-plus = { workspace = true }
@@ -39,6 +42,7 @@ uucore = { workspace = true, features = [
   "parser-size",
   "version-cmp",
   "i18n-decimal",
+  "i18n-collator",
 ] }
 fluent = { workspace = true }
 

src/uu/sort/src/sort.rs

@@ -23,6 +23,7 @@ use chunks::LineData;
 use clap::builder::ValueParser;
 use clap::{Arg, ArgAction, ArgMatches, Command};
 use custom_str_cmp::custom_str_cmp;
+
 use ext_sort::ext_sort;
 use fnv::FnvHasher;
 use numeric_str_cmp::{NumInfo, NumInfoParseSettings, human_numeric_str_cmp, numeric_str_cmp};
@@ -47,6 +48,8 @@ use uucore::error::{FromIo, strip_errno};
 use uucore::error::{UError, UResult, USimpleError, UUsageError};
 use uucore::extendedbigdecimal::ExtendedBigDecimal;
 use uucore::format_usage;
+#[cfg(feature = "i18n-collator")]
+use uucore::i18n::collator::locale_cmp;
 use uucore::i18n::decimal::locale_decimal_separator;
 use uucore::line_ending::LineEnding;
 use uucore::parser::num_parser::{ExtendedParser, ExtendedParserError};
@@ -318,7 +321,10 @@ impl GlobalSettings {
     /// Precompute some data needed for sorting.
     /// This function **must** be called before starting to sort, and `GlobalSettings` may not be altered
     /// afterwards.
-    fn init_precomputed(&mut self) {
+    ///
+    /// When i18n-collator is enabled, `disable_fast_lexicographic` should be set to true if we're
+    /// in a UTF-8 locale (to force locale-aware collation instead of byte comparison).
+    fn init_precomputed(&mut self, disable_fast_lexicographic: bool) {
         self.precomputed.needs_tokens = self.selectors.iter().any(|s| s.needs_tokens);
         self.precomputed.selections_per_line =
             self.selectors.iter().filter(|s| s.needs_selection).count();
@@ -333,11 +339,15 @@ impl GlobalSettings {
             .filter(|s| matches!(s.settings.mode, SortMode::GeneralNumeric))
             .count();
 
-        self.precomputed.fast_lexicographic = self.can_use_fast_lexicographic();
+        self.precomputed.fast_lexicographic =
+            !disable_fast_lexicographic && self.can_use_fast_lexicographic();
         self.precomputed.fast_ascii_insensitive = self.can_use_fast_ascii_insensitive();
     }
 
     /// Returns true when the fast lexicographic path can be used safely.
+    /// Note: When i18n-collator is enabled, the caller must have already determined
+    /// whether locale-aware collation is needed (via checking if we're in a UTF-8 locale).
+    /// This check is performed in uumain() before init_precomputed() is called.
     fn can_use_fast_lexicographic(&self) -> bool {
         self.mode == SortMode::Default
             && !self.ignore_case
@@ -2065,7 +2075,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
         emit_debug_warnings(&settings, &global_flags, &legacy_warnings);
     }
 
-    settings.init_precomputed();
+    // Initialize locale collation if needed (UTF-8 locales)
+    // This MUST happen before init_precomputed() to avoid the performance regression
+    #[cfg(feature = "i18n-collator")]
+    let needs_locale_collation = uucore::i18n::collator::init_locale_collation();
+
+    #[cfg(not(feature = "i18n-collator"))]
+    let needs_locale_collation = false;
+
+    settings.init_precomputed(needs_locale_collation);
 
     let result = exec(&mut files, &settings, output, &mut tmp_dir);
     // Wait here if `SIGINT` was received,
@@ -2446,13 +2464,36 @@ fn compare_by<'a>(
             }
             SortMode::Month => month_compare(a_str, b_str),
             SortMode::Version => version_cmp(a_str, b_str),
-            SortMode::Default => custom_str_cmp(
-                a_str,
-                b_str,
-                settings.ignore_non_printing,
-                settings.dictionary_order,
-                settings.ignore_case,
-            ),
+            SortMode::Default => {
+                // Use locale-aware comparison if feature is enabled and no custom flags are set
+                #[cfg(feature = "i18n-collator")]
+                {
+                    if settings.ignore_case
+                        || settings.dictionary_order
+                        || settings.ignore_non_printing
+                    {
+                        custom_str_cmp(
+                            a_str,
+                            b_str,
+                            settings.ignore_non_printing,
+                            settings.dictionary_order,
+                            settings.ignore_case,
+                        )
+                    } else {
+                        locale_cmp(a_str, b_str)
+                    }
+                }
+                #[cfg(not(feature = "i18n-collator"))]
+                {
+                    custom_str_cmp(
+                        a_str,
+                        b_str,
+                        settings.ignore_non_printing,
+                        settings.dictionary_order,
+                        settings.ignore_case,
+                    )
+                }
+            }
         };
         if cmp != Ordering::Equal {
             return if settings.reverse { cmp.reverse() } else { cmp };

src/uucore/src/lib/features/i18n/collator.rs

@@ -30,6 +30,45 @@ pub fn init_collator(opts: CollatorOptions) {
         .expect("Collator already initialized");
 }
 
+/// Initialize the collator for locale-aware string comparison if needed.
+///
+/// This function checks if the current locale requires locale-aware collation
+/// (UTF-8 encoding) and initializes the ICU collator with appropriate settings
+/// if necessary. For C/POSIX locales, no initialization is needed as byte
+/// comparison is sufficient.
+///
+/// # Returns
+///
+/// `true` if the collator was initialized for a UTF-8 locale, `false` if
+/// using C/POSIX locale (no initialization needed).
+///
+/// # Example
+///
+/// ```
+/// use uucore::i18n::collator::init_locale_collation;
+///
+/// if init_locale_collation() {
+///     // Using locale-aware collation
+/// } else {
+///     // Using byte comparison (C/POSIX locale)
+/// }
+/// ```
+pub fn init_locale_collation() -> bool {
+    use crate::i18n::{UEncoding, get_locale_encoding};
+
+    // Check if we need locale-aware collation
+    if get_locale_encoding() != UEncoding::Utf8 {
+        // C/POSIX locale - no collator needed
+        return false;
+    }
+
+    // UTF-8 locale - initialize collator with Shifted mode to match GNU behavior
+    let mut opts = CollatorOptions::default();
+    opts.alternate_handling = Some(AlternateHandling::Shifted);
+
+    try_init_collator(opts)
+}
+
 /// Compare both strings with regard to the current locale.
 pub fn locale_cmp(left: &[u8], right: &[u8]) -> Ordering {
     // If the detected locale is 'C', just do byte-wise comparison

src/uucore/src/lib/features/i18n/mod.rs

@@ -20,7 +20,9 @@ pub enum UEncoding {
     Utf8,
 }
 
-const DEFAULT_LOCALE: Locale = locale!("en-US-posix");
+// Use "und" (undefined) as the marker for C/POSIX locale
+// This ensures real locales like "en-US" won't match
+const DEFAULT_LOCALE: Locale = locale!("und");
 
 /// Look at 3 environment variables in the following order
 ///
@@ -38,6 +40,11 @@ fn get_locale_from_env(locale_name: &str) -> (Locale, UEncoding) {
         let mut split = locale_var_str.split(&['.', '@']);
 
         if let Some(simple) = split.next() {
+            // Handle explicit C and POSIX locales - these should always use byte comparison
+            if simple == "C" || simple == "POSIX" {
+                return (DEFAULT_LOCALE, UEncoding::Ascii);
+            }
+
             // Naively convert the locale name to BCP47 tag format.
             //
             // See https://en.wikipedia.org/wiki/IETF_language_tag

tests/by-util/test_sort.rs

@@ -2463,4 +2463,58 @@ fn test_start_buffer() {
         .stdout_only_bytes(&expected);
 }
 
+#[test]
+fn test_locale_collation_c_locale() {
+    // C locale uses byte order - this is deterministic and tests the fix for #9148
+    // Accented characters (UTF-8 multibyte) sort after ASCII letters
+    let input = "é\ne\nE\na\nA\nz\n";
+    // C locale byte order: A=0x41, E=0x45, a=0x61, e=0x65, z=0x7A, é=0xC3 0xA9
+    let expected = "A\nE\na\ne\nz\né\n";
+
+    new_ucmd!()
+        .env("LC_ALL", "C")
+        .pipe_in(input)
+        .succeeds()
+        .stdout_is(expected);
+}
+
+#[test]
+fn test_locale_collation_utf8() {
+    // Test French UTF-8 locale handling - behavior depends on i18n-collator feature
+    // With feature: locale-aware collation (é sorts near e)
+    // Without feature: byte order (é after z, since 0xC3A9 > 0x7A)
+    let input = "z\né\ne\na\n";
+
+    let result = new_ucmd!()
+        .env("LC_ALL", "fr_FR.UTF-8")
+        .pipe_in(input)
+        .succeeds();
+
+    let output = result.stdout_str();
+    let lines: Vec<&str> = output.lines().collect();
+
+    assert_eq!(lines.len(), 4, "Expected 4 sorted lines");
+    assert_eq!(lines[0], "a", "'a' (0x61) should always sort first");
+
+    // Validate based on which collation mode is active
+    if lines[3] == "é" {
+        // Byte order mode: é (0xC3A9) > z (0x7A)
+        assert_eq!(
+            lines,
+            vec!["a", "e", "z", "é"],
+            "Byte order mode: expected a < e < z < é"
+        );
+    } else {
+        // Locale collation mode: é sorts with base letter e
+        assert_eq!(lines[3], "z", "Locale mode: 'z' should sort last");
+        let z_pos = lines.iter().position(|&x| x == "z").unwrap();
+        let e_pos = lines.iter().position(|&x| x == "e").unwrap();
+        let e_accent_pos = lines.iter().position(|&x| x == "é").unwrap();
+        assert!(
+            e_pos < z_pos && e_accent_pos < z_pos,
+            "Locale mode: 'e' ({e_pos}) and 'é' ({e_accent_pos}) should sort before 'z' ({z_pos})"
+        );
+    }
+}
+
 /* spell-checker: enable */