Merge pull request #8386 from RenjiSann/improve-sort

sort: Handle non-utf8 sorting content

Author: cakebaker

src/uu/ls/src/ls.rs

@@ -2139,8 +2139,11 @@ fn sort_entries(entries: &mut [PathData], config: &Config, out: &mut BufWriter<S
         // The default sort in GNU ls is case insensitive
         Sort::Name => entries.sort_by(|a, b| a.display_name.cmp(&b.display_name)),
         Sort::Version => entries.sort_by(|a, b| {
-            version_cmp(&a.p_buf.to_string_lossy(), &b.p_buf.to_string_lossy())
-                .then(a.p_buf.to_string_lossy().cmp(&b.p_buf.to_string_lossy()))
+            version_cmp(
+                os_str_as_bytes_lossy(a.p_buf.as_os_str()).as_ref(),
+                os_str_as_bytes_lossy(b.p_buf.as_os_str()).as_ref(),
+            )
+            .then(a.p_buf.to_string_lossy().cmp(&b.p_buf.to_string_lossy()))
         }),
         Sort::Extension => entries.sort_by(|a, b| {
             a.p_buf

src/uu/sort/src/check.rs

@@ -72,7 +72,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> {
                 return Err(SortError::Disorder {
                     file: path.to_owned(),
                     line_number: line_idx,
-                    line: new_first.line.to_owned(),
+                    line: String::from_utf8_lossy(new_first.line).into_owned(),
                     silent: settings.check_silent,
                 }
                 .into());
@@ -86,7 +86,7 @@ pub fn check(path: &OsStr, settings: &GlobalSettings) -> UResult<()> {
                 return Err(SortError::Disorder {
                     file: path.to_owned(),
                     line_number: line_idx,
-                    line: b.line.to_owned(),
+                    line: String::from_utf8_lossy(b.line).into_owned(),
                     silent: settings.check_silent,
                 }
                 .into());

src/uu/sort/src/chunks.rs

@@ -17,9 +17,7 @@ use memchr::memchr_iter;
 use self_cell::self_cell;
 use uucore::error::{UResult, USimpleError};
 
-use crate::{
-    GeneralBigDecimalParseResult, GlobalSettings, Line, SortError, numeric_str_cmp::NumInfo,
-};
+use crate::{GeneralBigDecimalParseResult, GlobalSettings, Line, numeric_str_cmp::NumInfo};
 
 self_cell!(
     /// The chunk that is passed around between threads.
@@ -41,7 +39,7 @@ pub struct ChunkContents<'a> {
 
 #[derive(Debug)]
 pub struct LineData<'a> {
-    pub selections: Vec<&'a str>,
+    pub selections: Vec<&'a [u8]>,
     pub num_infos: Vec<NumInfo>,
     pub parsed_floats: Vec<GeneralBigDecimalParseResult>,
     pub line_num_floats: Vec<Option<f64>>,
@@ -68,7 +66,7 @@ impl Chunk {
             let selections = unsafe {
                 // SAFETY: (same as above) It is safe to (temporarily) transmute to a vector of &str with a longer lifetime,
                 // because the vector is empty.
-                std::mem::transmute::<Vec<&'_ str>, Vec<&'static str>>(std::mem::take(
+                std::mem::transmute::<Vec<&'_ [u8]>, Vec<&'static [u8]>>(std::mem::take(
                     &mut contents.line_data.selections,
                 ))
             };
@@ -100,7 +98,7 @@ impl Chunk {
 
 pub struct RecycledChunk {
     lines: Vec<Line<'static>>,
-    selections: Vec<&'static str>,
+    selections: Vec<&'static [u8]>,
     num_infos: Vec<NumInfo>,
     parsed_floats: Vec<GeneralBigDecimalParseResult>,
     line_num_floats: Vec<Option<f64>>,
@@ -180,15 +178,14 @@ pub fn read<T: Read>(
             let selections = unsafe {
                 // SAFETY: It is safe to transmute to an empty vector of selections with shorter lifetime.
                 // It was only temporarily transmuted to a Vec<Line<'static>> to make recycling possible.
-                std::mem::transmute::<Vec<&'static str>, Vec<&'_ str>>(selections)
+                std::mem::transmute::<Vec<&'static [u8]>, Vec<&'_ [u8]>>(selections)
             };
             let mut lines = unsafe {
                 // SAFETY: (same as above) It is safe to transmute to a vector of lines with shorter lifetime,
                 // because it was only temporarily transmuted to a Vec<Line<'static>> to make recycling possible.
                 std::mem::transmute::<Vec<Line<'static>>, Vec<Line<'_>>>(lines)
             };
-            let read = std::str::from_utf8(&buffer[..read])
-                .map_err(|error| SortError::Uft8Error { error })?;
+            let read = &buffer[..read];
             let mut line_data = LineData {
                 selections,
                 num_infos,
@@ -205,13 +202,13 @@ pub fn read<T: Read>(
 
 /// Split `read` into `Line`s, and add them to `lines`.
 fn parse_lines<'a>(
-    read: &'a str,
+    read: &'a [u8],
     lines: &mut Vec<Line<'a>>,
     line_data: &mut LineData<'a>,
     separator: u8,
     settings: &GlobalSettings,
 ) {
-    let read = read.strip_suffix(separator as char).unwrap_or(read);
+    let read = read.strip_suffix(&[separator]).unwrap_or(read);
 
     assert!(lines.is_empty());
     assert!(line_data.selections.is_empty());
@@ -220,7 +217,7 @@ fn parse_lines<'a>(
     assert!(line_data.line_num_floats.is_empty());
     let mut token_buffer = vec![];
     lines.extend(
-        read.split(separator as char)
+        read.split(|&c| c == separator)
             .enumerate()
             .map(|(index, line)| Line::create(line, index, line_data, &mut token_buffer, settings)),
     );

src/uu/sort/src/custom_str_cmp.rs

@@ -9,7 +9,7 @@
 
 use std::cmp::Ordering;
 
-fn filter_char(c: char, ignore_non_printing: bool, ignore_non_dictionary: bool) -> bool {
+fn filter_char(c: u8, ignore_non_printing: bool, ignore_non_dictionary: bool) -> bool {
     if ignore_non_dictionary && !(c.is_ascii_alphanumeric() || c.is_ascii_whitespace()) {
         return false;
     }
@@ -19,7 +19,7 @@ fn filter_char(c: char, ignore_non_printing: bool, ignore_non_dictionary: bool)
     true
 }
 
-fn cmp_chars(a: char, b: char, ignore_case: bool) -> Ordering {
+fn cmp_chars(a: u8, b: u8, ignore_case: bool) -> Ordering {
     if ignore_case {
         a.to_ascii_uppercase().cmp(&b.to_ascii_uppercase())
     } else {
@@ -28,8 +28,8 @@ fn cmp_chars(a: char, b: char, ignore_case: bool) -> Ordering {
 }
 
 pub fn custom_str_cmp(
-    a: &str,
-    b: &str,
+    a: &[u8],
+    b: &[u8],
     ignore_non_printing: bool,
     ignore_non_dictionary: bool,
     ignore_case: bool,
@@ -39,11 +39,11 @@ pub fn custom_str_cmp(
         return a.cmp(b);
     }
     let mut a_chars = a
-        .chars()
-        .filter(|&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
+        .iter()
+        .filter(|&&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
     let mut b_chars = b
-        .chars()
-        .filter(|&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
+        .iter()
+        .filter(|&&c| filter_char(c, ignore_non_printing, ignore_non_dictionary));
     loop {
         let a_char = a_chars.next();
         let b_char = b_chars.next();
@@ -52,7 +52,7 @@ pub fn custom_str_cmp(
             (Some(_), None) => return Ordering::Greater,
             (None, Some(_)) => return Ordering::Less,
             (Some(a_char), Some(b_char)) => {
-                let ordering = cmp_chars(a_char, b_char, ignore_case);
+                let ordering = cmp_chars(*a_char, *b_char, ignore_case);
                 if ordering != Ordering::Equal {
                     return ordering;
                 }

src/uu/sort/src/ext_sort.rs

@@ -272,7 +272,7 @@ fn write<I: WriteableTmpFile>(
 
 fn write_lines<T: Write>(lines: &[Line], writer: &mut T, separator: u8) {
     for s in lines {
-        writer.write_all(s.line.as_bytes()).unwrap();
+        writer.write_all(s.line).unwrap();
         writer.write_all(&[separator]).unwrap();
     }
 }