cksum: reject oversized SHAKE --length instead of aborting

src/uu/cksum/src/cksum.rs

@@ -12,7 +12,7 @@ use uu_checksum_common::{ChecksumCommand, checksum_main, default_checksum_app, o
 
 use uucore::checksum::compute::OutputFormat;
 use uucore::checksum::{
-    AlgoKind, BlakeLength, ChecksumError, HashLength, parse_blake_length,
+    AlgoKind, BlakeLength, ChecksumError, HashLength, MAX_SHAKE_OUTPUT_BITS, parse_blake_length,
     sanitize_sha2_sha3_length_str,
 };
 use uucore::error::UResult;
@@ -58,12 +58,14 @@ fn maybe_sanitize_length(
             sanitize_sha2_sha3_length_str(algo, s_len).map(Some)
         }
 
-        // SHAKE128 and SHAKE256 algorithms optionally take a bit length. No
-        // validation is performed on this length, any value is valid. If the
-        // given length is not a multiple of 8, the last byte of the output
-        // will have its extra bits set to zero.
+        // SHAKE128 and SHAKE256 algorithms optionally take a bit length. Any
+        // value up to `MAX_SHAKE_OUTPUT_BITS` is valid; larger requests are
+        // rejected instead of aborting the process when the (huge) output
+        // buffer fails to allocate. If the given length is not a multiple of 8,
+        // the last byte of the output will have its extra bits set to zero.
         (Some(AlgoKind::Shake128 | AlgoKind::Shake256), Some(len)) => match len.parse::<usize>() {
             Ok(0) => Ok(None),
+            Ok(l) if l > MAX_SHAKE_OUTPUT_BITS => Err(ChecksumError::ShakeLengthTooBig.into()),
             Ok(l) => Ok(Some(HashLength::from_bits(l))),
             Err(_) => Err(ChecksumError::InvalidLength(len.into()).into()),
         },

src/uucore/src/lib/features/checksum/mod.rs

@@ -253,6 +253,19 @@ impl TryFrom<usize> for ShaLength {
     }
 }
 
+/// Upper bound on the SHAKE (`shake128`/`shake256`) digest length, in bits.
+///
+/// SHAKE is an extendable-output function and has no inherent maximum, but
+/// `cksum` still has to materialize and hex-encode the whole digest in memory.
+/// Without a bound, a huge `--length` (e.g. `--length 10011111117721172727`)
+/// makes the output-buffer allocation abort the process instead of producing a
+/// clean error.
+///
+/// The value (2^31 bits = 256 MiB of digest) is intentionally generous, while
+/// staying strictly below `usize::MAX` on 32-bit targets so the bound check is
+/// always meaningful there rather than being optimized away.
+pub const MAX_SHAKE_OUTPUT_BITS: usize = 1 << 31;
+
 /// Stores a hash length in bits.
 #[derive(Debug, Clone, Copy)]
 pub struct HashLength {
@@ -455,6 +468,8 @@ pub enum ChecksumError {
     InvalidLength(String),
     #[error("maximum digest length for {} is 512 bits", .0.quote())]
     LengthTooBigForBlake(String),
+    #[error("maximum digest length for SHAKE is {MAX_SHAKE_OUTPUT_BITS} bits")]
+    ShakeLengthTooBig,
     #[error("length is not a multiple of 8")]
     LengthNotMultipleOf8,
     #[error("digest length for {} must be 224, 256, 384, or 512", .0.quote())]

tests/by-util/test_cksum.rs

@@ -3192,6 +3192,38 @@ fn test_shake128(#[case] args: &[&str], #[case] expected: &str) {
         .stdout_only(format!("SHAKE128-{bit_len} (-) = {expected}\n"));
 }
 
+#[test]
+fn test_shake_length_too_large() {
+    // An oversized --length must be rejected with a clean error instead of
+    // aborting the process while trying to allocate the output buffer.
+    // See GH issue #12869.
+    for algo in ["shake128", "shake256"] {
+        // Above the cap but within the 32-bit `usize` range, so the error is
+        // the same on 32- and 64-bit targets.
+        new_ucmd!()
+            .arg("-a")
+            .arg(algo)
+            .arg("--length")
+            .arg("3000000000")
+            .pipe_in("xxx")
+            .fails_with_code(1)
+            .no_stdout()
+            .stderr_contains("maximum digest length for SHAKE is");
+
+        // The originally reported value (larger than u64) must also be handled
+        // gracefully. The exact error differs by pointer width (out-of-range vs.
+        // above the cap), so only assert a clean failure rather than an abort.
+        new_ucmd!()
+            .arg("-a")
+            .arg(algo)
+            .arg("--length")
+            .arg("10011111117721172727")
+            .pipe_in("xxx")
+            .fails_with_code(1)
+            .no_stdout();
+    }
+}
+
 #[rstest]
 #[case::default_length(
     &[],