From 2c84e735429e239aa1ac0050a5a6e3e7f213666b Mon Sep 17 00:00:00 2001 From: mistaste Date: Sun, 14 Jun 2026 23:28:23 +0300 Subject: [PATCH] cksum: reject oversized SHAKE --length instead of aborting A huge `--length` with `--algorithm shake128`/`shake256` (e.g. `--length 10011111117721172727`) made cksum abort while trying to allocate the output buffer: memory allocation of 1251388889715146591 bytes failed SHAKE is an extendable-output function with no inherent maximum, but the digest still has to be materialized and hex-encoded in memory. Cap the requested length at MAX_SHAKE_OUTPUT_BITS and return a clean error for larger requests instead of letting the allocation abort the process. Fixes #12869 --- src/uu/cksum/src/cksum.rs | 12 ++++---- src/uucore/src/lib/features/checksum/mod.rs | 15 ++++++++++ tests/by-util/test_cksum.rs | 32 +++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index 9af2f6f00ff..548e9466341 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -12,7 +12,7 @@ use uu_checksum_common::{ChecksumCommand, checksum_main, default_checksum_app, o use uucore::checksum::compute::OutputFormat; use uucore::checksum::{ - AlgoKind, BlakeLength, ChecksumError, HashLength, parse_blake_length, + AlgoKind, BlakeLength, ChecksumError, HashLength, MAX_SHAKE_OUTPUT_BITS, parse_blake_length, sanitize_sha2_sha3_length_str, }; use uucore::error::UResult; @@ -58,12 +58,14 @@ fn maybe_sanitize_length( sanitize_sha2_sha3_length_str(algo, s_len).map(Some) } - // SHAKE128 and SHAKE256 algorithms optionally take a bit length. No - // validation is performed on this length, any value is valid. If the - // given length is not a multiple of 8, the last byte of the output - // will have its extra bits set to zero. + // SHAKE128 and SHAKE256 algorithms optionally take a bit length. Any + // value up to `MAX_SHAKE_OUTPUT_BITS` is valid; larger requests are + // rejected instead of aborting the process when the (huge) output + // buffer fails to allocate. If the given length is not a multiple of 8, + // the last byte of the output will have its extra bits set to zero. (Some(AlgoKind::Shake128 | AlgoKind::Shake256), Some(len)) => match len.parse::() { Ok(0) => Ok(None), + Ok(l) if l > MAX_SHAKE_OUTPUT_BITS => Err(ChecksumError::ShakeLengthTooBig.into()), Ok(l) => Ok(Some(HashLength::from_bits(l))), Err(_) => Err(ChecksumError::InvalidLength(len.into()).into()), }, diff --git a/src/uucore/src/lib/features/checksum/mod.rs b/src/uucore/src/lib/features/checksum/mod.rs index 45d4c8b5918..9d5591ae304 100644 --- a/src/uucore/src/lib/features/checksum/mod.rs +++ b/src/uucore/src/lib/features/checksum/mod.rs @@ -253,6 +253,19 @@ impl TryFrom for ShaLength { } } +/// Upper bound on the SHAKE (`shake128`/`shake256`) digest length, in bits. +/// +/// SHAKE is an extendable-output function and has no inherent maximum, but +/// `cksum` still has to materialize and hex-encode the whole digest in memory. +/// Without a bound, a huge `--length` (e.g. `--length 10011111117721172727`) +/// makes the output-buffer allocation abort the process instead of producing a +/// clean error. +/// +/// The value (2^31 bits = 256 MiB of digest) is intentionally generous, while +/// staying strictly below `usize::MAX` on 32-bit targets so the bound check is +/// always meaningful there rather than being optimized away. +pub const MAX_SHAKE_OUTPUT_BITS: usize = 1 << 31; + /// Stores a hash length in bits. #[derive(Debug, Clone, Copy)] pub struct HashLength { @@ -455,6 +468,8 @@ pub enum ChecksumError { InvalidLength(String), #[error("maximum digest length for {} is 512 bits", .0.quote())] LengthTooBigForBlake(String), + #[error("maximum digest length for SHAKE is {MAX_SHAKE_OUTPUT_BITS} bits")] + ShakeLengthTooBig, #[error("length is not a multiple of 8")] LengthNotMultipleOf8, #[error("digest length for {} must be 224, 256, 384, or 512", .0.quote())] diff --git a/tests/by-util/test_cksum.rs b/tests/by-util/test_cksum.rs index 8f81552b3bb..71305d6b814 100644 --- a/tests/by-util/test_cksum.rs +++ b/tests/by-util/test_cksum.rs @@ -3192,6 +3192,38 @@ fn test_shake128(#[case] args: &[&str], #[case] expected: &str) { .stdout_only(format!("SHAKE128-{bit_len} (-) = {expected}\n")); } +#[test] +fn test_shake_length_too_large() { + // An oversized --length must be rejected with a clean error instead of + // aborting the process while trying to allocate the output buffer. + // See GH issue #12869. + for algo in ["shake128", "shake256"] { + // Above the cap but within the 32-bit `usize` range, so the error is + // the same on 32- and 64-bit targets. + new_ucmd!() + .arg("-a") + .arg(algo) + .arg("--length") + .arg("3000000000") + .pipe_in("xxx") + .fails_with_code(1) + .no_stdout() + .stderr_contains("maximum digest length for SHAKE is"); + + // The originally reported value (larger than u64) must also be handled + // gracefully. The exact error differs by pointer width (out-of-range vs. + // above the cap), so only assert a clean failure rather than an abort. + new_ucmd!() + .arg("-a") + .arg(algo) + .arg("--length") + .arg("10011111117721172727") + .pipe_in("xxx") + .fails_with_code(1) + .no_stdout(); + } +} + #[rstest] #[case::default_length( &[],