Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
cdee06e
Add support for custom CRC parameters
onethumb Jun 10, 2025
99985e5
Add shared library support for custom CRC params
onethumb Jun 10, 2025
2eb380f
Fix formatting
onethumb Jun 10, 2025
54779da
Remove dead code
onethumb Jun 10, 2025
97d8d51
Add more custom parameter support
onethumb Jun 11, 2025
90538b7
Add a command-line tool to calculate CRC params
onethumb Jun 11, 2025
17f4057
Make CrcParams public
onethumb Jun 11, 2025
8c97d7e
Update custom CRC params support in FFI
onethumb Jun 11, 2025
caa1837
Add custom params support to the software fallback
onethumb Jun 11, 2025
d771b8e
Fix formatting
onethumb Jun 11, 2025
9272a48
Merge branch 'main' into support-custom-crc-parameters
onethumb Jun 11, 2025
786b4df
Enable custom parameters via CrcParams constructor
onethumb Jun 12, 2025
9a9f4ca
Generate specs w/Kiro for caching generated keys
onethumb Jul 10, 2025
0eb7765
Add docs for performance considerations
onethumb Jul 10, 2025
65708b4
Add a steering document about commenting
onethumb Jul 10, 2025
3dd32a3
Create cache module with core data structures
onethumb Jul 10, 2025
781d754
Implement thread-safe cache storage
onethumb Jul 10, 2025
243336e
Implement cache lookup and storage functions
onethumb Jul 10, 2025
d51fb6a
Add cache management utilities
onethumb Jul 10, 2025
9d3a433
Integrate cache into CrcParams::new()
onethumb Jul 10, 2025
e8697e6
Create comprehensive unit tests for cache functionality
onethumb Jul 10, 2025
5be2e91
Add thread safety tests
onethumb Jul 10, 2025
7c40333
Create integration tests for CrcParams compatibility
onethumb Jul 10, 2025
97504d6
Add comprehensive error handling tests
onethumb Jul 11, 2025
75464e4
Update existing tests to work with caching
onethumb Jul 11, 2025
322d954
Add documentation and finalize implementation
onethumb Jul 11, 2025
bf7d24a
Add and apply code quality requirements
onethumb Jul 11, 2025
dfc6de6
Create spec for future-proofing CrcParams
onethumb Jul 11, 2025
ccfe362
Add CrcKeysStorage enum and helper methods
onethumb Jul 11, 2025
d726a6a
Update architecture code to use safe accessors
onethumb Jul 11, 2025
35d62fe
Switch CrcParams to use CrcKeysStorage
onethumb Jul 11, 2025
7df28e6
Create comprehensive test suite for future-proof functionality
onethumb Jul 11, 2025
ced964b
Validate migration and run full test suite
onethumb Jul 11, 2025
c19ab97
Implement FFI future-proofing for C/C++ compatibility
onethumb Jul 11, 2025
dad60a2
Remove unused bindgen, update packages
onethumb Jul 16, 2025
01df5f5
Apply latest cargo clippy recommendations
onethumb Jul 16, 2025
026f179
Update dependencies
onethumb Jul 16, 2025
de88e02
Improve function names
onethumb Jul 16, 2025
50c500e
Improve docs
onethumb Jul 16, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions libcrc_fast.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ typedef enum CrcFastAlgorithm {
Crc32Bzip2,
Crc32CdRomEdc,
Crc32Cksum,
Crc32Custom,
Crc32Iscsi,
Crc32IsoHdlc,
Crc32Jamcrc,
Crc32Mef,
Crc32Mpeg2,
Crc32Xfer,
Crc64Custom,
Crc64Ecma182,
Crc64GoIso,
Crc64Ms,
Expand All @@ -50,6 +52,21 @@ typedef struct CrcFastDigestHandle {
struct CrcFastDigest *_0;
} CrcFastDigestHandle;

/**
* Custom CRC parameters
*/
typedef struct CrcFastParams {
enum CrcFastAlgorithm algorithm;
uint8_t width;
uint64_t poly;
uint64_t init;
bool refin;
bool refout;
uint64_t xorout;
uint64_t check;
uint64_t keys[23];
} CrcFastParams;

#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
Expand All @@ -59,6 +76,11 @@ extern "C" {
*/
struct CrcFastDigestHandle *crc_fast_digest_new(enum CrcFastAlgorithm algorithm);

/**
* Creates a new Digest to compute CRC checksums using custom parameters
*/
struct CrcFastDigestHandle *crc_fast_digest_new_with_params(struct CrcFastParams params);

/**
* Updates the Digest with data
*/
Expand Down Expand Up @@ -100,13 +122,27 @@ uint64_t crc_fast_digest_get_amount(struct CrcFastDigestHandle *handle);
*/
uint64_t crc_fast_checksum(enum CrcFastAlgorithm algorithm, const char *data, uintptr_t len);

/**
* Helper method to calculate a CRC checksum directly for data using custom parameters
*/
uint64_t crc_fast_checksum_with_params(struct CrcFastParams params,
const char *data,
uintptr_t len);

/**
* Helper method to just calculate a CRC checksum directly for a file using algorithm
*/
uint64_t crc_fast_checksum_file(enum CrcFastAlgorithm algorithm,
const uint8_t *path_ptr,
uintptr_t path_len);

/**
* Helper method to calculate a CRC checksum directly for a file using custom parameters
*/
uint64_t crc_fast_checksum_file_with_params(struct CrcFastParams params,
const uint8_t *path_ptr,
uintptr_t path_len);

/**
* Combine two CRC checksums using algorithm
*/
Expand All @@ -115,6 +151,25 @@ uint64_t crc_fast_checksum_combine(enum CrcFastAlgorithm algorithm,
uint64_t checksum2,
uint64_t checksum2_len);

/**
* Combine two CRC checksums using custom parameters
*/
uint64_t crc_fast_checksum_combine_with_custom_params(struct CrcFastParams params,
uint64_t checksum1,
uint64_t checksum2,
uint64_t checksum2_len);

/**
* Returns the custom CRC parameters for a given set of Rocksoft CRC parameters
*/
struct CrcFastParams crc_fast_get_custom_params(const char *name_ptr,
uint8_t width,
uint64_t poly,
uint64_t init,
bool reflected,
uint64_t xorout,
uint64_t check);

/**
* Gets the target build properties (CPU architecture and fine-tuning parameters) for this algorithm
*/
Expand Down
4 changes: 2 additions & 2 deletions src/algorithm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

use crate::consts::CRC_CHUNK_SIZE;
use crate::enums::{DataChunkProcessor, Reflector};
use crate::structs::{CrcParams, CrcState};
use crate::structs::CrcState;
use crate::traits::{ArchOps, EnhancedCrcWidth};
use crate::{crc32, crc64};
use crate::{crc32, crc64, CrcParams};

/// Main entry point that works for both CRC-32 and CRC-64
#[inline]
Expand Down
2 changes: 1 addition & 1 deletion src/arch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use std::arch::is_aarch64_feature_detected;
#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
use crate::algorithm;

use crate::structs::CrcParams;
use crate::CrcParams;

#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
use crate::structs::{Width32, Width64};
Expand Down
38 changes: 36 additions & 2 deletions src/arch/software.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
#![cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]

use crate::consts::CRC_64_NVME;
use crate::structs::CrcParams;
use crate::CrcAlgorithm;
use crc::Table;
use crate::CrcParams;
use crc::{Algorithm, Table};

const RUST_CRC32_AIXM: crc::Crc<u32, Table<16>> =
crc::Crc::<u32, Table<16>>::new(&crc::CRC_32_AIXM);
Expand Down Expand Up @@ -78,6 +78,23 @@ pub(crate) fn update(state: u64, data: &[u8], params: CrcParams) -> u64 {
CrcAlgorithm::Crc32Mef => RUST_CRC32_MEF,
CrcAlgorithm::Crc32Mpeg2 => RUST_CRC32_MPEG_2,
CrcAlgorithm::Crc32Xfer => RUST_CRC32_XFER,
CrcAlgorithm::Crc32Custom => {
let algorithm: Algorithm<u32> = Algorithm {
width: params.width as u8,
poly: params.poly as u32,
init: params.init as u32,
refin: params.refin,
refout: params.refout,
xorout: params.xorout as u32,
check: params.check as u32,
residue: 0x00000000, // unused in this context
};

// ugly, but the crc crate is difficult to work with...
let static_algorithm = Box::leak(Box::new(algorithm));

crc::Crc::<u32, Table<16>>::new(static_algorithm)
}
_ => panic!("Invalid algorithm for u32 CRC"),
};
update_u32(state as u32, data, params) as u64
Expand All @@ -91,6 +108,23 @@ pub(crate) fn update(state: u64, data: &[u8], params: CrcParams) -> u64 {
CrcAlgorithm::Crc64Redis => RUST_CRC64_REDIS,
CrcAlgorithm::Crc64We => RUST_CRC64_WE,
CrcAlgorithm::Crc64Xz => RUST_CRC64_XZ,
CrcAlgorithm::Crc64Custom => {
let algorithm: Algorithm<u64> = Algorithm {
width: params.width as u8,
poly: params.poly as u64,
init: params.init as u64,
refin: params.refin,
refout: params.refout,
xorout: params.xorout as u64,
check: params.check as u64,
residue: 0x0000000000000000, // unused in this context
};

// ugly, but the crc crate is difficult to work with...
let static_algorithm = Box::leak(Box::new(algorithm));

crc::Crc::<u64, Table<16>>::new(static_algorithm)
}
_ => panic!("Invalid algorithm for u64 CRC"),
};
update_u64(state, data, params)
Expand Down
185 changes: 185 additions & 0 deletions src/bin/get-custom-params.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
//! This is a simple program to get custom CRC parameters from the command line.

use std::env;
use std::process::ExitCode;

#[derive(Debug)]
struct Config {
width: Option<u32>,
polynomial: Option<u64>,
init: Option<u64>,
reflected: Option<bool>,
xorout: Option<u64>,
check: Option<u64>,
name: Option<String>,
}

impl Config {
fn new() -> Self {
Config {
width: None,
polynomial: None,
init: None,
reflected: None,
xorout: None,
check: None,
name: None,
}
}

fn is_complete(&self) -> bool {
self.width.is_some()
&& self.polynomial.is_some()
&& self.init.is_some()
&& self.reflected.is_some()
&& self.xorout.is_some()
&& self.check.is_some()
&& self.name.is_some()
}
}

fn parse_hex_or_decimal(s: &str) -> Result<u64, String> {
if s.starts_with("0x") || s.starts_with("0X") {
u64::from_str_radix(&s[2..], 16).map_err(|_| format!("Invalid hexadecimal value: {}", s))
} else {
s.parse::<u64>()
.map_err(|_| format!("Invalid decimal value: {}", s))
}
}

fn parse_bool(s: &str) -> Result<bool, String> {
match s.to_lowercase().as_str() {
"true" | "1" | "yes" | "on" => Ok(true),
"false" | "0" | "no" | "off" => Ok(false),
_ => Err(format!("Invalid boolean value: {} (use true/false)", s)),
}
}

fn parse_args(args: &[String]) -> Result<Config, String> {
let mut config = Config::new();
let mut i = 1; // Skip program name

while i < args.len() {
match args[i].as_str() {
"-n" => {
if i + 1 >= args.len() {
return Err("Missing value for -n (name)".to_string());
}
config.name = Some(args[i + 1].clone());
i += 2;
}
"-w" => {
if i + 1 >= args.len() {
return Err("Missing value for -w (width)".to_string());
}
config.width = Some(
args[i + 1]
.parse::<u32>()
.map_err(|_| format!("Invalid width value: {}", args[i + 1]))?,
);
i += 2;
}
"-p" => {
if i + 1 >= args.len() {
return Err("Missing value for -p (polynomial)".to_string());
}
config.polynomial = Some(parse_hex_or_decimal(&args[i + 1])?);
i += 2;
}
"-i" => {
if i + 1 >= args.len() {
return Err("Missing value for -i (init)".to_string());
}
config.init = Some(parse_hex_or_decimal(&args[i + 1])?);
i += 2;
}
"-r" => {
if i + 1 >= args.len() {
return Err("Missing value for -r (reflected)".to_string());
}
config.reflected = Some(parse_bool(&args[i + 1])?);
i += 2;
}
"-x" => {
if i + 1 >= args.len() {
return Err("Missing value for -x (xorout)".to_string());
}
config.xorout = Some(parse_hex_or_decimal(&args[i + 1])?);
i += 2;
}
"-c" => {
if i + 1 >= args.len() {
return Err("Missing value for -c (check)".to_string());
}
config.check = Some(parse_hex_or_decimal(&args[i + 1])?);
i += 2;
}
arg => {
return Err(format!("Unknown argument: {}", arg));
}
}
}

Ok(config)
}

fn print_usage() {
println!("Usage: get-custom-params -n <name> -w <width> -p <polynomial> -i <init> -r <reflected> -x <xorout> -c <check>");
println!();
println!("Example: get-custom-params -n CRC-32/ISCSI -w 32 -p 0x1edc6f41 -i 0xFFFFFFFF -r true -x 0xFFFFFFFF -c 0xe3069283");
println!("Example: get-custom-params -n CRC-64/NVME -w 64 -p 0xad93d23594c93659 -i 0xffffffffffffffff -r true -x 0xffffffffffffffff -c 0xae8b14860a799888");
println!();
println!("Arguments:");
println!(" -n <name> Name of the CRC algorithm (e.g., CRC-32/ISCSI)");
println!(" -w <width> CRC width (number of bits)");
println!(" -p <polynomial> CRC polynomial (hex or decimal)");
println!(" -i <init> Initial value (hex or decimal)");
println!(" -r <reflected> Reflected input/output (true/false)");
println!(" -x <xorout> XOR output value (hex or decimal)");
println!(" -c <check> Check value (hex or decimal)");
}

fn main() -> ExitCode {
let args: Vec<String> = env::args().collect();

if args.len() == 1 {
print_usage();
return ExitCode::from(1);
}

let config = match parse_args(&args) {
Ok(config) => config,
Err(error) => {
eprintln!("Error: {}", error);
println!();
print_usage();
return ExitCode::from(1);
}
};

// Check if all required arguments are provided
if !config.is_complete() {
eprintln!("Error: All arguments are required");
println!();
print_usage();
return ExitCode::from(1);
}

let static_name: &'static str = Box::leak(config.name.unwrap().into_boxed_str());

let params = crc_fast::get_custom_params(
static_name,
config.width.unwrap() as u8,
config.polynomial.unwrap(),
config.init.unwrap(),
config.reflected.unwrap(),
config.xorout.unwrap(),
config.check.unwrap(),
);

println!();
println!("{:#x?}", params);
println!();

ExitCode::from(0)
}
2 changes: 1 addition & 1 deletion src/combine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
http://reveng.sourceforge.net/crc-catalogue/all.htm
*/

use crate::structs::CrcParams;
use crate::CrcParams;

/* Multiply the GF(2) vector vec by the GF(2) matrix mat, returning the
resulting vector. The vector is stored as bits in a crc_t. The matrix is
Expand Down
2 changes: 1 addition & 1 deletion src/crc32/consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use crate::consts::{
NAME_CRC32_CD_ROM_EDC, NAME_CRC32_CKSUM, NAME_CRC32_ISCSI, NAME_CRC32_ISO_HDLC,
NAME_CRC32_JAMCRC, NAME_CRC32_MEF, NAME_CRC32_MPEG_2, NAME_CRC32_XFER,
};
use crate::structs::CrcParams;
use crate::CrcAlgorithm;
use crate::CrcParams;
use crc::{
CRC_32_AIXM, CRC_32_AUTOSAR, CRC_32_BASE91_D, CRC_32_BZIP2, CRC_32_CD_ROM_EDC, CRC_32_CKSUM,
CRC_32_ISCSI, CRC_32_ISO_HDLC, CRC_32_JAMCRC, CRC_32_MEF, CRC_32_MPEG_2, CRC_32_XFER,
Expand Down
2 changes: 1 addition & 1 deletion src/crc64/consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
#![allow(dead_code)]

use crate::consts::*;
use crate::structs::CrcParams;
use crate::CrcAlgorithm;
use crate::CrcParams;
use crc::{CRC_64_ECMA_182, CRC_64_GO_ISO, CRC_64_MS, CRC_64_REDIS, CRC_64_WE, CRC_64_XZ};

// width=64 poly=0x42f0e1eba9ea3693 init=0x0000000000000000 refin=false refout=false xorout=0x0000000000000000 check=0x6c40df5f0b497347 residue=0x0000000000000000 name="CRC-64/ECMA-182"
Expand Down
Loading