Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
cdee06e
Add support for custom CRC parameters
onethumb Jun 10, 2025
99985e5
Add shared library support for custom CRC params
onethumb Jun 10, 2025
2eb380f
Fix formatting
onethumb Jun 10, 2025
54779da
Remove dead code
onethumb Jun 10, 2025
97d8d51
Add more custom parameter support
onethumb Jun 11, 2025
90538b7
Add a command-line tool to calculate CRC params
onethumb Jun 11, 2025
17f4057
Make CrcParams public
onethumb Jun 11, 2025
8c97d7e
Update custom CRC params support in FFI
onethumb Jun 11, 2025
caa1837
Add custom params support to the software fallback
onethumb Jun 11, 2025
d771b8e
Fix formatting
onethumb Jun 11, 2025
9272a48
Merge branch 'main' into support-custom-crc-parameters
onethumb Jun 11, 2025
786b4df
Enable custom parameters via CrcParams constructor
onethumb Jun 12, 2025
9a9f4ca
Generate specs w/Kiro for caching generated keys
onethumb Jul 10, 2025
0eb7765
Add docs for performance considerations
onethumb Jul 10, 2025
65708b4
Add a steering document about commenting
onethumb Jul 10, 2025
3dd32a3
Create cache module with core data structures
onethumb Jul 10, 2025
781d754
Implement thread-safe cache storage
onethumb Jul 10, 2025
243336e
Implement cache lookup and storage functions
onethumb Jul 10, 2025
d51fb6a
Add cache management utilities
onethumb Jul 10, 2025
9d3a433
Integrate cache into CrcParams::new()
onethumb Jul 10, 2025
e8697e6
Create comprehensive unit tests for cache functionality
onethumb Jul 10, 2025
5be2e91
Add thread safety tests
onethumb Jul 10, 2025
7c40333
Create integration tests for CrcParams compatibility
onethumb Jul 10, 2025
97504d6
Add comprehensive error handling tests
onethumb Jul 11, 2025
75464e4
Update existing tests to work with caching
onethumb Jul 11, 2025
322d954
Add documentation and finalize implementation
onethumb Jul 11, 2025
bf7d24a
Add and apply code quality requirements
onethumb Jul 11, 2025
dfc6de6
Create spec for future-proofing CrcParams
onethumb Jul 11, 2025
ccfe362
Add CrcKeysStorage enum and helper methods
onethumb Jul 11, 2025
d726a6a
Update architecture code to use safe accessors
onethumb Jul 11, 2025
35d62fe
Switch CrcParams to use CrcKeysStorage
onethumb Jul 11, 2025
7df28e6
Create comprehensive test suite for future-proof functionality
onethumb Jul 11, 2025
ced964b
Validate migration and run full test suite
onethumb Jul 11, 2025
c19ab97
Implement FFI future-proofing for C/C++ compatibility
onethumb Jul 11, 2025
dad60a2
Remove unused bindgen, update packages
onethumb Jul 16, 2025
01df5f5
Apply latest cargo clippy recommendations
onethumb Jul 16, 2025
026f179
Update dependencies
onethumb Jul 16, 2025
de88e02
Improve function names
onethumb Jul 16, 2025
50c500e
Improve docs
onethumb Jul 16, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions libcrc_fast.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,21 @@ typedef struct CrcFastDigestHandle {
struct CrcFastDigest *_0;
} CrcFastDigestHandle;

/**
* Custom CRC parameters
*/
typedef struct CrcFastParams {
enum CrcFastAlgorithm algorithm;
uint8_t width;
uint64_t poly;
uint64_t init;
bool refin;
bool refout;
uint64_t xorout;
uint64_t check;
uint64_t keys[23];
} CrcFastParams;

#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
Expand All @@ -59,6 +74,11 @@ extern "C" {
*/
struct CrcFastDigestHandle *crc_fast_digest_new(enum CrcFastAlgorithm algorithm);

/**
* Creates a new Digest to compute CRC checksums using custom parameters
*/
struct CrcFastDigestHandle *crc_fast_digest_new_with_params(struct CrcFastParams params);

/**
* Updates the Digest with data
*/
Expand Down Expand Up @@ -100,13 +120,27 @@ uint64_t crc_fast_digest_get_amount(struct CrcFastDigestHandle *handle);
*/
uint64_t crc_fast_checksum(enum CrcFastAlgorithm algorithm, const char *data, uintptr_t len);

/**
* Helper method to calculate a CRC checksum directly for data using custom parameters
*/
uint64_t crc_fast_checksum_with_params(struct CrcFastParams params,
const char *data,
uintptr_t len);

/**
* Helper method to just calculate a CRC checksum directly for a file using algorithm
*/
uint64_t crc_fast_checksum_file(enum CrcFastAlgorithm algorithm,
const uint8_t *path_ptr,
uintptr_t path_len);

/**
* Helper method to calculate a CRC checksum directly for a file using custom parameters
*/
uint64_t crc_fast_checksum_file_with_params(struct CrcFastParams params,
const uint8_t *path_ptr,
uintptr_t path_len);

/**
* Combine two CRC checksums using algorithm
*/
Expand Down
81 changes: 81 additions & 0 deletions src/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#![cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))]

use crate::structs::CrcParams;
use crate::CrcAlgorithm;
use crate::{get_calculator_target, Digest};
use std::ffi::CStr;
Expand Down Expand Up @@ -68,6 +69,38 @@ impl From<CrcFastAlgorithm> for CrcAlgorithm {
}
}

/// Custom CRC parameters
#[repr(C)]
pub struct CrcFastParams {
pub algorithm: CrcFastAlgorithm,
pub width: u8,
pub poly: u64,
pub init: u64,
pub refin: bool,
pub refout: bool,
pub xorout: u64,
pub check: u64,
pub keys: [u64; 23],
}

// Convert from FFI struct to internal struct
impl From<CrcFastParams> for CrcParams {
fn from(value: CrcFastParams) -> Self {
CrcParams {
algorithm: value.algorithm.into(),
name: "custom", // C interface doesn't need the name field
width: value.width,
poly: value.poly,
init: value.init,
refin: value.refin,
refout: value.refout,
xorout: value.xorout,
check: value.check,
keys: value.keys,
}
}
}

/// Creates a new Digest to compute CRC checksums using algorithm
#[no_mangle]
pub extern "C" fn crc_fast_digest_new(algorithm: CrcFastAlgorithm) -> *mut CrcFastDigestHandle {
Expand All @@ -76,6 +109,16 @@ pub extern "C" fn crc_fast_digest_new(algorithm: CrcFastAlgorithm) -> *mut CrcFa
Box::into_raw(handle)
}

/// Creates a new Digest to compute CRC checksums using custom parameters
#[no_mangle]
pub extern "C" fn crc_fast_digest_new_with_params(
params: CrcFastParams,
) -> *mut CrcFastDigestHandle {
let digest = Box::new(Digest::new_with_params(params.into()));
let handle = Box::new(CrcFastDigestHandle(Box::into_raw(digest)));
Box::into_raw(handle)
}

/// Updates the Digest with data
#[no_mangle]
pub extern "C" fn crc_fast_digest_update(
Expand Down Expand Up @@ -197,6 +240,23 @@ pub extern "C" fn crc_fast_checksum(
}
}

/// Helper method to calculate a CRC checksum directly for data using custom parameters
#[no_mangle]
pub extern "C" fn crc_fast_checksum_with_params(
params: CrcFastParams,
data: *const c_char,
len: usize,
) -> u64 {
if data.is_null() {
return 0;
}
unsafe {
#[allow(clippy::unnecessary_cast)]
let bytes = slice::from_raw_parts(data as *const u8, len);
crate::checksum_with_params(params.into(), bytes)
}
}

/// Helper method to just calculate a CRC checksum directly for a file using algorithm
#[no_mangle]
pub extern "C" fn crc_fast_checksum_file(
Expand All @@ -218,6 +278,27 @@ pub extern "C" fn crc_fast_checksum_file(
}
}

/// Helper method to calculate a CRC checksum directly for a file using custom parameters
#[no_mangle]
pub extern "C" fn crc_fast_checksum_file_with_params(
params: CrcFastParams,
path_ptr: *const u8,
path_len: usize,
) -> u64 {
if path_ptr.is_null() {
return 0;
}

unsafe {
crate::checksum_file_with_params(
params.into(),
&convert_to_string(path_ptr, path_len),
None,
)
.unwrap_or(0) // Return 0 on error instead of panicking
}
}

/// Combine two CRC checksums using algorithm
#[no_mangle]
pub extern "C" fn crc_fast_checksum_combine(
Expand Down
143 changes: 132 additions & 11 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
//! Implements the [std::io::Write](https://doc.rust-lang.org/std/io/trait.Write.html) trait for
//! easier integration with existing code.
//!
//! ```no_run
//! ```rust
//! use std::env;
//! use std::fs::File;
//! use crc_fast::{Digest, CrcAlgorithm::Crc32IsoHdlc};
Expand Down Expand Up @@ -261,6 +261,19 @@ impl Digest {
}
}

/// Creates a new `Digest` instance with custom parameters.
#[inline(always)]
pub fn new_with_params(params: CrcParams) -> Self {
let calculator = Calculator::calculate as CalculatorFn;

Self {
state: params.init,
amount: 0,
params,
calculator,
}
}

/// Updates the CRC state with the given data.
#[inline(always)]
pub fn update(&mut self, data: &[u8]) {
Expand Down Expand Up @@ -361,6 +374,13 @@ pub fn checksum(algorithm: CrcAlgorithm, buf: &[u8]) -> u64 {
calculator(params.init, buf, params) ^ params.xorout
}

/// Computes the CRC checksum for the given data using the custom specified parameters.
pub fn checksum_with_params(params: CrcParams, buf: &[u8]) -> u64 {
let calculator = Calculator::calculate as CalculatorFn;

calculator(params.init, buf, params) ^ params.xorout
}

/// Computes the CRC checksum for the given file using the specified algorithm.
///
/// Appears to be much faster (~2X) than using Writer and io::*, at least on Apple M2 Ultra
Expand All @@ -371,7 +391,7 @@ pub fn checksum(algorithm: CrcAlgorithm, buf: &[u8]) -> u64 {
///
/// # Examples
/// ### checksum_file
///```no_run
///```rust
/// use std::env;
/// use crc_fast::{checksum_file, CrcAlgorithm::Crc32IsoHdlc};
///
Expand All @@ -389,7 +409,32 @@ pub fn checksum_file(
path: &str,
chunk_size: Option<usize>,
) -> Result<u64, std::io::Error> {
let mut digest = Digest::new(algorithm);
checksum_file_with_digest(Digest::new(algorithm), path, chunk_size)
}

/// Computes the CRC checksum for the given file using the custom specified parameters.
///
/// # Errors
///
/// This function will return an error if the file cannot be read.
pub fn checksum_file_with_params(
params: CrcParams,
path: &str,
chunk_size: Option<usize>,
) -> Result<u64, std::io::Error> {
checksum_file_with_digest(Digest::new_with_params(params), path, chunk_size)
}

/// Computes the CRC checksum for the given file using the specified Digest.
///
/// # Errors
///
/// This function will return an error if the file cannot be read.
fn checksum_file_with_digest(
mut digest: Digest,
path: &str,
chunk_size: Option<usize>,
) -> Result<u64, std::io::Error> {
let mut file = File::open(path)?;

// 512KiB KiB was fastest in my benchmarks on an Apple M2 Ultra
Expand Down Expand Up @@ -540,19 +585,65 @@ mod lib {
}
}

#[test]
fn test_checksum_with_custom_params() {
// CRC-32 reflected
assert_eq!(
checksum_with_params(CRC32_ISCSI, TEST_CHECK_STRING),
CRC32_ISCSI.check,
);

// CRC-32 forward
assert_eq!(
checksum_with_params(CRC32_BZIP2, TEST_CHECK_STRING),
CRC32_BZIP2.check,
);

// CRC-64 reflected
assert_eq!(
checksum_with_params(CRC64_NVME, TEST_CHECK_STRING),
CRC64_NVME.check,
);

// CRC-64 forward
assert_eq!(
checksum_with_params(CRC64_ECMA_182, TEST_CHECK_STRING),
CRC64_ECMA_182.check,
);
}

#[test]
fn test_digest_updates_check() {
for config in TEST_ALL_CONFIGS {
let mut digest = Digest::new(config.get_algorithm());
digest.update(b"123");
digest.update(b"456");
digest.update(b"789");
let result = digest.finalize();

assert_eq!(result, config.get_check());
check_digest(Digest::new(config.get_algorithm()), config.get_check());
}
}

#[test]
fn test_digest_updates_check_with_custom_params() {
// CRC-32 reflected
check_digest(Digest::new_with_params(CRC32_ISCSI), CRC32_ISCSI.check);

// CRC-32 forward
check_digest(Digest::new_with_params(CRC32_BZIP2), CRC32_BZIP2.check);

// CRC-64 reflected
check_digest(Digest::new_with_params(CRC64_NVME), CRC64_NVME.check);

// CRC-64 forward
check_digest(
Digest::new_with_params(CRC64_ECMA_182),
CRC64_ECMA_182.check,
);
}

fn check_digest(mut digest: Digest, check: u64) {
digest.update(b"123");
digest.update(b"456");
digest.update(b"789");
assert_eq!(digest.finalize(), check,);
}

#[test]
fn test_small_all_lengths() {
for config in TEST_ALL_CONFIGS {
Expand Down Expand Up @@ -634,7 +725,7 @@ mod lib {
// Create a test file with repeating zeros
let test_file_path = "test/test_crc32_hash_file.bin";
let data = vec![0u8; 1024 * 1024]; // 1 MiB of zeros
if let Err(e) = std::fs::write(test_file_path, &data) {
if let Err(e) = write(test_file_path, &data) {
eprintln!("Skipping test due to write error: {}", e);
return;
}
Expand All @@ -647,6 +738,36 @@ mod lib {
std::fs::remove_file(test_file_path).unwrap();
}

#[test]
fn test_checksum_file_with_custom_params() {
// Create a test file with repeating zeros
let test_file_path = "test/test_crc32_hash_file_custom.bin";
let data = vec![0u8; 1024 * 1024]; // 1 MiB of zeros
if let Err(e) = write(test_file_path, &data) {
eprintln!("Skipping test due to write error: {}", e);
return;
}

// CRC-32 reflected
check_file(CRC32_ISCSI, test_file_path, CRC32_ISCSI.check);

// CRC-32 forward
check_file(CRC32_BZIP2, test_file_path, CRC32_BZIP2.check);

// CRC-64 reflected
check_file(CRC64_NVME, test_file_path, CRC64_NVME.check);

// CRC-64 forward
check_file(CRC64_ECMA_182, test_file_path, CRC64_ECMA_182.check);

std::fs::remove_file(test_file_path).unwrap();
}

fn check_file(params: CrcParams, file_path: &str, check: u64) {
let result = checksum_file_with_params(params, file_path, None).unwrap();
assert_eq!(result, check);
}

#[test]
fn test_writer() {
// Create a test file with repeating zeros
Expand Down