add test for ZDICT_trainFromBuffer_cover

Author: folkertdev

c2rust-lib.rs

@@ -38,7 +38,7 @@ pub mod lib {
         mod zstdmt_compress;
     } // mod compress
     pub mod decompress;
-    mod dictBuilder {
+    pub(crate) mod dictBuilder {
         pub(crate) mod cover;
         pub(crate) mod divsufsort;
         pub(crate) mod fastcover;
@@ -68,7 +68,11 @@ pub use crate::lib::decompress::{
 
 pub use crate::lib::common::zstd_common::{ZSTD_getErrorName, ZSTD_isError};
 
-pub use crate::lib::zdict::{ZDICT_getErrorName, ZDICT_isError, ZDICT_trainFromBuffer};
+pub use crate::lib::dictBuilder::cover::ZDICT_trainFromBuffer_cover;
+pub use crate::lib::zdict::{
+    experimental::ZDICT_cover_params_t, ZDICT_getErrorName, ZDICT_isError, ZDICT_params_t,
+    ZDICT_trainFromBuffer,
+};
 
 pub use crate::lib::compress::zstd_compress::{
     ZSTD_compressBound, ZSTD_compress_usingDict, ZSTD_createCCtx, ZSTD_freeCCtx,

lib/dictBuilder/cover.rs

@@ -130,7 +130,7 @@ unsafe fn COVER_map_init(map: *mut COVER_map_t, size: u32) -> core::ffi::c_int {
 }
 static COVER_prime4bytes: u32 = 2654435761;
 unsafe fn COVER_map_hash(map: *mut COVER_map_t, key: u32) -> u32 {
-    (key * COVER_prime4bytes) >> 32u32.wrapping_sub((*map).sizeLog)
+    (key.wrapping_mul(COVER_prime4bytes)) >> 32u32.wrapping_sub((*map).sizeLog)
 }
 unsafe fn COVER_map_index(map: *mut COVER_map_t, key: u32) -> u32 {
     let hash = COVER_map_hash(map, key);

test-libzstd-rs-sys/src/decompress.rs

@@ -1,12 +1,10 @@
-use std::ffi::CStr;
+use std::ffi::{c_void, CStr};
 
 use crate::assert_eq_rs_c;
 
 macro_rules! decompress_stream {
     ($compressed:expr, $dict:expr) => {
         unsafe {
-            use core::ffi::c_void;
-
             // Allocate and initialize a decompression context
             let dctx = ZSTD_createDCtx();
             if dctx.is_null() {

test-libzstd-rs-sys/src/dict_builder.rs

@@ -0,0 +1,107 @@
+use crate::assert_eq_rs_c;
+use std::ffi::{c_void, CStr};
+
+#[test]
+#[cfg_attr(target_vendor = "apple", ignore = "test aborted with signal 10")]
+fn test_train_from_buffer_cover() {
+    let input_data = "The quick brown fox jumps high";
+
+    assert_eq_rs_c!({
+        let samples: [&str; 16] = [
+            "The quick brown fox jumps over the lazy dog",
+            "The quick brown fox jumps high",
+            "The slow turtle crawls under the energetic cat",
+            "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
+            "Pack my box with five dozen liquor jugs",
+            "Bright vixens jump; dozy fowl quack",
+            "Sphinx of black quartz, judge my vow",
+            "How razorback-jumping frogs can level six piqued gymnasts",
+            "Crazy Fredrick bought many very exquisite opal jewels",
+            "Five quacking zephyrs jolt my wax bed",
+            "Jackdaws love my big sphinx of quartz",
+            "Two driven jocks help fax my big quiz",
+            "The wizard quickly jinxed the gnomes before they vaporized",
+            "Quick zephyrs blow, vexing daft Jim",
+            "Heavy boxes perform quick waltzes and jigs",
+            "Jovial harpooned sharks quizzed exotic men drinking water",
+        ];
+
+        let mut sample_data = Vec::new();
+
+        let mut sample_sizes = Vec::new();
+
+        for &s in &samples {
+            sample_data.extend_from_slice(s.as_bytes());
+
+            sample_sizes.push(s.len());
+        }
+
+        let dict_capacity = 16 * 1024;
+
+        let mut dict_buffer = vec![0u8; dict_capacity];
+
+        let params = ZDICT_cover_params_t {
+            k: 200,
+            d: 8,
+            steps: 4,
+            nbThreads: 1,
+            splitPoint: 75.0,
+            shrinkDict: 0,
+            shrinkDictMaxRegression: 1,
+            zParams: ZDICT_params_t {
+                compressionLevel: 3,
+                notificationLevel: 0,
+                dictID: 0,
+            },
+        };
+
+        let dict_size = ZDICT_trainFromBuffer_cover(
+            dict_buffer.as_mut_ptr() as *mut c_void,
+            dict_buffer.len(),
+            sample_data.as_ptr() as *const c_void,
+            sample_sizes.as_ptr(),
+            sample_sizes.len() as u32,
+            params,
+        );
+
+        assert_eq!(
+            ZDICT_isError(dict_size),
+            0,
+            "Dict training failed {:?}",
+            CStr::from_ptr(ZDICT_getErrorName(dict_size)).to_str(),
+        );
+
+        dict_buffer.truncate(dict_size);
+
+        println!("Dictionary size: {}", dict_size);
+
+        let cctx = ZSTD_createCCtx();
+
+        assert!(!cctx.is_null());
+
+        let max_compressed_size = ZSTD_compressBound(input_data.len());
+
+        let mut compressed = vec![0u8; max_compressed_size];
+
+        let compressed_size = ZSTD_compress_usingDict(
+            cctx,
+            compressed.as_mut_ptr() as *mut c_void,
+            compressed.len(),
+            input_data.as_bytes().as_ptr() as *const c_void,
+            input_data.len(),
+            dict_buffer.as_ptr() as *const c_void,
+            dict_buffer.len(),
+            3, // compression level
+        );
+
+        assert_eq!(ZSTD_isError(compressed_size), 0, "Compression failed");
+
+        compressed.truncate(compressed_size);
+
+        println!("Compressed size: {}", compressed_size);
+
+        ZSTD_freeCCtx(cctx);
+
+        (compressed, dict_buffer)
+    });
+}

test-libzstd-rs-sys/src/lib.rs

@@ -1,5 +1,6 @@
 #[cfg(test)]
 mod decompress;
+mod dict_builder;
 
 #[macro_export]
 macro_rules! assert_eq_rs_c {