test ZDICT_trainFromBuffer_fastCover

Author: folkertdev

c2rust-lib.rs

@@ -140,9 +140,10 @@ pub use crate::lib::common::zstd_common::{
 };
 
 pub use crate::lib::dictBuilder::cover::ZDICT_trainFromBuffer_cover;
+pub use crate::lib::dictBuilder::fastcover::ZDICT_trainFromBuffer_fastCover;
 pub use crate::lib::zdict::{
-    experimental::ZDICT_cover_params_t, ZDICT_getDictID, ZDICT_getErrorName, ZDICT_isError,
-    ZDICT_params_t, ZDICT_trainFromBuffer,
+    experimental::{ZDICT_cover_params_t, ZDICT_fastCover_params_t},
+    ZDICT_getDictID, ZDICT_getErrorName, ZDICT_isError, ZDICT_params_t, ZDICT_trainFromBuffer,
 };
 
 pub use crate::lib::compress::zstd_compress::{

test-libzstd-rs-sys/src/dict_builder.rs

@@ -1,36 +1,36 @@
 use crate::assert_eq_rs_c;
 use std::ffi::{c_void, CStr};
 
+const SAMPLES: [&str; 16] = [
+    "The quick brown fox jumps over the lazy dog",
+    "The quick brown fox jumps high",
+    "The slow turtle crawls under the energetic cat",
+    "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
+    "Pack my box with five dozen liquor jugs",
+    "Bright vixens jump; dozy fowl quack",
+    "Sphinx of black quartz, judge my vow",
+    "How razorback-jumping frogs can level six piqued gymnasts",
+    "Crazy Fredrick bought many very exquisite opal jewels",
+    "Five quacking zephyrs jolt my wax bed",
+    "Jackdaws love my big sphinx of quartz",
+    "Two driven jocks help fax my big quiz",
+    "The wizard quickly jinxed the gnomes before they vaporized",
+    "Quick zephyrs blow, vexing daft Jim",
+    "Heavy boxes perform quick waltzes and jigs",
+    "Jovial harpooned sharks quizzed exotic men drinking water",
+];
+
 #[test]
 #[cfg(not(target_family = "wasm"))]
 fn test_train_from_buffer_cover() {
     let input_data = "The quick brown fox jumps high";
 
     assert_eq_rs_c!({
-        let samples: [&str; 16] = [
-            "The quick brown fox jumps over the lazy dog",
-            "The quick brown fox jumps high",
-            "The slow turtle crawls under the energetic cat",
-            "Lorem ipsum dolor sit amet, consectetur adipiscing elit",
-            "Pack my box with five dozen liquor jugs",
-            "Bright vixens jump; dozy fowl quack",
-            "Sphinx of black quartz, judge my vow",
-            "How razorback-jumping frogs can level six piqued gymnasts",
-            "Crazy Fredrick bought many very exquisite opal jewels",
-            "Five quacking zephyrs jolt my wax bed",
-            "Jackdaws love my big sphinx of quartz",
-            "Two driven jocks help fax my big quiz",
-            "The wizard quickly jinxed the gnomes before they vaporized",
-            "Quick zephyrs blow, vexing daft Jim",
-            "Heavy boxes perform quick waltzes and jigs",
-            "Jovial harpooned sharks quizzed exotic men drinking water",
-        ];
-
         let mut sample_data = Vec::new();
 
         let mut sample_sizes = Vec::new();
 
-        for &s in &samples {
+        for &s in &SAMPLES {
             sample_data.extend_from_slice(s.as_bytes());
 
             sample_sizes.push(s.len());
@@ -105,3 +105,91 @@ fn test_train_from_buffer_cover() {
         (compressed, dict_buffer)
     });
 }
+
+#[test]
+#[cfg(not(target_family = "wasm"))]
+fn test_train_from_buffer_fastcover() {
+    let input_data = "The quick brown fox jumps high";
+
+    assert_eq_rs_c!({
+        let mut sample_data = Vec::new();
+
+        let mut sample_sizes = Vec::new();
+
+        for &s in &SAMPLES {
+            sample_data.extend_from_slice(s.as_bytes());
+
+            sample_sizes.push(s.len());
+        }
+
+        let dict_capacity = 16 * 1024;
+
+        let mut dict_buffer = vec![0u8; dict_capacity];
+
+        let params = ZDICT_fastCover_params_t {
+            k: 200,
+            d: 8,
+            f: 20,
+            steps: 4,
+            nbThreads: 1,
+            splitPoint: 75.0,
+            shrinkDict: 0,
+            shrinkDictMaxRegression: 1,
+            zParams: ZDICT_params_t {
+                compressionLevel: 3,
+                notificationLevel: 0,
+                dictID: 0,
+            },
+            accel: 1,
+        };
+
+        let dict_size = ZDICT_trainFromBuffer_fastCover(
+            dict_buffer.as_mut_ptr() as *mut c_void,
+            dict_buffer.len(),
+            sample_data.as_ptr() as *const c_void,
+            sample_sizes.as_ptr(),
+            sample_sizes.len() as u32,
+            params,
+        );
+
+        assert_eq!(
+            ZDICT_isError(dict_size),
+            0,
+            "Dict training failed {:?}",
+            CStr::from_ptr(ZDICT_getErrorName(dict_size)).to_str(),
+        );
+
+        dict_buffer.truncate(dict_size);
+
+        println!("Dictionary size: {}", dict_size);
+
+        let cctx = ZSTD_createCCtx();
+
+        assert!(!cctx.is_null());
+
+        let max_compressed_size = ZSTD_compressBound(input_data.len());
+
+        let mut compressed = vec![0u8; max_compressed_size];
+
+        let compressed_size = ZSTD_compress_usingDict(
+            cctx,
+            compressed.as_mut_ptr() as *mut c_void,
+            compressed.len(),
+            input_data.as_bytes().as_ptr() as *const c_void,
+            input_data.len(),
+            dict_buffer.as_ptr() as *const c_void,
+            dict_buffer.len(),
+            3, // compression level
+        );
+
+        assert_eq!(ZSTD_isError(compressed_size), 0, "Compression failed");
+
+        compressed.truncate(compressed_size);
+
+        println!("Compressed size: {}", compressed_size);
+
+        ZSTD_freeCCtx(cctx);
+
+        (compressed, dict_buffer)
+    });
+}