Skip to content

Commit 52516fa

Browse files
committed
Added compression of the output file using zstd
1 parent 33337ee commit 52516fa

File tree

4 files changed

+152
-7
lines changed

4 files changed

+152
-7
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
/target
22
.vscode/
33
output.msgpack
4-
**.msgpack
4+
**.msgpack
5+
**.temp
6+
**.zst

Cargo.lock

Lines changed: 117 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ edition = "2021"
66
[dependencies]
77
rmp-serde = "1.3.0"
88
serde = "1.0.219"
9-
walkdir = "2.5.0"
9+
walkdir = "2.5.0"
10+
zstd = "0.13.3"

src/yadm/serialize.rs

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use walkdir::WalkDir;
33
use std::collections::HashMap;
44
use std::fs::File;
55
use std::io::Write;
6+
use zstd;
67

78
/// TODO: Faire la doc
89
pub fn serialize(path: &Path) {
@@ -28,14 +29,39 @@ pub fn serialize(path: &Path) {
2829
}).collect()
2930
}
3031

32+
// Step 3 : We serializing the Vector to a message pack.
33+
fn serialize_to_msgpack(hashmap: &Vec<HashMap<String, String>>) -> File {
34+
let encoded: Vec<u8> = rmp_serde::encode::to_vec(&hashmap).unwrap();
35+
let mut file: File = File::create("output.msgpack.temp").expect("Creation of the file impossible");
36+
file.write_all(&encoded).expect("Writing of the file impossible");
37+
// We will rewriting the file using the file content but this technique isn't good for production
38+
// In the futur commit, TODO: replacing this with a stream flux and output into a file AFTER.
39+
return file;
40+
}
41+
3142
// We check if the path go to only one File to skip the WalkDir call if not needed.
3243
let hashmap: Vec<HashMap<String, String>> = if path.is_file() {
3344
parsing_to_hashmap(&[path.to_path_buf()])
3445
} else {
3546
parsing_to_hashmap(&scan_dir(path))
3647
};
3748

38-
let encoded: Vec<u8> = rmp_serde::encode::to_vec(&hashmap).unwrap();
39-
let mut file: File = File::create("output.msgpack").expect("Creation of the file impossible");
40-
file.write_all(&encoded).expect("Writing of the file impossible");
41-
}
49+
// Sérialisation MsgPack dans fichier temporaire
50+
serialize_to_msgpack(&hashmap);
51+
52+
// Ouvrir le fichier sérialisé en lecture
53+
let mut uncompressed_file = File::open("output.msgpack.temp").expect("Failed to open serialized file");
54+
55+
// Ouvrir un fichier de sortie pour recevoir la version compressée
56+
let output_file = File::create("output.msgpack.zst").expect("Failed to create output compressed file");
57+
58+
// Compresser du fichier sérialisé vers le fichier compressé avec un niveau de compression à 22 (max)
59+
zstd::stream::copy_encode(&mut uncompressed_file, output_file, 22)
60+
.expect("Compression failed");
61+
62+
match std::fs::remove_file(Path::new("output.msgpack.temp")) {
63+
Ok(_) => println!("Fichier supprimé avec succès !"),
64+
Err(e) => eprintln!("Erreur lors de la suppression du fichier : {}", e),
65+
}
66+
67+
}

0 commit comments

Comments
 (0)