wordlist: Compress embedded wordlist using zstd

Author: cohaereo

Cargo.lock

@@ -0,0 +1 @@
+wordlist.zst

crates/strings/.gitignore

@@ -9,6 +9,11 @@ quicktag-core = { path = "../core" }
 anyhow.workspace = true
 binrw.workspace = true
 bytemuck = "1"
+lazy_static = "1.4.0"
 log.workspace = true
 rustc-hash.workspace = true
 tiger-pkg.workspace = true
+zstd = { version = "0.13.3", default-features = false }
+
+[build-dependencies]
+zstd = { version = "0.13.3", default-features = false }

crates/strings/Cargo.toml

@@ -0,0 +1,11 @@
+use std::{fs::File, io::BufReader};
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=../../wordlist.txt");
+
+    let source =
+        BufReader::new(File::open("../../wordlist.txt").expect("Failed to open wordlist.txt"));
+    let destination = File::create("wordlist.zst").expect("Failed to create wordlist.zst");
+    zstd::stream::copy_encode(source, destination, 11).expect("Failed to compress wordlist.txt");
+}

crates/strings/build.rs

@@ -1,9 +1,17 @@
 use std::{io::BufRead, time::Instant};
 
+use lazy_static::lazy_static;
 use log::{error, info};
 use quicktag_core::util::{FNV1_BASE, fnv1};
 
-const WORDLIST: &str = include_str!("../../../wordlist.txt");
+lazy_static! {
+    static ref WORDLIST: String = {
+        info!("Decompressing wordlist...");
+        const DATA: &[u8] = include_bytes!("../wordlist.zst");
+        let decompressed = zstd::stream::decode_all(&mut std::io::Cursor::new(DATA)).unwrap();
+        String::from_utf8(decompressed).unwrap()
+    };
+}
 
 pub fn load_wordlist<F: FnMut(&str, u32)>(mut callback: F) {
     let load_start = Instant::now();