auto add pad token id into disabled ids for safety

Author: Saibo-creator

src/lib.rs

@@ -35,12 +35,16 @@ impl CodebookConfig {
         pad_token_id: usize,
         disabled_ids: Option<HashSet<usize>>,
     ) -> Self {
+        // insert the pad token id in the disabled ids
+        let mut disabled_ids = disabled_ids.unwrap_or_else(|| HashSet::new());
+        disabled_ids.insert(pad_token_id);
+
         Self {
             initial_vocab_size,
             max_codebook_size,
             max_subtokens,
             pad_token_id,
-            disabled_ids: disabled_ids.unwrap_or_default(),
+            disabled_ids,
         }
     }
 }

tests/test.rs

@@ -4,6 +4,7 @@ use zip2zip_compression::{LZWCompressor, CodebookConfig, PaddingStrategy, Codebo
 fn get_alphabet_codebook_config() -> CodebookConfig {
     let mut disabled_ids: HashSet<usize> = HashSet::new();
     disabled_ids.insert(26); // 'z'
+    disabled_ids.insert(0); // '\0' padding token
 
     // 26 letters + 1 for the pad token
     CodebookConfig::new(27, 100, 5, 0, Some(disabled_ids))