diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 764163c438a15..1313f9be795ac 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -3598,6 +3598,9 @@ def set_vocab(self):
 
         self.gguf_writer.add_add_space_prefix(False)
 
+        special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
+        special_vocab.add_to_gguf(self.gguf_writer)
+
     def set_gguf_parameters(self):
         hparams = self.hparams
         block_count = hparams["num_hidden_layers"]