bugfix

Author: james

teal/grab_acts.py

@@ -55,13 +55,9 @@
 print(len(text))
 bsz, seq_len = 10, 2048
 
-input_ids = []
-for i in range(0, len(text), seq_len):
-    ttext = text[i:i+seq_len]
-    encodings = tokenizer(ttext, truncation=True, return_tensors="pt", max_length=seq_len, return_overflowing_tokens=True, padding="max_length")
-    input_ids.append(encodings.input_ids)
+encodings = tokenizer(text, truncation=True, return_tensors="pt", max_length=seq_len, return_overflowing_tokens=True, padding="max_length")
 
-input_ids = torch.cat(input_ids, dim=0)[:bsz,:].to(device="cuda:0")
+input_ids = encodings.input_ids[:bsz,:].to(device="cuda:0")
 print(input_ids.shape)
 
 hidden_states = model.model.embed_tokens(input_ids)

utils/utils.py

@@ -49,6 +49,8 @@ def forward(self, x):
         return self.apply(x)
 
     def apply(self, x):
+        nonzero = (x.abs() > self.threshold).sum()
+        print(f"Nonzero proportion: {nonzero / x.numel()}")
         return x.abs().gt(self.threshold) * x
 
     def get_threshold(self):
@@ -222,7 +224,7 @@ def get_sparse_model(model_name, device, histogram_path, **kwargs):
 
 def get_tokenizer(tokenizer_name):
     tokenizer = transformers.AutoTokenizer.from_pretrained(
-        tokenizer_name, use_fast=False, trust_remote_code=True
+        tokenizer_name, use_fast=True, trust_remote_code=True
     )
 
     if tokenizer.pad_token_id is None: