commenting out v_ and q_ biases as they are always const

Author: ganik

DeBERTa/apps/train.py

@@ -265,9 +265,10 @@ def run_onnx_training(args, model, device, train_data, prefix=None):
   for step, batch in enumerate(AsyncDataLoader(train_dataloader, 100)):
     #import pdb
     #pdb.set_trace()
+    lr = torch.tensor([0.0000000e+00]).to(device)
     batch = batch_to(batch, device)
     with torch.no_grad():
-      trainer.train_step(batch['input_ids'], batch['type_ids'], batch['position_ids'], batch['input_mask'], batch['labels'])
+      trainer.train_step(batch['input_ids'], batch['type_ids'], batch['position_ids'], batch['input_mask'], batch['labels'], lr)
       # conversion fails now with:
       # site-packages/torch/onnx/utils.py:617: UserWarning: ONNX export failed on ATen operator broadcast_tensors
       # because torch.onnx.symbolic_opset10.broadcast_tensors does not exist

DeBERTa/deberta/disentangled_attention.py

@@ -77,8 +77,9 @@ def __init__(self, config):
         self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
         self.all_head_size = self.num_attention_heads * self.attention_head_size
         self.in_proj = torch.nn.Linear(config.hidden_size, self.all_head_size*3, bias=False)
-        self.q_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
-        self.v_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
+        # Looks like params below are never updated and const, so removing them
+        #self.q_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
+        #self.v_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
         self.pos_att_type = [x.strip() for x in getattr(config, 'pos_att_type', 'none').lower().split('|')] # c2p|p2c
 
         self.relative_attention = getattr(config, 'relative_attention', False)
@@ -148,8 +149,10 @@ def linear(w,b,x):
             k,v = [linear(qkvw[i], qkvb[i], hidden_states) for i in range(1,3)]
             query_layer, key_layer, value_layer = [self.transpose_for_scores(x) for x in [q,k,v]]
 
-        query_layer += self.transpose_for_scores(self.q_bias.unsqueeze(0).unsqueeze(0))
-        value_layer += self.transpose_for_scores(self.v_bias.unsqueeze(0).unsqueeze(0))
+        q_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
+        v_bias = torch.nn.Parameter(torch.zeros((self.all_head_size), dtype=torch.float))
+        query_layer += self.transpose_for_scores(q_bias.unsqueeze(0).unsqueeze(0))
+        value_layer += self.transpose_for_scores(v_bias.unsqueeze(0).unsqueeze(0))
 
         rel_att = None
         # Take the dot product between "query" and "key" to get the raw attention scores.