[Update] Use char sequences to get the final hiiden states

Author: seanliu96

layers.py

@@ -55,7 +55,7 @@ def forward(self, x, x_mask):
         Output:
             x_encoded: batch * len * hdim_encoded
         """
-        if x_mask.data.sum() == 0:
+        if x_mask.data.sum() == 0 or x_mask.data.eq(1).long().sum(1).min() == 0:
             # No padding necessary.
             output = self._forward_unpadded(x, x_mask)
         elif self.padding or not self.training:

m_reader.py

@@ -45,7 +45,7 @@ def __init__(self, args, normalize=True):
             dropout_output=args.dropout_rnn_output,
             concat_layers=False,
             rnn_type=self.RNN_TYPES[args.rnn_type],
-            padding=args.rnn_padding,
+            padding=False,
         )
 
         doc_input_size = args.embedding_dim + args.char_hidden_size * 2 + args.num_features
@@ -127,8 +127,14 @@ def forward(self, x1, x1_c, x1_f, x1_mask, x2, x2_c, x2_f, x2_mask):
             x2_c_emb = F.dropout(x2_c_emb, p=self.args.dropout_emb, training=self.training)
 
         # Generate char features
-        x1_c_features = self.char_rnn(x1_c_emb, x1_mask)[:,-1,:]
-        x2_c_features = self.char_rnn(x2_c_emb, x2_mask)[:,-1,:]
+        x1_c_features = self.char_rnn(
+            x1_c_emb.reshape((x1_c_emb.size(0) * x1_c_emb.size(1), x1_c_emb.size(2), x1_c_emb.size(3))), 
+            x1_mask.unsqueeze(2).repeat(1, 1, x1_c_emb.size(2)).reshape((x1_c_emb.size(0) * x1_c_emb.size(1), x1_c_emb.size(2)))
+            ).reshape((x1_c_emb.size(0), x1_c_emb.size(1), x1_c_emb.size(2), -1))[:,:,-1,:]
+        x2_c_features = self.char_rnn(
+            x2_c_emb.reshape((x2_c_emb.size(0) * x2_c_emb.size(1), x2_c_emb.size(2), x2_c_emb.size(3))), 
+            x2_mask.unsqueeze(2).repeat(1, 1, x2_c_emb.size(2)).reshape((x2_c_emb.size(0) * x2_c_emb.size(1), x2_c_emb.size(2)))
+            ).reshape((x2_c_emb.size(0), x2_c_emb.size(1), x2_c_emb.size(2), -1))[:,:,-1,:] 
 
         # Combine input
         crnn_input = [x1_emb, x1_c_features]
@@ -156,4 +162,4 @@ def forward(self, x1, x1_c, x1_f, x1_mask, x2, x2_c, x2_f, x2_mask):
         # Predict
         start_scores, end_scores = self.mem_ans_ptr.forward(c_check, q, x1_mask, x2_mask)
 
-        return start_scores, end_scores
+        return start_scores, end_scores

r_net.py

@@ -45,7 +45,7 @@ def __init__(self, args, normalize=True):
             dropout_output=args.dropout_rnn_output,
             concat_layers=False,
             rnn_type=self.RNN_TYPES[args.rnn_type],
-            padding=args.rnn_padding,
+            padding=False,
         )
 
         doc_input_size = args.embedding_dim + args.char_hidden_size * 2
@@ -146,8 +146,14 @@ def forward(self, x1, x1_c, x1_f, x1_mask, x2, x2_c, x2_f, x2_mask):
             x2_c_emb = F.dropout(x2_c_emb, p=self.args.dropout_emb, training=self.training)
 
         # Generate char features
-        x1_c_features = self.char_rnn(x1_c_emb, x1_mask)[:,-1,:]
-        x2_c_features = self.char_rnn(x2_c_emb, x2_mask)[:,-1,:]
+        x1_c_features = self.char_rnn(
+            x1_c_emb.reshape((x1_c_emb.size(0) * x1_c_emb.size(1), x1_c_emb.size(2), x1_c_emb.size(3))), 
+            x1_mask.unsqueeze(2).repeat(1, 1, x1_c_emb.size(2)).reshape((x1_c_emb.size(0) * x1_c_emb.size(1), x1_c_emb.size(2)))
+            ).reshape((x1_c_emb.size(0), x1_c_emb.size(1), x1_c_emb.size(2), -1))[:,:,-1,:]
+        x2_c_features = self.char_rnn(
+            x2_c_emb.reshape((x2_c_emb.size(0) * x2_c_emb.size(1), x2_c_emb.size(2), x2_c_emb.size(3))), 
+            x2_mask.unsqueeze(2).repeat(1, 1, x2_c_emb.size(2)).reshape((x2_c_emb.size(0) * x2_c_emb.size(1), x2_c_emb.size(2)))
+            ).reshape((x2_c_emb.size(0), x2_c_emb.size(1), x2_c_emb.size(2), -1))[:,:,-1,:] 
 
         # Combine input
         crnn_input = [x1_emb, x1_c_features]

spacy_tokenizer.py

@@ -48,9 +48,9 @@ def chars(self, uncased=False):
             uncased: lower cases characters
         """
         if uncased:
-            return [c.lower() for t in self.data for c in t[self.CHAR]]
+            return [[c.lower() for c in t[self.CHAR]] for t in self.data]
         else:
-            return [c for t in self.data for c in t[self.CHAR]]
+            return [[c for c in t[self.CHAR]] for t in self.data]
 
     def words(self, uncased=False):
         """Returns a list of the text of each token
@@ -174,7 +174,6 @@ def tokenize(self, text):
 
             data.append((
                 tokens[i].text,
-                # tokens[i].text[0] if len(tokens[i].text) > 0 else '',
                 list(tokens[i].text),
                 text[start_ws: end_ws],
                 (tokens[i].idx, tokens[i].idx + len(tokens[i].text)),

utils.py

@@ -141,11 +141,12 @@ def index_embedding_chars(char_embedding_file):
 def load_chars(args, examples):
     """Iterate and index all the chars in examples (documents + questions)."""
     def _insert(iterable):
-        for c in iterable:
-            c = Dictionary.normalize(c)
-            if valid_chars and c not in valid_chars:
-                continue
-            chars.add(c)
+        for cs in iterable:
+            for c in cs: 
+                c = Dictionary.normalize(c)
+                if valid_chars and c not in valid_chars:
+                    continue
+                chars.add(c)
 
     if args.restrict_vocab and args.char_embedding_file:
         logger.info('Restricting to chars in %s' % args.char_embedding_file)

vector.py

@@ -19,9 +19,9 @@ def vectorize(ex, model, single_answer=False):
 
     # Index words
     document = torch.LongTensor([word_dict[w] for w in ex['document']])
-    document_char = torch.LongTensor([char_dict[c] for c in ex['document_char']])
+    document_char = [torch.LongTensor([char_dict[c] for c in cs]) for cs in ex['document_char']]
     question = torch.LongTensor([word_dict[w] for w in ex['question']])
-    question_char = torch.LongTensor([char_dict[c] for c in ex['question_char']])
+    question_char = [torch.LongTensor([char_dict[c] for c in cs]) for cs in ex['question_char']]
 
     # Create extra features vector
     if len(feature_dict) > 0:
@@ -120,8 +120,10 @@ def batchify(batch):
 
     # Batch documents and features
     max_length = max([d.size(0) for d in docs])
+    # max_char_length = max([c.size(0) for cs in doc_chars for c in cs])
+    max_char_length = 13
     x1 = torch.LongTensor(len(docs), max_length).zero_()
-    x1_c = torch.LongTensor(len(docs), max_length).zero_()
+    x1_c = torch.LongTensor(len(docs), max_length, max_char_length).zero_()
     x1_mask = torch.ByteTensor(len(docs), max_length).fill_(1)
     if c_features[0] is None:
         x1_f = None
@@ -132,13 +134,15 @@ def batchify(batch):
         x1_mask[i, :d.size(0)].fill_(0)
         if x1_f is not None:
             x1_f[i, :d.size(0)].copy_(c_features[i])
-    for i, c in enumerate(doc_chars):
-        x1_c[i, :c.size(0)].copy_(c)
+    for i, cs in enumerate(doc_chars):
+        for j, c in enumerate(cs):
+            c_ = c[:max_char_length]
+            x1_c[i, j, :c_.size(0)].copy_(c_)
 
     # Batch questions
     max_length = max([q.size(0) for q in questions])
     x2 = torch.LongTensor(len(questions), max_length).zero_()
-    x2_c = torch.LongTensor(len(questions), max_length).zero_()
+    x2_c = torch.LongTensor(len(questions), max_length, max_char_length).zero_()
     x2_mask = torch.ByteTensor(len(questions), max_length).fill_(1)
     if q_features[0] is None:
         x2_f = None
@@ -149,8 +153,10 @@ def batchify(batch):
         x2_mask[i, :d.size(0)].fill_(0)
         if x2_f is not None:
             x2_f[i, :d.size(0)].copy_(q_features[i])
-    for i, c in enumerate(question_chars):
-        x2_c[i, :c.size(0)].copy_(c)
+    for i, cs in enumerate(question_chars):
+        for j, c in enumerate(cs):
+            c_ = c[:max_char_length]
+            x2_c[i, j, :c_.size(0)].copy_(c_)
 
     # Maybe return without targets
     if len(batch[0]) == NUM_INPUTS + NUM_EXTRA: