fix char embedding

atulkum · atulkum · commit d55b6f3dc83b · 2019-01-27T00:37:28.000-08:00
diff --git a/neural_ner/data_utils/batcher.py b/neural_ner/data_utils/batcher.py
@@ -66,8 +66,6 @@ def prepare_batch(self, batch):
         all_chars = []
         raw_sentences = []
 
-        max_length = max(words_len)
-
         for i in idx:
             datum = batch[i]
             for v in features:
@@ -76,7 +74,7 @@ def prepare_batch(self, batch):
             raw_sentences.append(datum['raw_sentence'])
 
             chars = datum['chars']
-            chars_padded, chars_padded_lens = pad_chars(chars, max_length)
+            chars_padded, chars_padded_lens = pad_chars(chars)
             chars_padded = torch.Tensor(chars_padded).long()
             chars_padded_lens = torch.Tensor(chars_padded_lens).long()
             if self.config.is_cuda:
diff --git a/neural_ner/data_utils/sentence_utils.py b/neural_ner/data_utils/sentence_utils.py
@@ -15,7 +15,7 @@ def pad_items(items, is_tag=False):
 
     return np.array(padded_items), np.array(padded_items_len)
 
-def pad_chars(items, max_word_len):
+def pad_chars(items):
     padded_items = []
     padded_items_len = [len(item) for item in items]
     max_length = max(padded_items_len)
@@ -24,10 +24,6 @@ def pad_chars(items, max_word_len):
     for item in items:
         padding = [pad_id] * (max_length - len(item))
         padded_items.append(item + padding)
-    for i in range(len(items), max_word_len):
-        padding = [pad_id] * max_length
-        padded_items.append(padding)
-        padded_items_len.append(1)
 
     return np.array(padded_items), np.array(padded_items_len)
 
@@ -62,8 +58,8 @@ def prepare_sentence(s, vocab, config):
     str_words = [w[0] for w in s]
     word_seq, word_char_seq = get_char_word_seq(str_words, config.lower, config.zeros)
 
-    words = [vocab.word_to_id[w] if w in vocab.word_to_id else Constants.UNK_ID for w in word_seq]
-    chars = [[vocab.char_to_id[c] for c in char_seq if c in vocab.char_to_id] for char_seq in word_char_seq]
+    words = [vocab.word_to_id.get(w, Constants.UNK_ID) for w in word_seq]
+    chars = [[vocab.char_to_id.get(c, Constants.UNK_ID) for c in char_seq] for char_seq in word_char_seq]
     caps = [cap_feature(w) for w in str_words]
 
     return {
@@ -74,3 +70,16 @@ def prepare_sentence(s, vocab, config):
         'raw_sentence':str_words
     }
 
+if __name__ == '__main__':
+    from config import config
+    from data_utils.vocab import Vocab
+    from data_utils.utils import prepare_dataset
+
+    vocab = Vocab(config)
+
+    sentences = [['a O', 'b O', 'c O', '| O']]
+
+    data = prepare_dataset(sentences, vocab, config)
+    datum = data[0]
+    chars = datum['chars']
+    chars_padded, chars_padded_lens = pad_chars(chars)
diff --git a/neural_ner/model.py b/neural_ner/model.py
@@ -55,17 +55,17 @@ def __init__(self, vocab, config):
     def forward(self, batch):
         sentence = batch['words']
         lengths = batch['words_lens']
+
         if self.config.is_caps:
             caps = batch['caps']
-
+        max_length = torch.max(lengths)
         char_emb = []
         word_embed = self.word_embeds(sentence)
         for chars, char_len in batch['chars']:
             seq_embed = self.char_embeds(chars)
             seq_lengths, sort_idx = torch.sort(char_len, descending=True)
             _, unsort_idx = torch.sort(sort_idx)
             seq_embed = seq_embed[sort_idx]
-
             packed = pack_padded_sequence(seq_embed, seq_lengths, batch_first=True)
             output, hidden = self.lstm_char(packed)
             lstm_feats, _ = pad_packed_sequence(output, batch_first=True)
@@ -79,7 +79,10 @@ def forward(self, batch):
             seq_rep_bwd = seq_rep[unsort_idx, last_idx, 1]
 
             seq_out = torch.cat([seq_rep_fwd, seq_rep_bwd], 1)
+            # fill up the dummy char embedding for padding
+            seq_out = F.pad(seq_out, (0, 0, 0, max_length - seq_out.size(0)))
             char_emb.append(seq_out.unsqueeze(0))
+
         char_emb = torch.cat(char_emb, 0) #b x n x c_dim
 
         if self.config.is_caps:
diff --git a/neural_ner/process_training.py b/neural_ner/process_training.py
@@ -108,8 +108,6 @@ def evalute_test_dev(self, summary_writer, epoch, global_step, exp_loss):
         logging.info("Dev: Epoch %d, Iter %d, loss: %f, F1: %f" % (epoch, global_step, dev_loss, dev_f1))
         write_summary(dev_f1, "dev/F1", summary_writer, global_step)
 
-        self.evaluate_test()
-
         return dev_f1
 
     def inference(self, str):
