unknown token

yzhangcs · yzhangcs · commit 9346c7116f1f · 2020-12-24T00:31:06.000+09:00
diff --git a/supar/utils/field.py b/supar/utils/field.py
@@ -318,8 +318,8 @@ def transform(self, sequences):
         if self.fix_len <= 0:
             self.fix_len = max(len(token) for seq in sequences for token in seq)
         if self.use_vocab:
-            sequences = [[[self.vocab[i] for i in token] if token else [self.unk_index] for token in seq]
-                         for seq in sequences]
+            sequences = [[[self.vocab[i] if i in self.vocab else self.unk_index for i in token] if token else [self.unk_index]
+                         for token in seq] for seq in sequences]
         if self.bos:
             sequences = [[[self.bos_index]] + seq for seq in sequences]
         if self.eos:
