Merge pull request yzhangcs#52 from KoichiYasuoka/main

yzhangcs · web-flow · commit 0da03adefdb7 · 2020-12-24T10:54:23.000+08:00
For UNK chars inside token
diff --git a/setup.py b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name='supar',
-    version='1.0.0',
+    version='1.0.0+dev20201223',
     author='Yu Zhang',
     author_email='yzhang.cs@outlook.com',
     description='Syntactic Parsing Models',
@@ -32,6 +32,6 @@
             'crf-constituency=supar.cmds.crf_constituency:main'
         ]
     },
-    python_requires='>=3.7',
+    python_requires='>=3.6',
     zip_safe=False
 )
diff --git a/supar/cmds/cmd.py b/supar/cmds/cmd.py
@@ -13,7 +13,8 @@ def parse(parser):
     parser.add_argument('--seed', '-s', default=1, type=int, help='seed for generating random numbers')
     parser.add_argument('--threads', '-t', default=16, type=int, help='max num of threads')
     parser.add_argument('--batch-size', default=5000, type=int, help='batch size')
-    parser.add_argument("--local_rank", type=int, default=-1, help='node rank for distributed training')
+    parser.add_argument('--local_rank', default=-1, type=int, help='node rank for distributed training')
+    parser.add_argument('--epochs', default=5000, type=int, help='epochs')
     args, unknown = parser.parse_known_args()
     args, _ = parser.parse_known_args(unknown, args)
     args = Config(**vars(args))
diff --git a/supar/utils/field.py b/supar/utils/field.py
@@ -318,8 +318,8 @@ def transform(self, sequences):
         if self.fix_len <= 0:
             self.fix_len = max(len(token) for seq in sequences for token in seq)
         if self.use_vocab:
-            sequences = [[[self.vocab[i] for i in token] if token else [self.unk_index] for token in seq]
-                         for seq in sequences]
+            sequences = [[[self.vocab[i] if i in self.vocab else self.unk_index for i in token] if token else [self.unk_index]
+                         for token in seq] for seq in sequences]
         if self.bos:
             sequences = [[[self.bos_index]] + seq for seq in sequences]
         if self.eos:
