Minor revisions

yzhangcs · yzhangcs · commit 8d573843191f · 2021-04-27T00:49:30.000+08:00
diff --git a/supar/utils/tokenizer.py b/supar/utils/tokenizer.py
@@ -6,10 +6,10 @@ class Tokenizer:
     def __init__(self, lang='en'):
         import stanza
         try:
-            self.pipeline = stanza.Pipeline(lang=lang, processors='tokenize', tokenize_no_ssplit=True)
+            self.pipeline = stanza.Pipeline(lang=lang, processors='tokenize', verbose=False, tokenize_no_ssplit=True)
         except Exception:
             stanza.download(lang=lang, resources_url='stanford')
-            self.pipeline = stanza.Pipeline(lang=lang, processors='tokenize', tokenize_no_ssplit=True)
+            self.pipeline = stanza.Pipeline(lang=lang, processors='tokenize', verbose=False, tokenize_no_ssplit=True)
 
     def __call__(self, text):
         return [i.text for i in self.pipeline(text).sentences[0].tokens]
diff --git a/supar/utils/vocab.py b/supar/utils/vocab.py
@@ -63,6 +63,9 @@ def __setstate__(self, state):
         state['stoi'] = stoi
         self.__dict__.update(state)
 
+    def items(self):
+        return self.stoi.items()
+
     def extend(self, tokens):
         self.itos.extend(sorted(set(tokens).difference(self.stoi)))
         self.stoi.update({token: i for i, token in enumerate(self.itos)})
