CI finally failing?

julien-c · julien-c · commit 9e3059ae9a44 · 2020-11-19T14:12:58.000+01:00
diff --git a/tests/test_tokenization_auto.py b/tests/test_tokenization_auto.py
@@ -20,6 +20,7 @@
     BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
     GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
     AutoTokenizer,
+    BertJapaneseTokenizer,
     BertTokenizer,
     BertTokenizerFast,
     GPT2Tokenizer,
@@ -72,6 +73,11 @@ def test_tokenizer_from_tokenizer_class(self):
         self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
         self.assertEqual(tokenizer.vocab_size, 12)
 
+    def test_tokenizer_bert_japanese(self):
+        EXAMPLE_BERT_JAPANESE_ID = "cl-tohoku/bert-base-japanese"
+        tokenizer = AutoTokenizer.from_pretrained(EXAMPLE_BERT_JAPANESE_ID)
+        self.assertIsInstance(tokenizer, BertJapaneseTokenizer)
+
     @require_tokenizers
     def test_tokenizer_identifier_with_correct_config(self):
         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
