format

SaulLu · SaulLu · commit 37a40df37903 · 2022-01-25T18:20:49.000+01:00
diff --git a/src/transformers/models/bartpho/tokenization_bartpho.py b/src/transformers/models/bartpho/tokenization_bartpho.py
@@ -160,11 +160,9 @@ def __init__(
 
         # Keep order of special tokens for backward compatibility
         self.fairseq_tokens_to_ids = {
-            token: token_id
+            str(token): token_id
             for token_id, token in enumerate(
-                dict.fromkeys(
-                    [str(bos_token), str(pad_token), str(eos_token), str(unk_token), str(sep_token), str(cls_token)]
-                ).keys()
+                dict.fromkeys([bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]).keys()
             )
         }
         with open(monolingual_vocab_file, "r", encoding="utf-8") as f:

Original file line number	Diff line number	Diff line change
`@@ -160,11 +160,9 @@ def __init__(`
`160`	`160`
`161`	`161`	`# Keep order of special tokens for backward compatibility`
`162`	`162`	`self.fairseq_tokens_to_ids = {`
`163`		`- token: token_id`
	`163`	`+ str(token): token_id`
`164`	`164`	`for token_id, token in enumerate(`
`165`		`- dict.fromkeys(`
`166`		`- [str(bos_token), str(pad_token), str(eos_token), str(unk_token), str(sep_token), str(cls_token)]`
`167`		`- ).keys()`
	`165`	`+ dict.fromkeys([bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]).keys()`
`168`	`166`	`)`
`169`	`167`	`}`
`170`	`168`	`with open(monolingual_vocab_file, "r", encoding="utf-8") as f:`