We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent b8458ad commit 201db80Copy full SHA for 201db80
src/transformers/tokenization_utils_base.py
@@ -3163,11 +3163,13 @@ def get_special_tokens_mask(
3163
@staticmethod
3164
def clean_up_tokenization(out_string: str) -> str:
3165
"""
3166
- clean up a list of simple english tokenization artifacts like spaces before punctuations and abbreviated forms.
+ Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms.
3167
3168
- args: out_string (:obj:`str`): the text to clean up.
+ Args:
3169
+ out_string (:obj:`str`): the text to clean up.
3170
- returns: :obj:`str`: the cleaned-up string.
3171
+ Returns:
3172
+ :obj:`str`: the cleaned-up string.
3173
3174
out_string = (
3175
out_string.replace(" .", ".")
0 commit comments