from tokenize import generate_tokens
from io import StringIO
import pprint
pprint.pprint(list(generate_tokens(StringIO('a').readline)))

[TokenInfo(type=1 (NAME), string='a', start=(1, 0), end=(1, 1), line='a\n'),
 TokenInfo(type=4 (NEWLINE), string='', start=(1, 1), end=(1, 2), line='a'),
 TokenInfo(type=0 (ENDMARKER), string='', start=(2, 0), end=(2, 0), line='')]