> cat a.py
# foo
print(1)

> python -m tokenize a.py
0,0-0,0:            ENCODING       'utf-8'        
1,0-1,5:            COMMENT        '# foo'        
1,5-1,6:            NL             '\n'           
2,0-2,5:            NAME           'print'        
2,5-2,6:            OP             '('            
2,6-2,7:            NUMBER         '1'            
2,7-2,8:            OP             ')'            
2,8-2,9:            NEWLINE        ''             
3,0-3,0:            ENDMARKER      ''

    /* Skip comment, unless it's a type comment */
    if (c == '#') {
        const char *prefix, *p, *type_start;
        int current_starting_col_offset;

        while (c != EOF && c != '\n') {
            c = tok_nextc(tok);
        }

        if (tok->type_comments) {
            p = tok->start;
            current_starting_col_offset = tok->starting_col_offset;
            prefix = type_comment_prefix;
            while (*prefix && p < tok->cur) {
                if (*prefix == ' ') {
                    while (*p == ' ' || *p == '\t') {
                        p++;
                        current_starting_col_offset++;
                    }
                } else if (*prefix == *p) {
                    p++;
                    current_starting_col_offset++;
                } else {
                    break;
                }

                prefix++;
            }

It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators. ...