@@ -59,13 +59,13 @@ def initialize(*args)
59
59
@shell = false
60
60
end
61
61
62
- def scan_tokens tokens , options
62
+ def scan_tokens encoder , options
63
63
64
64
until eos?
65
65
kind = match = nil
66
66
67
67
if match = scan ( /\n / )
68
- tokens << [ match , :end_line ]
68
+ encoder . text_token ( match , :end_line )
69
69
next
70
70
end
71
71
@@ -85,17 +85,17 @@ def scan_tokens tokens, options
85
85
elsif match = scan ( /"/ )
86
86
@state = :quote
87
87
@quote = match
88
- tokens . begin_group :string
89
- tokens << [ match , :delimiter ]
88
+ encoder . begin_group :string
89
+ encoder . text_token ( match , :delimiter )
90
90
next
91
91
elsif match = scan ( /`/ )
92
92
if @shell
93
- tokens . end_group :shell
93
+ encoder . end_group :shell
94
94
else
95
- tokens . begin_group :shell
95
+ encoder . begin_group :shell
96
96
end
97
97
@shell = ( not @shell )
98
- tokens << [ match , :delimiter ]
98
+ encoder . text_token ( match , :delimiter )
99
99
next
100
100
elsif match = scan ( /'[^']*'/ )
101
101
kind = :string
@@ -127,8 +127,8 @@ def scan_tokens tokens, options
127
127
if str . to_s . strip . empty?
128
128
kind = IDENT_KIND [ pre ]
129
129
kind = :instance_variable if kind == :ident
130
- tokens << [ pre , kind ]
131
- tokens << [ op , :operator ]
130
+ encoder . text_token ( pre , kind )
131
+ encoder . text_token ( op , :operator )
132
132
next
133
133
end
134
134
elsif match = scan ( /[A-Za-z_]+\[ [A-Za-z_\d ]+\] / )
@@ -141,18 +141,18 @@ def scan_tokens tokens, options
141
141
kind = :instance_variable if kind == :ident
142
142
elsif match = scan ( /read \S +/ )
143
143
match =~ /read(\s +)(\S +)/
144
- tokens << [ 'read' , :method ]
145
- tokens << [ $1, :space ]
146
- tokens << [ $2, :instance_variable ]
144
+ encoder . text_token ( 'read' , :method )
145
+ encoder . text_token ( $1, :space )
146
+ encoder . text_token ( $2, :instance_variable )
147
147
next
148
148
elsif match = scan ( /[\! \: \[ \] \{ \} ]/ )
149
149
kind = :reserved
150
150
elsif match = scan ( / [A-Za-z_][A-Za-z_\d ]*;? /x )
151
151
match =~ /([^;]+);?/
152
152
kind = IDENT_KIND [ $1]
153
153
if match [ /([^;]+);$/ ]
154
- tokens << [ $1, kind ]
155
- tokens << [ ';' , :delimiter ]
154
+ encoder . text_token ( $1, kind )
155
+ encoder . text_token ( ';' , :delimiter )
156
156
next
157
157
end
158
158
elsif match = scan ( /(?: = | - | \+ | \{ | \} | \( | \) | && | \| \| | ;; | ! )/ox )
@@ -169,8 +169,8 @@ def scan_tokens tokens, options
169
169
if ( match = scan ( /\\ .?/ ) )
170
170
kind = :content
171
171
elsif match = scan ( /#{ @quote } / )
172
- tokens << [ match , :delimiter ]
173
- tokens . end_group :string
172
+ encoder . text_token ( match , :delimiter )
173
+ encoder . end_group :string
174
174
@quote = nil
175
175
@state = :initial
176
176
next
@@ -193,10 +193,10 @@ def scan_tokens tokens, options
193
193
end
194
194
195
195
match ||= matched
196
- tokens << [ match , kind ]
196
+ encoder . text_token ( match , kind )
197
197
end
198
198
199
- tokens
199
+ encoder
200
200
end
201
201
202
202
0 commit comments