@@ -62,30 +62,31 @@ def scan_tokens encoder, options
62
62
if match = scan ( /[ \t \f \v ]+/ )
63
63
encoder . text_token match , :space
64
64
65
- elsif match = scan ( /\n / )
65
+ elsif skip ( /\n / )
66
66
if heredocs
67
67
unscan # heredoc scanning needs \n at start
68
68
state = heredocs . shift
69
69
encoder . begin_group state . type
70
70
heredocs = nil if heredocs . empty?
71
71
else
72
72
state = :initial if state == :undef_comma_expected
73
- encoder . text_token match , :space
73
+ encoder . text_token " \n " , :space
74
74
value_expected = true
75
75
end
76
76
77
77
elsif match = scan ( bol? ? / \# (!)?.* | #{ patterns ::RUBYDOC_OR_DATA } /ox : /\# .*/ )
78
78
encoder . text_token match , self [ 1 ] ? :doctype : :comment
79
79
80
- elsif match = scan ( /\\ \n / )
80
+ elsif skip ( /\\ \n / )
81
81
if heredocs
82
82
unscan # heredoc scanning needs \n at start
83
- encoder . text_token scan ( /\\ / ) , :space
83
+ skip ( /\\ / )
84
+ encoder . text_token '\\' , :space
84
85
state = heredocs . shift
85
86
encoder . begin_group state . type
86
87
heredocs = nil if heredocs . empty?
87
88
else
88
- encoder . text_token match , :space
89
+ encoder . text_token " \\ \n " , :space
89
90
end
90
91
91
92
elsif state == :initial
@@ -96,7 +97,7 @@ def scan_tokens encoder, options
96
97
/#{ patterns ::METHOD_NAME } /o )
97
98
98
99
kind = patterns ::IDENT_KIND [ match ]
99
- if value_expected != :colon_expected && scan ( /:(?!:)/ )
100
+ if value_expected != :colon_expected && skip ( /:(?!:)/ )
100
101
value_expected = true
101
102
encoder . text_token match , :key
102
103
encoder . text_token ':' , :operator
@@ -181,9 +182,9 @@ def scan_tokens encoder, options
181
182
value_expected = false
182
183
encoder . text_token match , :instance_variable
183
184
184
- elsif value_expected && match = scan ( /\/ / )
185
+ elsif value_expected && skip ( /\/ / )
185
186
encoder . begin_group :regexp
186
- encoder . text_token match , :delimiter
187
+ encoder . text_token '/' , :delimiter
187
188
state = self . class ::StringState . new :regexp , true , '/'
188
189
189
190
elsif match = scan ( value_expected ? /[-+]?#{ patterns ::NUMERIC } /o : /#{ patterns ::NUMERIC } /o )
@@ -225,10 +226,10 @@ def scan_tokens encoder, options
225
226
value_expected = match == '?' ? :colon_expected : true
226
227
encoder . text_token match , :operator
227
228
228
- elsif match = scan ( /`/ )
229
+ elsif skip ( /`/ )
229
230
encoder . begin_group :shell
230
- encoder . text_token match , :delimiter
231
- state = self . class ::StringState . new :shell , true , match
231
+ encoder . text_token '`' , :delimiter
232
+ state = self . class ::StringState . new :shell , true , '`'
232
233
233
234
elsif match = scan ( unicode ? /#{ patterns ::GLOBAL_VARIABLE } /uo :
234
235
/#{ patterns ::GLOBAL_VARIABLE } /o )
@@ -240,8 +241,8 @@ def scan_tokens encoder, options
240
241
encoder . text_token match , :class_variable
241
242
value_expected = false
242
243
243
- elsif match = scan ( /\\ \z / )
244
- encoder . text_token match , :space
244
+ elsif skip ( /\\ \z / )
245
+ encoder . text_token '\\' , :space
245
246
246
247
else
247
248
if method_call_expected
@@ -293,8 +294,8 @@ def scan_tokens encoder, options
293
294
end
294
295
295
296
elsif state == :module_expected
296
- if match = scan ( /<</ )
297
- encoder . text_token match , :operator
297
+ if skip ( /<</ )
298
+ encoder . text_token '<<' , :operator
298
299
else
299
300
state = :initial
300
301
if match = scan ( unicode ? / (?:#{ patterns ::IDENT } ::)* #{ patterns ::IDENT } /oux :
@@ -325,8 +326,8 @@ def scan_tokens encoder, options
325
326
end
326
327
327
328
elsif state == :undef_comma_expected
328
- if match = scan ( /,/ )
329
- encoder . text_token match , :operator
329
+ if skip ( /,/ )
330
+ encoder . text_token ',' , :operator
330
331
state = :undef_expected
331
332
else
332
333
state = :initial
0 commit comments