Skip to content

Commit 19e48b5

Browse files
kornykorny
committed
Merge branch 'master' into escape-utils
2 parents 4e13ac4 + a31b366 commit 19e48b5

File tree

14 files changed

+99
-66
lines changed

14 files changed

+99
-66
lines changed

‎Changes.textile

+8-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ h2. Changes in 1.1
1010
* New scanner: Taskpaper [#39, thanks to shimomura]
1111
* Diff scanner: Highlight inline changes in multi-line changes [#99]
1212
* JavaScript scanner: Highlight multi-line comments in diff correctly
13+
* Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman]
1314
* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
1415
* HTML scanner displays style tags and attributes now [#145]
1516
* Remove double-click toggle handler from HTML table output
@@ -22,11 +23,17 @@ h2. Changes in 1.1
2223
* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
2324
* New token type @:id@ for CSS/Sass [#27]
2425
* New token type @:done@ for Taskpaper [#39]
25-
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
26+
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman]
27+
* New token type @:unknown@ for Debug scanner
2628
* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
2729
* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
2830
* Fixed @:docstring@ token type style
31+
* Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148]
32+
* @PluginHost@ now works with Strings instead of Symbols internally (to avoid using @#to_sym@)
33+
* The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input)
34+
* The @Raydebug@ scanner highlights unknown token kinds as @:plain@
2935
* @Plugin@ does not warn about fallback when default is defined
36+
* Fixed @HTML@ encoder when output is a StringIO (eg. when using @-HTML@)
3037
* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
3138
* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
3239
* @Debug@ encoder will not warn about errors in the token stream

‎lib/coderay/encoders/debug_lint.rb

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ class DebugLint < Debug
1818
register_for :debug_lint
1919

2020
def text_token text, kind
21-
raise Lint::EmptyToken, 'empty token' if text.empty?
21+
raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty?
22+
raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
2223
super
2324
end
2425

‎lib/coderay/encoders/html.rb

+14-9
Original file line numberDiff line numberDiff line change
@@ -182,13 +182,15 @@ def finish options
182182
@last_opened = nil
183183
end
184184

185-
@out.extend Output
186-
@out.css = @css
187-
if options[:line_numbers]
188-
Numbering.number! @out, options[:line_numbers], options
185+
if @out.respond_to? :to_str
186+
@out.extend Output
187+
@out.css = @css
188+
if options[:line_numbers]
189+
Numbering.number! @out, options[:line_numbers], options
190+
end
191+
@out.wrap! options[:wrap]
192+
@out.apply_title! options[:title]
189193
end
190-
@out.wrap! options[:wrap]
191-
@out.apply_title! options[:title]
192194

193195
if defined?(@real_out) && @real_out
194196
@real_out << @out
@@ -276,7 +278,7 @@ def style_for_kinds kinds
276278

277279
def make_span_for_kinds method, hint
278280
Hash.new do |h, kinds|
279-
h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
281+
begin
280282
css_class = css_class_for_kinds(kinds)
281283
title = HTML.token_path_to_hint hint, kinds if hint
282284

@@ -288,6 +290,9 @@ def make_span_for_kinds method, hint
288290
"<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
289291
end
290292
end
293+
end.tap do |span|
294+
h.clear if h.size >= 100
295+
h[kinds] = span
291296
end
292297
end
293298
end
@@ -300,8 +305,8 @@ def check_group_nesting name, kind
300305

301306
def break_lines text, style
302307
reopen = ''
303-
@opened.each_with_index do |k, index|
304-
reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
308+
@opened.each_with_index do |kind, index|
309+
reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '<span>')
305310
end
306311
text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
307312
end

‎lib/coderay/encoders/lint.rb

+3-1
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@ class Lint < Debug
1717

1818
InvalidTokenStream = Class.new StandardError
1919
EmptyToken = Class.new InvalidTokenStream
20+
UnknownTokenKind = Class.new InvalidTokenStream
2021
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
2122

2223
def text_token text, kind
23-
raise EmptyToken, 'empty token' if text.empty?
24+
raise EmptyToken, 'empty token for %p' % [kind] if text.empty?
25+
raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
2426
end
2527

2628
def begin_group kind

‎lib/coderay/helpers/file_type.rb

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def [] filename, read_shebang = false
3838
(TypeFromExt[ext2.downcase] if ext2) ||
3939
TypeFromName[name] ||
4040
TypeFromName[name.downcase]
41-
type ||= shebang(filename) if read_shebang
41+
type ||= type_from_shebang(filename) if read_shebang
4242

4343
type
4444
end
@@ -63,7 +63,7 @@ def fetch filename, default = nil, read_shebang = false
6363

6464
protected
6565

66-
def shebang filename
66+
def type_from_shebang filename
6767
return unless File.exist? filename
6868
File.open filename, 'r' do |f|
6969
if first_line = f.gets

‎lib/coderay/helpers/plugin.rb

+11-11
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ module PluginHost
3030
# * a file could not be found
3131
# * the requested Plugin is not registered
3232
PluginNotFound = Class.new LoadError
33-
HostNotFound = Class.new LoadError
33+
HostNotFound = Class.new LoadError
3434

3535
PLUGIN_HOSTS = []
3636
PLUGIN_HOSTS_BY_ID = {} # dummy hash
@@ -49,8 +49,8 @@ def load_all
4949
def [] id, *args, &blk
5050
plugin = validate_id(id)
5151
begin
52-
plugin = plugin_hash.[] plugin, *args, &blk
53-
end while plugin.is_a? Symbol
52+
plugin = plugin_hash.[](plugin, *args, &blk)
53+
end while plugin.is_a? String
5454
plugin
5555
end
5656

@@ -95,7 +95,7 @@ def plugin_path *args
9595
def map hash
9696
for from, to in hash
9797
from = validate_id from
98-
to = validate_id to
98+
to = validate_id to
9999
plugin_hash[from] = to unless plugin_hash.has_key? from
100100
end
101101
end
@@ -197,22 +197,22 @@ def path_to plugin_id
197197
File.join plugin_path, "#{plugin_id}.rb"
198198
end
199199

200-
# Converts +id+ to a Symbol if it is a String,
201-
# or returns +id+ if it already is a Symbol.
200+
# Converts +id+ to a valid plugin ID String, or returns +nil+.
202201
#
203202
# Raises +ArgumentError+ for all other objects, or if the
204203
# given String includes non-alphanumeric characters (\W).
205204
def validate_id id
206-
if id.is_a? Symbol or id.nil?
207-
id
208-
elsif id.is_a? String
205+
case id
206+
when Symbol
207+
id.to_s
208+
when String
209209
if id[/\w+/] == id
210-
id.downcase.to_sym
210+
id.downcase
211211
else
212212
raise ArgumentError, "Invalid id given: #{id}"
213213
end
214214
else
215-
raise ArgumentError, "String or Symbol expected, but #{id.class} given."
215+
raise ArgumentError, "Symbol or String expected, but #{id.class} given."
216216
end
217217
end
218218

‎lib/coderay/scanners/debug.rb

+18-8
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
1+
require 'set'
2+
13
module CodeRay
24
module Scanners
35

46
# = Debug Scanner
57
#
6-
# Interprets the output of the Encoders::Debug encoder.
8+
# Interprets the output of the Encoders::Debug encoder (basically the inverse function).
79
class Debug < Scanner
810

911
register_for :debug
1012
title 'CodeRay Token Dump Import'
1113

1214
protected
1315

16+
def setup
17+
super
18+
@known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
19+
end
20+
1421
def scan_tokens encoder, options
1522

1623
opened_tokens = []
@@ -21,16 +28,19 @@ def scan_tokens encoder, options
2128
encoder.text_token match, :space
2229

2330
elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
24-
kind = self[1].to_sym
25-
match = self[2].gsub(/\\(.)/m, '\1')
26-
unless TokenKinds.has_key? kind
27-
kind = :error
28-
match = matched
31+
if @known_token_kinds.include? self[1]
32+
encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
33+
else
34+
encoder.text_token matched, :unknown
2935
end
30-
encoder.text_token match, kind
3136

3237
elsif match = scan(/ (\w+) ([<\[]) /x)
33-
kind = self[1].to_sym
38+
if @known_token_kinds.include? self[1]
39+
kind = self[1].to_sym
40+
else
41+
kind = :unknown
42+
end
43+
3444
opened_tokens << kind
3545
case self[2]
3646
when '<'

‎lib/coderay/scanners/raydebug.rb

+20-11
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
require 'set'
2+
13
module CodeRay
24
module Scanners
35

4-
# = Debug Scanner
6+
# = Raydebug Scanner
57
#
6-
# Parses the output of the Encoders::Debug encoder.
8+
# Highlights the output of the Encoders::Debug encoder.
79
class Raydebug < Scanner
810

911
register_for :raydebug
@@ -12,6 +14,11 @@ class Raydebug < Scanner
1214

1315
protected
1416

17+
def setup
18+
super
19+
@known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
20+
end
21+
1522
def scan_tokens encoder, options
1623

1724
opened_tokens = []
@@ -26,20 +33,22 @@ def scan_tokens encoder, options
2633
encoder.text_token kind, :class
2734
encoder.text_token '(', :operator
2835
match = self[2]
29-
encoder.text_token match, kind.to_sym unless match.empty?
36+
unless match.empty?
37+
if @known_token_kinds.include? kind
38+
encoder.text_token match, kind.to_sym
39+
else
40+
encoder.text_token match, :plain
41+
end
42+
end
3043
encoder.text_token match, :operator if match = scan(/\)/)
3144

3245
elsif match = scan(/ (\w+) ([<\[]) /x)
33-
kind = self[1]
34-
case self[2]
35-
when '<'
36-
encoder.text_token kind, :class
37-
when '['
38-
encoder.text_token kind, :class
46+
encoder.text_token self[1], :class
47+
if @known_token_kinds.include? self[1]
48+
kind = self[1].to_sym
3949
else
40-
raise 'CodeRay bug: This case should not be reached.'
50+
kind = :unknown
4151
end
42-
kind = kind.to_sym
4352
opened_tokens << kind
4453
encoder.begin_group kind
4554
encoder.text_token self[2], :operator

‎lib/coderay/scanners/ruby/patterns.rb

+4-1
Original file line numberDiff line numberDiff line change
@@ -157,13 +157,16 @@ module Ruby::Patterns # :nodoc: all
157157
yield
158158
])
159159

160-
FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
160+
FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
161161
FANCY_STRING_KIND = Hash.new(:string).merge({
162+
'i' => :symbol,
163+
'I' => :symbol,
162164
'r' => :regexp,
163165
's' => :symbol,
164166
'x' => :shell,
165167
})
166168
FANCY_STRING_INTERPRETED = Hash.new(true).merge({
169+
'i' => false,
167170
'q' => false,
168171
's' => false,
169172
'w' => false,

‎lib/coderay/scanners/ruby/string_state.rb

+7-7
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
1616

1717
STRING_PATTERN = Hash.new do |h, k|
1818
delim, interpreted = *k
19-
# delim = delim.dup # workaround for old Ruby
2019
delim_pattern = Regexp.escape(delim)
2120
if closing_paren = CLOSING_PAREN[delim]
2221
delim_pattern << Regexp.escape(closing_paren)
@@ -29,12 +28,13 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
2928
# '| [|?*+(){}\[\].^$]'
3029
# end
3130

32-
h[k] =
33-
if interpreted && delim != '#'
34-
/ (?= [#{delim_pattern}] | \# [{$@] ) /mx
35-
else
36-
/ (?= [#{delim_pattern}] ) /mx
37-
end
31+
if interpreted && delim != '#'
32+
/ (?= [#{delim_pattern}] | \# [{$@] ) /mx
33+
else
34+
/ (?= [#{delim_pattern}] ) /mx
35+
end.tap do |pattern|
36+
h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256
37+
end
3838
end
3939

4040
def initialize kind, interpreted, delim, heredoc = false

‎lib/coderay/styles/alpha.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ class Alpha < Style
125125
.string .modifier { color: #E40 }
126126
.symbol { color:#A60 }
127127
.symbol .content { color:#A60 }
128-
.symbol .delimiter { color:#630 }
128+
.symbol .delimiter { color:#740 }
129129
.tag { color:#070; font-weight:bold }
130130
.type { color:#339; font-weight:bold }
131131
.value { color: #088 }

‎lib/coderay/token_kinds.rb

+3-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
module CodeRay
22

33
# A Hash of all known token kinds and their associated CSS classes.
4-
TokenKinds = Hash.new do |h, k|
5-
warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
6-
false
7-
end
4+
TokenKinds = Hash.new(false)
85

96
# speedup
107
TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
@@ -83,5 +80,6 @@ module CodeRay
8380
:plain => false # almost all scanners
8481
)
8582

86-
TokenKinds[:method] = TokenKinds[:function]
83+
TokenKinds[:method] = TokenKinds[:function]
84+
TokenKinds[:unknown] = TokenKinds[:plain]
8785
end

‎test/functional/basic.rb

+1-3
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,7 @@ def test_token_kinds
164164
end
165165
end
166166
assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
167-
assert_warning 'Undefined Token kind: :shibboleet' do
168-
assert_equal false, CodeRay::TokenKinds[:shibboleet]
169-
end
167+
assert_equal false, CodeRay::TokenKinds[:shibboleet]
170168
end
171169

172170
class Milk < CodeRay::Encoders::Encoder

0 commit comments

Comments
 (0)