Skip to content

Commit 3fc5573

Browse files
kornykorny
committed
Merge branch 'master' into ruby-skip
2 parents cc3185f + 718c0ac commit 3fc5573

File tree

15 files changed

+100
-67
lines changed

15 files changed

+100
-67
lines changed

‎Changes.textile

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ h2. Changes in 1.1
1010
* New scanner: Taskpaper [#39, thanks to shimomura]
1111
* Diff scanner: Highlight inline changes in multi-line changes [#99]
1212
* JavaScript scanner: Highlight multi-line comments in diff correctly
13+
* Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman]
1314
* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
1415
* HTML scanner displays style tags and attributes now [#145]
1516
* Remove double-click toggle handler from HTML table output
@@ -22,11 +23,17 @@ h2. Changes in 1.1
2223
* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
2324
* New token type @:id@ for CSS/Sass [#27]
2425
* New token type @:done@ for Taskpaper [#39]
25-
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
26+
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman]
27+
* New token type @:unknown@ for Debug scanner
2628
* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
2729
* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
2830
* Fixed @:docstring@ token type style
31+
* Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148]
32+
* @PluginHost@ now works with Strings instead of Symbols internally (to avoid using @#to_sym@)
33+
* The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input)
34+
* The @Raydebug@ scanner highlights unknown token kinds as @:plain@
2935
* @Plugin@ does not warn about fallback when default is defined
36+
* Fixed @HTML@ encoder when output is a StringIO (eg. when using @-HTML@)
3037
* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
3138
* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
3239
* @Debug@ encoder will not warn about errors in the token stream

‎Rakefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,4 @@ else
3434
rd.rdoc_dir = 'doc'
3535
end
3636

37-
end
37+
end

‎lib/coderay/encoders/debug_lint.rb

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ class DebugLint < Debug
1818
register_for :debug_lint
1919

2020
def text_token text, kind
21-
raise Lint::EmptyToken, 'empty token' if text.empty?
21+
raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty?
22+
raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
2223
super
2324
end
2425

‎lib/coderay/encoders/html.rb

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -197,13 +197,15 @@ def finish options
197197
@last_opened = nil
198198
end
199199

200-
@out.extend Output
201-
@out.css = @css
202-
if options[:line_numbers]
203-
Numbering.number! @out, options[:line_numbers], options
200+
if @out.respond_to? :to_str
201+
@out.extend Output
202+
@out.css = @css
203+
if options[:line_numbers]
204+
Numbering.number! @out, options[:line_numbers], options
205+
end
206+
@out.wrap! options[:wrap]
207+
@out.apply_title! options[:title]
204208
end
205-
@out.wrap! options[:wrap]
206-
@out.apply_title! options[:title]
207209

208210
if defined?(@real_out) && @real_out
209211
@real_out << @out
@@ -285,7 +287,7 @@ def style_for_kinds kinds
285287

286288
def make_span_for_kinds method, hint
287289
Hash.new do |h, kinds|
288-
h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
290+
begin
289291
css_class = css_class_for_kinds(kinds)
290292
title = HTML.token_path_to_hint hint, kinds if hint
291293

@@ -297,6 +299,9 @@ def make_span_for_kinds method, hint
297299
"<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
298300
end
299301
end
302+
end.tap do |span|
303+
h.clear if h.size >= 100
304+
h[kinds] = span
300305
end
301306
end
302307
end
@@ -309,8 +314,8 @@ def check_group_nesting name, kind
309314

310315
def break_lines text, style
311316
reopen = ''
312-
@opened.each_with_index do |k, index|
313-
reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
317+
@opened.each_with_index do |kind, index|
318+
reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '<span>')
314319
end
315320
text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
316321
end

‎lib/coderay/encoders/lint.rb

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@ class Lint < Debug
1717

1818
InvalidTokenStream = Class.new StandardError
1919
EmptyToken = Class.new InvalidTokenStream
20+
UnknownTokenKind = Class.new InvalidTokenStream
2021
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
2122

2223
def text_token text, kind
23-
raise EmptyToken, 'empty token' if text.empty?
24+
raise EmptyToken, 'empty token for %p' % [kind] if text.empty?
25+
raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
2426
end
2527

2628
def begin_group kind

‎lib/coderay/helpers/file_type.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def [] filename, read_shebang = false
3838
(TypeFromExt[ext2.downcase] if ext2) ||
3939
TypeFromName[name] ||
4040
TypeFromName[name.downcase]
41-
type ||= shebang(filename) if read_shebang
41+
type ||= type_from_shebang(filename) if read_shebang
4242

4343
type
4444
end
@@ -63,7 +63,7 @@ def fetch filename, default = nil, read_shebang = false
6363

6464
protected
6565

66-
def shebang filename
66+
def type_from_shebang filename
6767
return unless File.exist? filename
6868
File.open filename, 'r' do |f|
6969
if first_line = f.gets

‎lib/coderay/helpers/plugin.rb

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ module PluginHost
3030
# * a file could not be found
3131
# * the requested Plugin is not registered
3232
PluginNotFound = Class.new LoadError
33-
HostNotFound = Class.new LoadError
33+
HostNotFound = Class.new LoadError
3434

3535
PLUGIN_HOSTS = []
3636
PLUGIN_HOSTS_BY_ID = {} # dummy hash
@@ -49,8 +49,8 @@ def load_all
4949
def [] id, *args, &blk
5050
plugin = validate_id(id)
5151
begin
52-
plugin = plugin_hash.[] plugin, *args, &blk
53-
end while plugin.is_a? Symbol
52+
plugin = plugin_hash.[](plugin, *args, &blk)
53+
end while plugin.is_a? String
5454
plugin
5555
end
5656

@@ -95,7 +95,7 @@ def plugin_path *args
9595
def map hash
9696
for from, to in hash
9797
from = validate_id from
98-
to = validate_id to
98+
to = validate_id to
9999
plugin_hash[from] = to unless plugin_hash.has_key? from
100100
end
101101
end
@@ -197,22 +197,22 @@ def path_to plugin_id
197197
File.join plugin_path, "#{plugin_id}.rb"
198198
end
199199

200-
# Converts +id+ to a Symbol if it is a String,
201-
# or returns +id+ if it already is a Symbol.
200+
# Converts +id+ to a valid plugin ID String, or returns +nil+.
202201
#
203202
# Raises +ArgumentError+ for all other objects, or if the
204203
# given String includes non-alphanumeric characters (\W).
205204
def validate_id id
206-
if id.is_a? Symbol or id.nil?
207-
id
208-
elsif id.is_a? String
205+
case id
206+
when Symbol
207+
id.to_s
208+
when String
209209
if id[/\w+/] == id
210-
id.downcase.to_sym
210+
id.downcase
211211
else
212212
raise ArgumentError, "Invalid id given: #{id}"
213213
end
214214
else
215-
raise ArgumentError, "String or Symbol expected, but #{id.class} given."
215+
raise ArgumentError, "Symbol or String expected, but #{id.class} given."
216216
end
217217
end
218218

‎lib/coderay/scanners/debug.rb

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
1+
require 'set'
2+
13
module CodeRay
24
module Scanners
35

46
# = Debug Scanner
57
#
6-
# Interprets the output of the Encoders::Debug encoder.
8+
# Interprets the output of the Encoders::Debug encoder (basically the inverse function).
79
class Debug < Scanner
810

911
register_for :debug
1012
title 'CodeRay Token Dump Import'
1113

1214
protected
1315

16+
def setup
17+
super
18+
@known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
19+
end
20+
1421
def scan_tokens encoder, options
1522

1623
opened_tokens = []
@@ -21,16 +28,19 @@ def scan_tokens encoder, options
2128
encoder.text_token match, :space
2229

2330
elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
24-
kind = self[1].to_sym
25-
match = self[2].gsub(/\\(.)/m, '\1')
26-
unless TokenKinds.has_key? kind
27-
kind = :error
28-
match = matched
31+
if @known_token_kinds.include? self[1]
32+
encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
33+
else
34+
encoder.text_token matched, :unknown
2935
end
30-
encoder.text_token match, kind
3136

3237
elsif match = scan(/ (\w+) ([<\[]) /x)
33-
kind = self[1].to_sym
38+
if @known_token_kinds.include? self[1]
39+
kind = self[1].to_sym
40+
else
41+
kind = :unknown
42+
end
43+
3444
opened_tokens << kind
3545
case self[2]
3646
when '<'

‎lib/coderay/scanners/raydebug.rb

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
require 'set'
2+
13
module CodeRay
24
module Scanners
35

4-
# = Debug Scanner
6+
# = Raydebug Scanner
57
#
6-
# Parses the output of the Encoders::Debug encoder.
8+
# Highlights the output of the Encoders::Debug encoder.
79
class Raydebug < Scanner
810

911
register_for :raydebug
@@ -12,6 +14,11 @@ class Raydebug < Scanner
1214

1315
protected
1416

17+
def setup
18+
super
19+
@known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
20+
end
21+
1522
def scan_tokens encoder, options
1623

1724
opened_tokens = []
@@ -26,20 +33,22 @@ def scan_tokens encoder, options
2633
encoder.text_token kind, :class
2734
encoder.text_token '(', :operator
2835
match = self[2]
29-
encoder.text_token match, kind.to_sym unless match.empty?
36+
unless match.empty?
37+
if @known_token_kinds.include? kind
38+
encoder.text_token match, kind.to_sym
39+
else
40+
encoder.text_token match, :plain
41+
end
42+
end
3043
encoder.text_token match, :operator if match = scan(/\)/)
3144

3245
elsif match = scan(/ (\w+) ([<\[]) /x)
33-
kind = self[1]
34-
case self[2]
35-
when '<'
36-
encoder.text_token kind, :class
37-
when '['
38-
encoder.text_token kind, :class
46+
encoder.text_token self[1], :class
47+
if @known_token_kinds.include? self[1]
48+
kind = self[1].to_sym
3949
else
40-
raise 'CodeRay bug: This case should not be reached.'
50+
kind = :unknown
4151
end
42-
kind = kind.to_sym
4352
opened_tokens << kind
4453
encoder.begin_group kind
4554
encoder.text_token self[2], :operator

‎lib/coderay/scanners/ruby/patterns.rb

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,13 +157,16 @@ module Ruby::Patterns # :nodoc: all
157157
yield
158158
])
159159

160-
FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
160+
FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
161161
FANCY_STRING_KIND = Hash.new(:string).merge({
162+
'i' => :symbol,
163+
'I' => :symbol,
162164
'r' => :regexp,
163165
's' => :symbol,
164166
'x' => :shell,
165167
})
166168
FANCY_STRING_INTERPRETED = Hash.new(true).merge({
169+
'i' => false,
167170
'q' => false,
168171
's' => false,
169172
'w' => false,

‎lib/coderay/scanners/ruby/string_state.rb

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
1616

1717
STRING_PATTERN = Hash.new do |h, k|
1818
delim, interpreted = *k
19-
# delim = delim.dup # workaround for old Ruby
2019
delim_pattern = Regexp.escape(delim)
2120
if closing_paren = CLOSING_PAREN[delim]
2221
delim_pattern << Regexp.escape(closing_paren)
@@ -29,12 +28,13 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
2928
# '| [|?*+(){}\[\].^$]'
3029
# end
3130

32-
h[k] =
33-
if interpreted && delim != '#'
34-
/ (?= [#{delim_pattern}] | \# [{$@] ) /mx
35-
else
36-
/ (?= [#{delim_pattern}] ) /mx
37-
end
31+
if interpreted && delim != '#'
32+
/ (?= [#{delim_pattern}] | \# [{$@] ) /mx
33+
else
34+
/ (?= [#{delim_pattern}] ) /mx
35+
end.tap do |pattern|
36+
h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256
37+
end
3838
end
3939

4040
def initialize kind, interpreted, delim, heredoc = false

‎lib/coderay/styles/alpha.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ class Alpha < Style
125125
.string .modifier { color: #E40 }
126126
.symbol { color:#A60 }
127127
.symbol .content { color:#A60 }
128-
.symbol .delimiter { color:#630 }
128+
.symbol .delimiter { color:#740 }
129129
.tag { color:#070; font-weight:bold }
130130
.type { color:#339; font-weight:bold }
131131
.value { color: #088 }

‎lib/coderay/token_kinds.rb

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
module CodeRay
22

33
# A Hash of all known token kinds and their associated CSS classes.
4-
TokenKinds = Hash.new do |h, k|
5-
warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
6-
false
7-
end
4+
TokenKinds = Hash.new(false)
85

96
# speedup
107
TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
@@ -83,5 +80,6 @@ module CodeRay
8380
:plain => false # almost all scanners
8481
)
8582

86-
TokenKinds[:method] = TokenKinds[:function]
83+
TokenKinds[:method] = TokenKinds[:function]
84+
TokenKinds[:unknown] = TokenKinds[:plain]
8785
end

‎test/functional/basic.rb

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,7 @@ def test_token_kinds
164164
end
165165
end
166166
assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
167-
assert_warning 'Undefined Token kind: :shibboleet' do
168-
assert_equal false, CodeRay::TokenKinds[:shibboleet]
169-
end
167+
assert_equal false, CodeRay::TokenKinds[:shibboleet]
170168
end
171169

172170
class Milk < CodeRay::Encoders::Encoder

0 commit comments

Comments
 (0)