Skip to content

Commit 0013b64

Browse files
kornykorny
committed
Merge branch 'master' into go-scanner
Conflicts: lib/coderay/helpers/file_type.rb
2 parents addcbd4 + 64ca2ae commit 0013b64

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+1150
-2979
lines changed

‎.travis.yml

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ branches:
1414
- master
1515
matrix:
1616
allow_failures:
17+
- rvm: ruby-head
18+
- rvm: jruby-head
1719
- rvm: rbx-18mode
1820
- rvm: rbx-19mode
1921
script: "rake test" # test:scanners"

‎Changes.textile

+11-1
Original file line numberDiff line numberDiff line change
@@ -4,22 +4,32 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release
44

55
h2. Changes in 1.1
66

7+
* New scanner: Lua [#21, #22, thanks to Quintus]
78
* New scanner: Sass [#93]
89
* New scanner: Taskpaper [#39, thanks to shimomura]
910
* Diff scanner: Highlight inline changes in multi-line changes [#99]
1011
* JavaScript scanner: Highlight multi-line comments in diff correctly
1112
* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
13+
* HTML scanner displays style tags and attributes now [#145]
1214
* Remove double-click toggle handler from HTML table output
1315
* Fixes to CSS scanner (floats, pseudoclasses)
16+
* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144]
17+
* Added @:keep_state@ functionality to more scanners [#116]
1418
* CSS scanner uses @:id@ and @:tag@ now [#27]
19+
* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
1520
* Add .xaml file type [#121, thanks to Kozman Bálint]
1621
* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
1722
* New token type @:id@ for CSS/Sass [#27]
1823
* New token type @:done@ for Taskpaper [#39]
24+
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
1925
* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
20-
* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
26+
* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
2127
* Fixed @:docstring@ token type style
2228
* @Plugin@ does not warn about fallback when default is defined
29+
* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
30+
* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
31+
* @Debug@ encoder will not warn about errors in the token stream
32+
* New @DebugLint@ encoder that checks for empty tokens and correct nesting
2333

2434
h2. Changes in 1.0.9
2535

‎bench/bench.rb

+14-40
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,10 @@
1414
lang = ARGV.fetch(0) do
1515
puts <<-HELP
1616
Usage:
17-
ruby bench.rb (c|ruby|dump) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
17+
ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
1818
1919
SIZE defaults to 100 kB (= 100,000 bytes).
2020
SIZE = 0 means the whole input.
21-
SIZE is ignored when dump is input.
2221
2322
-p generates a profile (slow! use with SIZE = 1)
2423
-o shows the output
@@ -48,10 +47,6 @@
4847
end
4948
end
5049

51-
$dump_input = lang == 'dump'
52-
$dump_output = format == 'dump'
53-
require 'coderay/helpers/gzip_simple.rb' if $dump_input
54-
5550
def here fn = nil
5651
return MYDIR unless fn
5752
File.join here, fn
@@ -66,59 +61,38 @@ def here fn = nil
6661

6762
data = nil
6863
File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
69-
if $dump_input
70-
@size = CodeRay::Tokens.load(data).text.size
71-
else
72-
raise 'Example file is empty.' if data.empty?
73-
unless @size.zero?
74-
data += data until data.size >= @size
75-
data = data[0, @size]
76-
end
77-
@size = data.size
64+
raise 'Example file is empty.' if data.empty?
65+
unless @size.zero?
66+
data += data until data.size >= @size
67+
data = data[0, @size]
7868
end
79-
69+
@size = data.size
70+
8071
options = {
8172
:tab_width => 2,
8273
# :line_numbers => :inline,
8374
:css => $style ? :style : :class,
8475
}
85-
$hl = CodeRay.encoder(format, options) unless $dump_output
76+
$hl = CodeRay.encoder(format, options)
8677
time = bm.report('CodeRay') do
8778
if $stream || true
88-
if $dump_input
89-
raise 'Can\'t stream dump.'
90-
elsif $dump_output
91-
raise 'Can\'t dump stream.'
92-
end
9379
$o = $hl.encode(data, lang, options)
9480
else
95-
if $dump_input
96-
tokens = CodeRay::Tokens.load data
97-
else
98-
tokens = CodeRay.scan(data, lang)
99-
end
81+
tokens = CodeRay.scan(data, lang)
10082
tokens.optimize! if $optimize
101-
if $dump_output
102-
$o = tokens.optimize.dump
103-
else
104-
$o = tokens.encode($hl)
105-
end
83+
$o = tokens.encode($hl)
10684
end
10785
end
108-
$file_created = here('test.' +
109-
($dump_output ? 'dump' : $hl.file_extension))
86+
$file_created = here('test.' + $hl.file_extension)
11087
File.open($file_created, 'wb') do |f|
11188
# f.write $o
11289
end
113-
Dir.chdir(here) do
114-
FileUtils.copy 'test.dump', 'example.dump' if $dump_output
115-
end
116-
90+
11791
time_real = time.real
118-
92+
11993
puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
12094
puts $o if ARGV.include? '-o'
121-
95+
12296
end
12397
end
12498
puts "Files created: #$file_created"

‎bin/coderay

+3-2
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,9 @@ when 'highlight', nil
155155
puts boom.message
156156
end
157157
# puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
158-
rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE
159-
# this is sometimes raised by pagers; ignore [TODO: wtf?]
158+
rescue CodeRay::Scanners::Scanner::ScanError
159+
# this is sometimes raised by pagers; ignore
160+
# FIXME: rescue Errno::EPIPE
160161
ensure
161162
file.close if output_file
162163
end

‎lib/coderay.rb

+1-2
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ def self.coderay_path *path
134134
File.join CODERAY_PATH, *path
135135
end
136136

137-
require coderay_path('version')
137+
require 'coderay/version'
138138

139139
# helpers
140140
autoload :FileType, coderay_path('helpers', 'file_type')
@@ -166,7 +166,6 @@ class << self
166166
#
167167
# See also demo/demo_simple.
168168
def scan code, lang, options = {}, &block
169-
# FIXME: return a proxy for direct-stream encoding
170169
TokensProxy.new code, lang, options, block
171170
end
172171

‎lib/coderay/encoders/debug.rb

+5-18
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ module Encoders
99
#
1010
# You cannot fully restore the tokens information from the
1111
# output, because consecutive :space tokens are merged.
12-
# Use Tokens#dump for caching purposes.
1312
#
1413
# See also: Scanners::Debug
1514
class Debug < Encoder
@@ -18,38 +17,26 @@ class Debug < Encoder
1817

1918
FILE_EXTENSION = 'raydebug'
2019

21-
def initialize options = {}
22-
super
23-
@opened = []
24-
end
25-
2620
def text_token text, kind
27-
raise 'empty token' if $CODERAY_DEBUG && text.empty?
2821
if kind == :space
2922
@out << text
3023
else
31-
# TODO: Escape (
32-
text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/)
33-
@out << kind.to_s << '(' << text << ')'
24+
text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
25+
text = text.gsub(')', '\\\\)') if text.index(')')
26+
@out << "#{kind}(#{text})"
3427
end
3528
end
3629

3730
def begin_group kind
38-
@opened << kind
39-
@out << kind.to_s << '<'
31+
@out << "#{kind}<"
4032
end
4133

4234
def end_group kind
43-
if @opened.last != kind
44-
puts @out
45-
raise "we are inside #{@opened.inspect}, not #{kind}"
46-
end
47-
@opened.pop
4835
@out << '>'
4936
end
5037

5138
def begin_line kind
52-
@out << kind.to_s << '['
39+
@out << "#{kind}["
5340
end
5441

5542
def end_line kind

‎lib/coderay/encoders/debug_lint.rb

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
module CodeRay
2+
module Encoders
3+
4+
# = Debug Lint Encoder
5+
#
6+
# Debug encoder with additional checks for:
7+
#
8+
# - empty tokens
9+
# - incorrect nesting
10+
#
11+
# It will raise an InvalidTokenStream exception when any of the above occurs.
12+
#
13+
# See also: Encoders::Debug
14+
class DebugLint < Debug
15+
16+
register_for :debug_lint
17+
18+
InvalidTokenStream = Class.new StandardError
19+
EmptyToken = Class.new InvalidTokenStream
20+
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
21+
22+
def text_token text, kind
23+
raise EmptyToken, 'empty token' if text.empty?
24+
super
25+
end
26+
27+
def begin_group kind
28+
@opened << kind
29+
super
30+
end
31+
32+
def end_group kind
33+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
34+
@opened.pop
35+
super
36+
end
37+
38+
def begin_line kind
39+
@opened << kind
40+
super
41+
end
42+
43+
def end_line kind
44+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
45+
@opened.pop
46+
super
47+
end
48+
49+
protected
50+
51+
def setup options
52+
super
53+
@opened = []
54+
end
55+
56+
def finish options
57+
raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
58+
super
59+
end
60+
61+
end
62+
63+
end
64+
end

‎lib/coderay/encoders/html.rb

-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,6 @@ def setup options
193193

194194
def finish options
195195
unless @opened.empty?
196-
warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
197196
@out << '</span>' while @opened.pop
198197
@last_opened = nil
199198
end

‎lib/coderay/encoders/html/numbering.rb

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def self.number! output, mode = :table, options = {}
2626
"<a href=\"##{anchor}\" name=\"#{anchor}\">#{line}</a>"
2727
end
2828
else
29-
proc { |line| line.to_s } # :to_s.to_proc in Ruby 1.8.7+
29+
:to_s.to_proc
3030
end
3131

3232
bold_every = options[:bold_every]
@@ -75,7 +75,7 @@ def self.number! output, mode = :table, options = {}
7575
line_number = start
7676
output.gsub!(/^.*$\n?/) do |line|
7777
line_number_text = bolding.call line_number
78-
indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
78+
indent = ' ' * (max_width - line_number.to_s.size)
7979
line_number += 1
8080
"<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}"
8181
end

‎lib/coderay/encoders/statistic.rb

-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ def text_token text, kind
6767
@type_stats['TOTAL'].count += 1
6868
end
6969

70-
# TODO Hierarchy handling
7170
def begin_group kind
7271
block_token ':begin_group', kind
7372
end

0 commit comments

Comments
 (0)