Skip to content

Commit 5b1a49f

Browse files
kornykorny
committed
Merge branch 'master' into lua-scanner
Conflicts: lib/coderay/styles/alpha.rb lib/coderay/token_kinds.rb
2 parents 69246fc + 2abfc49 commit 5b1a49f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+723
-2922
lines changed

‎Changes.textile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ h2. Changes in 1.1
1212
* Remove double-click toggle handler from HTML table output
1313
* Fixes to CSS scanner (floats, pseudoclasses)
1414
* CSS scanner uses @:id@ and @:tag@ now [#27]
15+
* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
1516
* Add .xaml file type [#121, thanks to Kozman Bálint]
1617
* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
1718
* New token type @:id@ for CSS/Sass [#27]
@@ -20,6 +21,7 @@ h2. Changes in 1.1
2021
* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
2122
* Fixed @:docstring@ token type style
2223
* @Plugin@ does not warn about fallback when default is defined
24+
* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
2325

2426
h2. Changes in 1.0.9
2527

‎bench/bench.rb

Lines changed: 14 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,10 @@
1414
lang = ARGV.fetch(0) do
1515
puts <<-HELP
1616
Usage:
17-
ruby bench.rb (c|ruby|dump) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
17+
ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
1818
1919
SIZE defaults to 100 kB (= 100,000 bytes).
2020
SIZE = 0 means the whole input.
21-
SIZE is ignored when dump is input.
2221
2322
-p generates a profile (slow! use with SIZE = 1)
2423
-o shows the output
@@ -48,10 +47,6 @@
4847
end
4948
end
5049

51-
$dump_input = lang == 'dump'
52-
$dump_output = format == 'dump'
53-
require 'coderay/helpers/gzip_simple.rb' if $dump_input
54-
5550
def here fn = nil
5651
return MYDIR unless fn
5752
File.join here, fn
@@ -66,59 +61,38 @@ def here fn = nil
6661

6762
data = nil
6863
File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
69-
if $dump_input
70-
@size = CodeRay::Tokens.load(data).text.size
71-
else
72-
raise 'Example file is empty.' if data.empty?
73-
unless @size.zero?
74-
data += data until data.size >= @size
75-
data = data[0, @size]
76-
end
77-
@size = data.size
64+
raise 'Example file is empty.' if data.empty?
65+
unless @size.zero?
66+
data += data until data.size >= @size
67+
data = data[0, @size]
7868
end
79-
69+
@size = data.size
70+
8071
options = {
8172
:tab_width => 2,
8273
# :line_numbers => :inline,
8374
:css => $style ? :style : :class,
8475
}
85-
$hl = CodeRay.encoder(format, options) unless $dump_output
76+
$hl = CodeRay.encoder(format, options)
8677
time = bm.report('CodeRay') do
8778
if $stream || true
88-
if $dump_input
89-
raise 'Can\'t stream dump.'
90-
elsif $dump_output
91-
raise 'Can\'t dump stream.'
92-
end
9379
$o = $hl.encode(data, lang, options)
9480
else
95-
if $dump_input
96-
tokens = CodeRay::Tokens.load data
97-
else
98-
tokens = CodeRay.scan(data, lang)
99-
end
81+
tokens = CodeRay.scan(data, lang)
10082
tokens.optimize! if $optimize
101-
if $dump_output
102-
$o = tokens.optimize.dump
103-
else
104-
$o = tokens.encode($hl)
105-
end
83+
$o = tokens.encode($hl)
10684
end
10785
end
108-
$file_created = here('test.' +
109-
($dump_output ? 'dump' : $hl.file_extension))
86+
$file_created = here('test.' + $hl.file_extension)
11087
File.open($file_created, 'wb') do |f|
11188
# f.write $o
11289
end
113-
Dir.chdir(here) do
114-
FileUtils.copy 'test.dump', 'example.dump' if $dump_output
115-
end
116-
90+
11791
time_real = time.real
118-
92+
11993
puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
12094
puts $o if ARGV.include? '-o'
121-
95+
12296
end
12397
end
12498
puts "Files created: #$file_created"

‎bin/coderay

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,9 @@ when 'highlight', nil
155155
puts boom.message
156156
end
157157
# puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
158-
rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE
159-
# this is sometimes raised by pagers; ignore [TODO: wtf?]
158+
rescue CodeRay::Scanners::Scanner::ScanError
159+
# this is sometimes raised by pagers; ignore
160+
# FIXME: rescue Errno::EPIPE
160161
ensure
161162
file.close if output_file
162163
end

‎lib/coderay.rb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ class << self
166166
#
167167
# See also demo/demo_simple.
168168
def scan code, lang, options = {}, &block
169-
# FIXME: return a proxy for direct-stream encoding
170169
TokensProxy.new code, lang, options, block
171170
end
172171

‎lib/coderay/encoders/debug.rb

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ module Encoders
99
#
1010
# You cannot fully restore the tokens information from the
1111
# output, because consecutive :space tokens are merged.
12-
# Use Tokens#dump for caching purposes.
1312
#
1413
# See also: Scanners::Debug
1514
class Debug < Encoder
@@ -18,38 +17,26 @@ class Debug < Encoder
1817

1918
FILE_EXTENSION = 'raydebug'
2019

21-
def initialize options = {}
22-
super
23-
@opened = []
24-
end
25-
2620
def text_token text, kind
27-
raise 'empty token' if $CODERAY_DEBUG && text.empty?
2821
if kind == :space
2922
@out << text
3023
else
31-
# TODO: Escape (
32-
text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/)
33-
@out << kind.to_s << '(' << text << ')'
24+
text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
25+
text = text.gsub(')', '\\\\)') if text.index(')')
26+
@out << "#{kind}(#{text})"
3427
end
3528
end
3629

3730
def begin_group kind
38-
@opened << kind
39-
@out << kind.to_s << '<'
31+
@out << "#{kind}<"
4032
end
4133

4234
def end_group kind
43-
if @opened.last != kind
44-
puts @out
45-
raise "we are inside #{@opened.inspect}, not #{kind}"
46-
end
47-
@opened.pop
4835
@out << '>'
4936
end
5037

5138
def begin_line kind
52-
@out << kind.to_s << '['
39+
@out << "#{kind}["
5340
end
5441

5542
def end_line kind

‎lib/coderay/encoders/debug_lint.rb

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
module CodeRay
2+
module Encoders
3+
4+
# = Debug Lint Encoder
5+
#
6+
# Debug encoder with additional checks for:
7+
#
8+
# - empty tokens
9+
# - incorrect nesting
10+
#
11+
# It will raise an InvalidTokenStream exception when any of the above occurs.
12+
#
13+
# See also: Encoders::Debug
14+
class DebugLint < Debug
15+
16+
register_for :debug_lint
17+
18+
InvalidTokenStream = Class.new StandardError
19+
EmptyToken = Class.new InvalidTokenStream
20+
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
21+
22+
def initialize options = {}
23+
super
24+
@opened = []
25+
end
26+
27+
def text_token text, kind
28+
raise EmptyToken, 'empty token' if text.empty?
29+
super
30+
end
31+
32+
def begin_group kind
33+
@opened << kind
34+
super
35+
end
36+
37+
def end_group kind
38+
raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
39+
super
40+
end
41+
42+
def begin_line kind
43+
@opened << kind
44+
super
45+
end
46+
47+
def end_line kind
48+
raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
49+
super
50+
end
51+
52+
end
53+
54+
end
55+
end

‎lib/coderay/encoders/html/numbering.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def self.number! output, mode = :table, options = {}
2626
"<a href=\"##{anchor}\" name=\"#{anchor}\">#{line}</a>"
2727
end
2828
else
29-
proc { |line| line.to_s } # :to_s.to_proc in Ruby 1.8.7+
29+
:to_s.to_proc
3030
end
3131

3232
bold_every = options[:bold_every]
@@ -75,7 +75,7 @@ def self.number! output, mode = :table, options = {}
7575
line_number = start
7676
output.gsub!(/^.*$\n?/) do |line|
7777
line_number_text = bolding.call line_number
78-
indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
78+
indent = ' ' * (max_width - line_number.to_s.size)
7979
line_number += 1
8080
"<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}"
8181
end

‎lib/coderay/encoders/statistic.rb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ def text_token text, kind
6767
@type_stats['TOTAL'].count += 1
6868
end
6969

70-
# TODO Hierarchy handling
7170
def begin_group kind
7271
block_token ':begin_group', kind
7372
end

0 commit comments

Comments
 (0)