Skip to content

Commit bf395bd

Browse files
kornykorny
committed
Merge branch 'master' into cleanup-output
2 parents ffe0c90 + 7493dcb commit bf395bd

27 files changed

+769
-229
lines changed

‎.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,4 @@ Gemfile.lock
1111
test/executable/source.rb.html
1212
test/executable/source.rb.json
1313
test/scanners
14-
bench/test.div.html
1514
old-stuff

‎.travis.yml

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ branches:
1414
- master
1515
matrix:
1616
allow_failures:
17+
- rvm: ruby-head
18+
- rvm: jruby-head
1719
- rvm: rbx-18mode
1820
- rvm: rbx-19mode
1921
script: "rake test" # test:scanners"

‎Changes.textile

+11-2
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,33 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release
44

55
h2. Changes in 1.1
66

7+
* New scanner: Lua [#21, #22, thanks to Quintus]
78
* New scanner: Sass [#93]
9+
* New scanner: Go [#28, thanks to Eric Guo and Nathan Youngman]
810
* New scanner: Taskpaper [#39, thanks to shimomura]
911
* Diff scanner: Highlight inline changes in multi-line changes [#99]
1012
* JavaScript scanner: Highlight multi-line comments in diff correctly
1113
* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
14+
* HTML scanner displays style tags and attributes now [#145]
1215
* Remove double-click toggle handler from HTML table output
13-
* Fixes to CSS scanner (floats, pseudoclasses)
16+
* Fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143]
17+
* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144]
18+
* Added @:keep_state@ functionality to more scanners [#116]
1419
* CSS scanner uses @:id@ and @:tag@ now [#27]
1520
* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
1621
* Add .xaml file type [#121, thanks to Kozman Bálint]
1722
* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
1823
* New token type @:id@ for CSS/Sass [#27]
1924
* New token type @:done@ for Taskpaper [#39]
25+
* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
2026
* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
21-
* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
27+
* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
2228
* Fixed @:docstring@ token type style
2329
* @Plugin@ does not warn about fallback when default is defined
30+
* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
2431
* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
32+
* @Debug@ encoder will not warn about errors in the token stream
33+
* New @DebugLint@ encoder that checks for empty tokens and correct nesting
2534

2635
h2. Changes in 1.0.9
2736

‎Rakefile

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
require 'bundler/gem_tasks'
2+
13
$:.unshift File.dirname(__FILE__) unless $:.include? '.'
24

35
ROOT = '.'

‎bench/bench.rb

+31-127
Original file line numberDiff line numberDiff line change
@@ -1,142 +1,46 @@
1-
# The most ugly test script I've ever written!
2-
# Shame on me!
3-
4-
require 'pathname'
5-
require 'profile' if ARGV.include? '-p'
6-
7-
MYDIR = File.dirname(__FILE__)
8-
LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s
9-
$:.unshift MYDIR, LIBDIR
1+
require 'benchmark'
2+
$: << File.expand_path('../../lib', __FILE__)
103
require 'coderay'
114

12-
@size = ARGV.fetch(2, 100).to_i * 1000
13-
14-
lang = ARGV.fetch(0) do
15-
puts <<-HELP
16-
Usage:
17-
ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
18-
19-
SIZE defaults to 100 kB (= 100,000 bytes).
20-
SIZE = 0 means the whole input.
21-
22-
-p generates a profile (slow! use with SIZE = 1)
23-
-o shows the output
24-
stream enabled streaming mode
25-
26-
Sorry for the strange interface. I will improve it in the next release.
27-
HELP
5+
if ARGV.include? '-h'
6+
puts DATA.read
287
exit
298
end
309

31-
format = ARGV.fetch(1, 'html').downcase
32-
33-
$stream = ARGV.include? 'stream'
34-
$optimize = ARGV.include? 'opt'
35-
$style = ARGV.include? 'style'
36-
37-
require 'benchmark'
38-
require 'fileutils'
10+
lang = ARGV.fetch(0, 'ruby')
11+
data = nil
12+
File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read }
13+
raise 'Example file is empty.' if data.empty?
3914

40-
if format == 'comp'
41-
format = 'page'
42-
begin
43-
require 'syntax'
44-
require 'syntax/convertors/html.rb'
45-
rescue LoadError
46-
puts 'Syntax no found!! (Try % gem install syntax)'
47-
end
48-
end
15+
format = ARGV.fetch(1, 'html').downcase
16+
encoder = CodeRay.encoder(format)
4917

50-
def here fn = nil
51-
return MYDIR unless fn
52-
File.join here, fn
18+
size = ARGV.fetch(2, 1000).to_i * 1000
19+
unless size.zero?
20+
data += data until data.size >= size
21+
data = data[0, size]
5322
end
23+
size = data.size
24+
puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round]
5425

55-
n = ARGV.find { |a| a[/^N/] }
56-
N = if n then n[/\d+/].to_i else 1 end
57-
$filename = ARGV.include?('strange') ? 'strange' : 'example'
58-
59-
Benchmark.bm(20) do |bm|
60-
N.times do
61-
62-
data = nil
63-
File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
64-
raise 'Example file is empty.' if data.empty?
65-
unless @size.zero?
66-
data += data until data.size >= @size
67-
data = data[0, @size]
68-
end
69-
@size = data.size
70-
71-
options = {
72-
:tab_width => 2,
73-
# :line_numbers => :inline,
74-
:css => $style ? :style : :class,
75-
}
76-
$hl = CodeRay.encoder(format, options)
77-
time = bm.report('CodeRay') do
78-
if $stream || true
79-
$o = $hl.encode(data, lang, options)
80-
else
81-
tokens = CodeRay.scan(data, lang)
82-
tokens.optimize! if $optimize
83-
$o = tokens.encode($hl)
84-
end
85-
end
86-
$file_created = here('test.' + $hl.file_extension)
87-
File.open($file_created, 'wb') do |f|
88-
# f.write $o
89-
end
90-
91-
time_real = time.real
92-
93-
puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
94-
puts $o if ARGV.include? '-o'
95-
96-
end
26+
n = ARGV.fetch(3, 5).to_s[/\d+/].to_i
27+
require 'profile' if ARGV.include? '-p'
28+
n.times do |i|
29+
time = Benchmark.realtime { encoder.encode(data, lang) }
30+
puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0]
9731
end
98-
puts "Files created: #$file_created"
9932

100-
STDIN.gets if ARGV.include? 'wait'
33+
STDIN.gets if ARGV.include? '-w'
10134

10235
__END__
103-
.ruby .normal {}
104-
.ruby .comment { color: #005; font-style: italic; }
105-
.ruby .keyword { color: #A00; font-weight: bold; }
106-
.ruby .method { color: #077; }
107-
.ruby .class { color: #074; }
108-
.ruby .module { color: #050; }
109-
.ruby .punct { color: #447; font-weight: bold; }
110-
.ruby .symbol { color: #099; }
111-
.ruby .string { color: #944; background: #FFE; }
112-
.ruby .char { color: #F07; }
113-
.ruby .ident { color: #004; }
114-
.ruby .constant { color: #07F; }
115-
.ruby .regex { color: #B66; background: #FEF; }
116-
.ruby .number { color: #F99; }
117-
.ruby .attribute { color: #7BB; }
118-
.ruby .global { color: #7FB; }
119-
.ruby .expr { color: #227; }
120-
.ruby .escape { color: #277; }
36+
Usage:
37+
ruby bench.rb [lang] [format] [size in kB] [number of runs]
12138

122-
.xml .normal {}
123-
.xml .namespace { color: #B66; font-weight: bold; }
124-
.xml .tag { color: #F88; }
125-
.xml .comment { color: #005; font-style: italic; }
126-
.xml .punct { color: #447; font-weight: bold; }
127-
.xml .string { color: #944; }
128-
.xml .number { color: #F99; }
129-
.xml .attribute { color: #BB7; }
39+
- lang defaults to ruby.
40+
- format defaults to html.
41+
- size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input.
42+
- number of runs defaults to 5.
13043

131-
.yaml .normal {}
132-
.yaml .document { font-weight: bold; color: #07F; }
133-
.yaml .type { font-weight: bold; color: #05C; }
134-
.yaml .key { color: #F88; }
135-
.yaml .comment { color: #005; font-style: italic; }
136-
.yaml .punct { color: #447; font-weight: bold; }
137-
.yaml .string { color: #944; }
138-
.yaml .number { color: #F99; }
139-
.yaml .time { color: #F99; }
140-
.yaml .date { color: #F99; }
141-
.yaml .ref { color: #944; }
142-
.yaml .anchor { color: #944; }
44+
-h prints this help
45+
-p generates a profile (slow, use with SIZE = 1)
46+
-w waits after the benchmark (for debugging memory usw)

‎lib/coderay.rb

+2-2
Original file line numberDiff line numberDiff line change
@@ -127,14 +127,14 @@ module CodeRay
127127

128128
$CODERAY_DEBUG ||= false
129129

130-
CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay'
130+
CODERAY_PATH = File.expand_path('../coderay', __FILE__)
131131

132132
# Assuming the path is a subpath of lib/coderay/
133133
def self.coderay_path *path
134134
File.join CODERAY_PATH, *path
135135
end
136136

137-
require coderay_path('version')
137+
require 'coderay/version'
138138

139139
# helpers
140140
autoload :FileType, coderay_path('helpers', 'file_type')

‎lib/coderay/encoders/debug_lint.rb

+19-12
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module CodeRay
22
module Encoders
33

4+
load :lint
5+
46
# = Debug Lint Encoder
57
#
68
# Debug encoder with additional checks for:
@@ -15,17 +17,8 @@ class DebugLint < Debug
1517

1618
register_for :debug_lint
1719

18-
InvalidTokenStream = Class.new StandardError
19-
EmptyToken = Class.new InvalidTokenStream
20-
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
21-
22-
def initialize options = {}
23-
super
24-
@opened = []
25-
end
26-
2720
def text_token text, kind
28-
raise EmptyToken, 'empty token' if text.empty?
21+
raise Lint::EmptyToken, 'empty token' if text.empty?
2922
super
3023
end
3124

@@ -35,7 +28,8 @@ def begin_group kind
3528
end
3629

3730
def end_group kind
38-
raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
31+
raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
32+
@opened.pop
3933
super
4034
end
4135

@@ -45,7 +39,20 @@ def begin_line kind
4539
end
4640

4741
def end_line kind
48-
raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
42+
raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
43+
@opened.pop
44+
super
45+
end
46+
47+
protected
48+
49+
def setup options
50+
super
51+
@opened = []
52+
end
53+
54+
def finish options
55+
raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
4956
super
5057
end
5158

‎lib/coderay/encoders/html.rb

-1
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,6 @@ def setup options
193193

194194
def finish options
195195
unless @opened.empty?
196-
warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
197196
@out << '</span>' while @opened.pop
198197
@last_opened = nil
199198
end

‎lib/coderay/encoders/lint.rb

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
module CodeRay
2+
module Encoders
3+
4+
# = Lint Encoder
5+
#
6+
# Checks for:
7+
#
8+
# - empty tokens
9+
# - incorrect nesting
10+
#
11+
# It will raise an InvalidTokenStream exception when any of the above occurs.
12+
#
13+
# See also: Encoders::DebugLint
14+
class Lint < Debug
15+
16+
register_for :lint
17+
18+
InvalidTokenStream = Class.new StandardError
19+
EmptyToken = Class.new InvalidTokenStream
20+
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
21+
22+
def text_token text, kind
23+
raise EmptyToken, 'empty token' if text.empty?
24+
end
25+
26+
def begin_group kind
27+
@opened << kind
28+
end
29+
30+
def end_group kind
31+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
32+
@opened.pop
33+
end
34+
35+
def begin_line kind
36+
@opened << kind
37+
end
38+
39+
def end_line kind
40+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
41+
@opened.pop
42+
end
43+
44+
protected
45+
46+
def setup options
47+
@opened = []
48+
end
49+
50+
def finish options
51+
raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
52+
end
53+
54+
end
55+
56+
end
57+
end

0 commit comments

Comments
 (0)