Skip to content

Commit a2c625b

Browse files
kornykorny
committed
Merge branch 'master' into go-scanner
2 parents dd9ec43 + dc57601 commit a2c625b

File tree

7 files changed

+100
-140
lines changed

7 files changed

+100
-140
lines changed

‎.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,4 @@ Gemfile.lock
1111
test/executable/source.rb.html
1212
test/executable/source.rb.json
1313
test/scanners
14-
bench/test.div.html
1514
old-stuff

‎bench/bench.rb

Lines changed: 31 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -1,142 +1,46 @@
1-
# The most ugly test script I've ever written!
2-
# Shame on me!
3-
4-
require 'pathname'
5-
require 'profile' if ARGV.include? '-p'
6-
7-
MYDIR = File.dirname(__FILE__)
8-
LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s
9-
$:.unshift MYDIR, LIBDIR
1+
require 'benchmark'
2+
$: << File.expand_path('../../lib', __FILE__)
103
require 'coderay'
114

12-
@size = ARGV.fetch(2, 100).to_i * 1000
13-
14-
lang = ARGV.fetch(0) do
15-
puts <<-HELP
16-
Usage:
17-
ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
18-
19-
SIZE defaults to 100 kB (= 100,000 bytes).
20-
SIZE = 0 means the whole input.
21-
22-
-p generates a profile (slow! use with SIZE = 1)
23-
-o shows the output
24-
stream enabled streaming mode
25-
26-
Sorry for the strange interface. I will improve it in the next release.
27-
HELP
5+
if ARGV.include? '-h'
6+
puts DATA.read
287
exit
298
end
309

31-
format = ARGV.fetch(1, 'html').downcase
32-
33-
$stream = ARGV.include? 'stream'
34-
$optimize = ARGV.include? 'opt'
35-
$style = ARGV.include? 'style'
36-
37-
require 'benchmark'
38-
require 'fileutils'
10+
lang = ARGV.fetch(0, 'ruby')
11+
data = nil
12+
File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read }
13+
raise 'Example file is empty.' if data.empty?
3914

40-
if format == 'comp'
41-
format = 'page'
42-
begin
43-
require 'syntax'
44-
require 'syntax/convertors/html.rb'
45-
rescue LoadError
46-
puts 'Syntax no found!! (Try % gem install syntax)'
47-
end
48-
end
15+
format = ARGV.fetch(1, 'html').downcase
16+
encoder = CodeRay.encoder(format)
4917

50-
def here fn = nil
51-
return MYDIR unless fn
52-
File.join here, fn
18+
size = ARGV.fetch(2, 1000).to_i * 1000
19+
unless size.zero?
20+
data += data until data.size >= size
21+
data = data[0, size]
5322
end
23+
size = data.size
24+
puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round]
5425

55-
n = ARGV.find { |a| a[/^N/] }
56-
N = if n then n[/\d+/].to_i else 1 end
57-
$filename = ARGV.include?('strange') ? 'strange' : 'example'
58-
59-
Benchmark.bm(20) do |bm|
60-
N.times do
61-
62-
data = nil
63-
File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
64-
raise 'Example file is empty.' if data.empty?
65-
unless @size.zero?
66-
data += data until data.size >= @size
67-
data = data[0, @size]
68-
end
69-
@size = data.size
70-
71-
options = {
72-
:tab_width => 2,
73-
# :line_numbers => :inline,
74-
:css => $style ? :style : :class,
75-
}
76-
$hl = CodeRay.encoder(format, options)
77-
time = bm.report('CodeRay') do
78-
if $stream || true
79-
$o = $hl.encode(data, lang, options)
80-
else
81-
tokens = CodeRay.scan(data, lang)
82-
tokens.optimize! if $optimize
83-
$o = tokens.encode($hl)
84-
end
85-
end
86-
$file_created = here('test.' + $hl.file_extension)
87-
File.open($file_created, 'wb') do |f|
88-
# f.write $o
89-
end
90-
91-
time_real = time.real
92-
93-
puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
94-
puts $o if ARGV.include? '-o'
95-
96-
end
26+
n = ARGV.fetch(3, 5).to_s[/\d+/].to_i
27+
require 'profile' if ARGV.include? '-p'
28+
n.times do |i|
29+
time = Benchmark.realtime { encoder.encode(data, lang) }
30+
puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0]
9731
end
98-
puts "Files created: #$file_created"
9932

100-
STDIN.gets if ARGV.include? 'wait'
33+
STDIN.gets if ARGV.include? '-w'
10134

10235
__END__
103-
.ruby .normal {}
104-
.ruby .comment { color: #005; font-style: italic; }
105-
.ruby .keyword { color: #A00; font-weight: bold; }
106-
.ruby .method { color: #077; }
107-
.ruby .class { color: #074; }
108-
.ruby .module { color: #050; }
109-
.ruby .punct { color: #447; font-weight: bold; }
110-
.ruby .symbol { color: #099; }
111-
.ruby .string { color: #944; background: #FFE; }
112-
.ruby .char { color: #F07; }
113-
.ruby .ident { color: #004; }
114-
.ruby .constant { color: #07F; }
115-
.ruby .regex { color: #B66; background: #FEF; }
116-
.ruby .number { color: #F99; }
117-
.ruby .attribute { color: #7BB; }
118-
.ruby .global { color: #7FB; }
119-
.ruby .expr { color: #227; }
120-
.ruby .escape { color: #277; }
36+
Usage:
37+
ruby bench.rb [lang] [format] [size in kB] [number of runs]
12138

122-
.xml .normal {}
123-
.xml .namespace { color: #B66; font-weight: bold; }
124-
.xml .tag { color: #F88; }
125-
.xml .comment { color: #005; font-style: italic; }
126-
.xml .punct { color: #447; font-weight: bold; }
127-
.xml .string { color: #944; }
128-
.xml .number { color: #F99; }
129-
.xml .attribute { color: #BB7; }
39+
- lang defaults to ruby.
40+
- format defaults to html.
41+
- size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input.
42+
- number of runs defaults to 5.
13043

131-
.yaml .normal {}
132-
.yaml .document { font-weight: bold; color: #07F; }
133-
.yaml .type { font-weight: bold; color: #05C; }
134-
.yaml .key { color: #F88; }
135-
.yaml .comment { color: #005; font-style: italic; }
136-
.yaml .punct { color: #447; font-weight: bold; }
137-
.yaml .string { color: #944; }
138-
.yaml .number { color: #F99; }
139-
.yaml .time { color: #F99; }
140-
.yaml .date { color: #F99; }
141-
.yaml .ref { color: #944; }
142-
.yaml .anchor { color: #944; }
44+
-h prints this help
45+
-p generates a profile (slow, use with SIZE = 1)
46+
-w waits after the benchmark (for debugging memory usw)

‎lib/coderay/encoders/debug_lint.rb

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module CodeRay
22
module Encoders
33

4+
load :lint
5+
46
# = Debug Lint Encoder
57
#
68
# Debug encoder with additional checks for:
@@ -15,12 +17,8 @@ class DebugLint < Debug
1517

1618
register_for :debug_lint
1719

18-
InvalidTokenStream = Class.new StandardError
19-
EmptyToken = Class.new InvalidTokenStream
20-
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
21-
2220
def text_token text, kind
23-
raise EmptyToken, 'empty token' if text.empty?
21+
raise Lint::EmptyToken, 'empty token' if text.empty?
2422
super
2523
end
2624

@@ -30,7 +28,7 @@ def begin_group kind
3028
end
3129

3230
def end_group kind
33-
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
31+
raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
3432
@opened.pop
3533
super
3634
end
@@ -41,7 +39,7 @@ def begin_line kind
4139
end
4240

4341
def end_line kind
44-
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
42+
raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
4543
@opened.pop
4644
super
4745
end

‎lib/coderay/encoders/lint.rb

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
module CodeRay
2+
module Encoders
3+
4+
# = Lint Encoder
5+
#
6+
# Checks for:
7+
#
8+
# - empty tokens
9+
# - incorrect nesting
10+
#
11+
# It will raise an InvalidTokenStream exception when any of the above occurs.
12+
#
13+
# See also: Encoders::DebugLint
14+
class Lint < Debug
15+
16+
register_for :lint
17+
18+
InvalidTokenStream = Class.new StandardError
19+
EmptyToken = Class.new InvalidTokenStream
20+
IncorrectTokenGroupNesting = Class.new InvalidTokenStream
21+
22+
def text_token text, kind
23+
raise EmptyToken, 'empty token' if text.empty?
24+
end
25+
26+
def begin_group kind
27+
@opened << kind
28+
end
29+
30+
def end_group kind
31+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
32+
@opened.pop
33+
end
34+
35+
def begin_line kind
36+
@opened << kind
37+
end
38+
39+
def end_line kind
40+
raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
41+
@opened.pop
42+
end
43+
44+
protected
45+
46+
def setup options
47+
@opened = []
48+
end
49+
50+
def finish options
51+
raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
52+
end
53+
54+
end
55+
56+
end
57+
end

‎lib/coderay/helpers/file_type.rb

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ def shebang filename
117117
'rhtml' => :erb,
118118
'rjs' => :ruby,
119119
'rpdf' => :ruby,
120-
'ru' => :ruby,
120+
'ru' => :ruby, # config.ru
121121
'rxml' => :ruby,
122122
'sass' => :sass,
123123
'sql' => :sql,
@@ -141,6 +141,9 @@ def shebang filename
141141
'Rakefile' => :ruby,
142142
'Rantfile' => :ruby,
143143
'Gemfile' => :ruby,
144+
'Guardfile' => :ruby,
145+
'Vagrantfile' => :ruby,
146+
'Appraisals' => :ruby
144147
}
145148

146149
end

‎rake_tasks/benchmark.rake

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
desc 'Do a benchmark'
22
task :benchmark do
3-
ruby "-v"
4-
ruby "-wIlib bench/bench.rb ruby div 3000 N5"
3+
ruby 'bench/bench.rb ruby html 3000'
54
end
65

76
task :bench => :benchmark

‎test/executable/suite.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase
1818
EXE_COMMAND =
1919
if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success?
2020
# use Nailgun
21-
"#{RUBY_COMMAND}--ng -w -I%s %s"
21+
"#{RUBY_COMMAND}--ng -I%s %s"
2222
else
23-
"#{RUBY_COMMAND} -w -I%s %s"
23+
"#{RUBY_COMMAND} -I%s %s"
2424
end % [ROOT_DIR + 'lib', EXECUTABLE]
2525

2626
def coderay args, options = {}

0 commit comments

Comments
 (0)