Skip to content

Commit 0840836

Browse files
kornykorny
committed
cleanups, documentation for TokensProxy
1 parent 8b8d32e commit 0840836

File tree

11 files changed

+46
-138
lines changed

11 files changed

+46
-138
lines changed

‎Changes-1.0.textile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ h3. @Tokens@
6363

6464
* *NEW* methods @count@, @begin_group@, @end_group@, @begin_line@, and @end_line@.
6565
* *REMOVED* methods @#stream?@, @#each_text_token@.
66+
* *REMOVED* methods @#optimize@, @#fix@, @#split_into_lines@ along with their bang! variants.
6667
* *REMOVED* @#text@ and @#text_size@ methods. Use the @Text@ encoder instead.
6768
* *REMOVED* special implementation of @#each@ taking a filter parameter. Use @TokenKindFilter@ instead.
6869

‎etc/CodeRay.tmproj

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
<key>documents</key>
66
<array>
77
<dict>
8-
<key>expanded</key>
9-
<true/>
108
<key>name</key>
119
<string>lib</string>
1210
<key>regexFolderFilter</key>
@@ -15,6 +13,8 @@
1513
<string>../lib</string>
1614
</dict>
1715
<dict>
16+
<key>expanded</key>
17+
<true/>
1818
<key>name</key>
1919
<string>bin</string>
2020
<key>regexFolderFilter</key>
@@ -26,15 +26,15 @@
2626
<key>filename</key>
2727
<string>../coderay.gemspec</string>
2828
<key>lastUsed</key>
29-
<date>2011-08-23T02:18:44Z</date>
29+
<date>2011-09-20T13:58:45Z</date>
30+
<key>selected</key>
31+
<true/>
3032
</dict>
3133
<dict>
3234
<key>filename</key>
3335
<string>../Changes-1.0.textile</string>
3436
<key>lastUsed</key>
35-
<date>2011-09-14T00:53:48Z</date>
36-
<key>selected</key>
37-
<true/>
37+
<date>2011-09-19T00:26:49Z</date>
3838
</dict>
3939
<dict>
4040
<key>filename</key>
@@ -101,6 +101,8 @@
101101
<string>../rake_helpers</string>
102102
</dict>
103103
<dict>
104+
<key>expanded</key>
105+
<true/>
104106
<key>name</key>
105107
<string>rake_tasks</string>
106108
<key>regexFolderFilter</key>
@@ -122,7 +124,7 @@
122124
<key>filename</key>
123125
<string>../Rakefile</string>
124126
<key>lastUsed</key>
125-
<date>2011-08-23T02:18:45Z</date>
127+
<date>2011-09-18T23:49:40Z</date>
126128
</dict>
127129
<dict>
128130
<key>name</key>

‎lib/coderay/encoder.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class << self
3434
# downcase class name instead.
3535
def const_missing sym
3636
if sym == :FILE_EXTENSION
37-
(@plugin_id || name[/\w+$/].downcase).to_s
37+
(defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s
3838
else
3939
super
4040
end

‎lib/coderay/scanners/java/builtin_types.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module Scanners
33

44
module Java::BuiltinTypes # :nodoc:
55

6+
#:nocov:
67
List = %w[
78
AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection
89
AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel
@@ -412,6 +413,7 @@ module Java::BuiltinTypes # :nodoc:
412413
XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver
413414
XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView
414415
]
416+
#:nocov:
415417

416418
end
417419

‎lib/coderay/tokens.rb

Lines changed: 0 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -83,97 +83,6 @@ def method_missing meth, options = {}
8383
super
8484
end
8585

86-
# Returns the tokens compressed by joining consecutive
87-
# tokens of the same kind.
88-
#
89-
# This can not be undone, but should yield the same output
90-
# in most Encoders. It basically makes the output smaller.
91-
#
92-
# Combined with dump, it saves space for the cost of time.
93-
#
94-
# If the scanner is written carefully, this is not required -
95-
# for example, consecutive //-comment lines could already be
96-
# joined in one comment token by the Scanner.
97-
def optimize
98-
raise NotImplementedError, 'Tokens#optimize needs to be rewritten.'
99-
# last_kind = last_text = nil
100-
# new = self.class.new
101-
# for text, kind in self
102-
# if text.is_a? String
103-
# if kind == last_kind
104-
# last_text << text
105-
# else
106-
# new << [last_text, last_kind] if last_kind
107-
# last_text = text
108-
# last_kind = kind
109-
# end
110-
# else
111-
# new << [last_text, last_kind] if last_kind
112-
# last_kind = last_text = nil
113-
# new << [text, kind]
114-
# end
115-
# end
116-
# new << [last_text, last_kind] if last_kind
117-
# new
118-
end
119-
120-
# Compact the object itself; see optimize.
121-
def optimize!
122-
replace optimize
123-
end
124-
125-
# Ensure that all begin_group tokens have a correspondent end_group.
126-
#
127-
# TODO: Test this!
128-
def fix
129-
raise NotImplementedError, 'Tokens#fix needs to be rewritten.'
130-
# tokens = self.class.new
131-
# # Check token nesting using a stack of kinds.
132-
# opened = []
133-
# for type, kind in self
134-
# case type
135-
# when :begin_group
136-
# opened.push [:begin_group, kind]
137-
# when :begin_line
138-
# opened.push [:end_line, kind]
139-
# when :end_group, :end_line
140-
# expected = opened.pop
141-
# if [type, kind] != expected
142-
# # Unexpected end; decide what to do based on the kind:
143-
# # - token was never opened: delete the end (just skip it)
144-
# next unless opened.rindex expected
145-
# # - token was opened earlier: also close tokens in between
146-
# tokens << token until (token = opened.pop) == expected
147-
# end
148-
# end
149-
# tokens << [type, kind]
150-
# end
151-
# # Close remaining opened tokens
152-
# tokens << token while token = opened.pop
153-
# tokens
154-
end
155-
156-
def fix!
157-
replace fix
158-
end
159-
160-
# TODO: Scanner#split_into_lines
161-
#
162-
# Makes sure that:
163-
# - newlines are single tokens
164-
# (which means all other token are single-line)
165-
# - there are no open tokens at the end the line
166-
#
167-
# This makes it simple for encoders that work line-oriented,
168-
# like HTML with list-style numeration.
169-
def split_into_lines
170-
raise NotImplementedError
171-
end
172-
173-
def split_into_lines!
174-
replace split_into_lines
175-
end
176-
17786
# Split the tokens into parts of the given +sizes+.
17887
#
17988
# The result will be an Array of Tokens objects. The parts have

‎lib/coderay/tokens_proxy.rb

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,23 @@
11
module CodeRay
22

3-
class TokensProxy < Struct.new :input, :lang, :options, :block
3+
# The result of a scan operation is a TokensProxy, but should act like Tokens.
4+
#
5+
# This proxy makes it possible to use the classic CodeRay.scan.encode API
6+
# while still providing the benefits of direct streaming.
7+
class TokensProxy
48

9+
attr_accessor :input, :lang, :options, :block
10+
11+
# Create a new TokensProxy with the arguments of CodeRay.scan.
12+
def initialize input, lang, options = {}, block = nil
13+
@input = input
14+
@lang = lang
15+
@options = options
16+
@block = block
17+
end
18+
19+
# Call CodeRay.encode if +encoder+ is a Symbol;
20+
# otherwise, convert the receiver to tokens and call encoder.encode_tokens.
521
def encode encoder, options = {}
622
if encoder.respond_to? :to_sym
723
CodeRay.encode(input, lang, encoder, options)
@@ -10,29 +26,30 @@ def encode encoder, options = {}
1026
end
1127
end
1228

29+
# Tries to call encode;
30+
# delegates to tokens otherwise.
1331
def method_missing method, *args, &blk
14-
encode method, *args
32+
encode method.to_sym, *args
1533
rescue PluginHost::PluginNotFound
1634
tokens.send(method, *args, &blk)
1735
end
1836

37+
# The (cached) result of the tokenized input; a Tokens instance.
1938
def tokens
2039
@tokens ||= scanner.tokenize(input)
2140
end
2241

42+
# A (cached) scanner instance to use for the scan task.
2343
def scanner
2444
@scanner ||= CodeRay.scanner(lang, options, &block)
2545
end
2646

47+
# Overwrite Struct#each.
2748
def each *args, &blk
2849
tokens.each(*args, &blk)
2950
self
3051
end
3152

32-
def count
33-
tokens.count
34-
end
35-
3653
end
3754

3855
end

‎rake_helpers/html_coderay_generator.rb

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1087,13 +1087,12 @@ def old_markup_code(tokens)
10871087
def markup_code(tokens)
10881088
code = tokens.map { |t| t.text }.join
10891089
options = {
1090-
:css => :class,
1091-
:line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1,
1092-
:bold_every => :no_bolding,
1090+
:css => :class,
1091+
:line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1,
1092+
:bold_every => :no_bolding,
10931093
}
10941094
options[:line_numbers] = nil unless Options.instance.include_line_numbers
1095-
out = CodeRay.scan(code, :ruby).div(options)
1096-
out.wrap! :div
1095+
CodeRay.scan(code, :ruby).div(options)
10971096
end
10981097

10991098
# we rely on the fact that the first line of a source code

‎test/functional/basic.rb

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,9 @@ def test_token_kinds
148148
end
149149
end
150150
assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
151-
assert_equal false, CodeRay::TokenKinds[:shibboleet]
151+
assert_warning 'Undefined Token kind: :shibboleet' do
152+
assert_equal false, CodeRay::TokenKinds[:shibboleet]
153+
end
152154
end
153155

154156
class Milk < CodeRay::Encoders::Encoder

‎test/functional/suite.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
require 'test/unit'
22

3+
$VERBOSE = $CODERAY_DEBUG = true
34
$:.unshift File.expand_path('../../../lib', __FILE__)
45
require 'coderay'
56

‎test/unit/suite.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
require 'test/unit'
22
require 'rubygems'
3+
4+
$VERBOSE = $CODERAY_DEBUG = true
35
$:.unshift 'lib'
46

57
mydir = File.dirname(__FILE__)

‎test/unit/tokens.rb

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -37,33 +37,6 @@ def test_encode_with_nonsense
3737
end
3838
end
3939

40-
def test_optimize
41-
assert_raise NotImplementedError do
42-
make_tokens.optimize
43-
end
44-
assert_raise NotImplementedError do
45-
make_tokens.optimize!
46-
end
47-
end
48-
49-
def test_fix
50-
assert_raise NotImplementedError do
51-
make_tokens.fix
52-
end
53-
assert_raise NotImplementedError do
54-
make_tokens.fix!
55-
end
56-
end
57-
58-
def test_split_into_lines
59-
assert_raise NotImplementedError do
60-
make_tokens.split_into_lines
61-
end
62-
assert_raise NotImplementedError do
63-
make_tokens.split_into_lines!
64-
end
65-
end
66-
6740
def test_split_into_parts
6841
parts_4_3 = [
6942
["stri", :type],

0 commit comments

Comments
 (0)