Skip to content

Commit a6ddf46

Browse files
kornykorny
committed
Merge branch 'master' into remove-dump
Conflicts: lib/coderay/tokens.rb
2 parents e423275 + dc95284 commit a6ddf46

File tree

6 files changed

+20
-29
lines changed

6 files changed

+20
-29
lines changed

‎bin/coderay

+3-2
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,9 @@ when 'highlight', nil
155155
puts boom.message
156156
end
157157
# puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
158-
rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE
159-
# this is sometimes raised by pagers; ignore [TODO: wtf?]
158+
rescue CodeRay::Scanners::Scanner::ScanError
159+
# this is sometimes raised by pagers; ignore
160+
# FIXME: rescue Errno::EPIPE
160161
ensure
161162
file.close if output_file
162163
end

‎lib/coderay.rb

-1
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ class << self
166166
#
167167
# See also demo/demo_simple.
168168
def scan code, lang, options = {}, &block
169-
# FIXME: return a proxy for direct-stream encoding
170169
TokensProxy.new code, lang, options, block
171170
end
172171

‎lib/coderay/encoders/html/numbering.rb

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def self.number! output, mode = :table, options = {}
2626
"<a href=\"##{anchor}\" name=\"#{anchor}\">#{line}</a>"
2727
end
2828
else
29-
proc { |line| line.to_s } # :to_s.to_proc in Ruby 1.8.7+
29+
:to_s.to_proc
3030
end
3131

3232
bold_every = options[:bold_every]
@@ -75,7 +75,7 @@ def self.number! output, mode = :table, options = {}
7575
line_number = start
7676
output.gsub!(/^.*$\n?/) do |line|
7777
line_number_text = bolding.call line_number
78-
indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
78+
indent = ' ' * (max_width - line_number.to_s.size)
7979
line_number += 1
8080
"<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}"
8181
end

‎lib/coderay/encoders/statistic.rb

-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ def text_token text, kind
6767
@type_stats['TOTAL'].count += 1
6868
end
6969

70-
# TODO Hierarchy handling
7170
def begin_group kind
7271
block_token ':begin_group', kind
7372
end

‎lib/coderay/token_kinds.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ module CodeRay
4444
:important => 'important', # CSS, Taskpaper
4545
:include => 'include', # C, Groovy, Java, Python, Sass
4646
:inline => 'inline', # nested code, eg. inline string evaluation; lots of scanners
47-
:inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why?
47+
:inline_delimiter => 'inline-delimiter', # used instead of :inline > :delimiter FIXME: Why use inline_delimiter?
4848
:instance_variable => 'instance-variable', # Ruby
4949
:integer => 'integer', # most scanners
5050
:key => 'key', # lots of scanners, used together with :value

‎lib/coderay/tokens.rb

+14-22
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,22 @@
11
module CodeRay
22

3-
# = Tokens TODO: Rewrite!
3+
# The Tokens class represents a list of tokens returned from
4+
# a Scanner. It's actually just an Array with a few helper methods.
45
#
5-
# The Tokens class represents a list of tokens returnd from
6-
# a Scanner.
7-
#
8-
# A token is not a special object, just a two-element Array
6+
# A token itself is not a special object, just a two-element Array
97
# consisting of
108
# * the _token_ _text_ (the original source of the token in a String) or
119
# a _token_ _action_ (begin_group, end_group, begin_line, end_line)
1210
# * the _token_ _kind_ (a Symbol representing the type of the token)
1311
#
14-
# A token looks like this:
12+
# It looks like this:
1513
#
1614
# ['# It looks like this', :comment]
1715
# ['3.1415926', :float]
1816
# ['$^', :error]
1917
#
2018
# Some scanners also yield sub-tokens, represented by special
21-
# token actions, namely begin_group and end_group.
19+
# token actions, for example :begin_group and :end_group.
2220
#
2321
# The Ruby scanner, for example, splits "a string" into:
2422
#
@@ -30,22 +28,17 @@ module CodeRay
3028
# [:end_group, :string]
3129
# ]
3230
#
33-
# Tokens is the interface between Scanners and Encoders:
34-
# The input is split and saved into a Tokens object. The Encoder
35-
# then builds the output from this object.
36-
#
37-
# Thus, the syntax below becomes clear:
38-
#
39-
# CodeRay.scan('price = 2.59', :ruby).html
40-
# # the Tokens object is here -------^
31+
# Tokens can be used to save the output of a Scanners in a simple
32+
# Ruby object that can be send to an Encoder later:
4133
#
42-
# See how small it is? ;)
34+
# tokens = CodeRay.scan('price = 2.59', :ruby).tokens
35+
# tokens.encode(:html)
36+
# tokens.html
37+
# CodeRay.encoder(:html).encode_tokens(tokens)
4338
#
4439
# Tokens gives you the power to handle pre-scanned code very easily:
45-
# You can convert it to a webpage, a YAML file, or a .raydebug representation.
46-
#
47-
# It also allows you to generate tokens directly (without using a scanner),
48-
# to load them from a file, and still use any Encoder that CodeRay provides.
40+
# You can serialize it to a JSON string and store it in a database, pass it
41+
# around to encode it more than once, send it to other algorithms...
4942
class Tokens < Array
5043

5144
# The Scanner instance that created the tokens.
@@ -54,8 +47,7 @@ class Tokens < Array
5447
# Encode the tokens using encoder.
5548
#
5649
# encoder can be
57-
# * a symbol like :html oder :statistic
58-
# * an Encoder class
50+
# * a plugin name like :html oder 'statistic'
5951
# * an Encoder object
6052
#
6153
# options are passed to the encoder.

0 commit comments

Comments
 (0)