Merge branch 'master' into remove-dump

korny · korny · commit a6ddf469fade · 2013-06-16T01:18:34.000+02:00
Conflicts:
	lib/coderay/tokens.rb
diff --git a/bin/coderay b/bin/coderay
@@ -155,8 +155,9 @@ when 'highlight', nil
         puts boom.message
       end
       # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
-    rescue CodeRay::Scanners::Scanner::ScanError  # FIXME: rescue Errno::EPIPE
-      # this is sometimes raised by pagers; ignore [TODO: wtf?]
+    rescue CodeRay::Scanners::Scanner::ScanError
+      # this is sometimes raised by pagers; ignore
+    # FIXME: rescue Errno::EPIPE
     ensure
       file.close if output_file
     end
diff --git a/lib/coderay.rb b/lib/coderay.rb
@@ -166,7 +166,6 @@ class << self
     #
     # See also demo/demo_simple.
     def scan code, lang, options = {}, &block
-      # FIXME: return a proxy for direct-stream encoding
       TokensProxy.new code, lang, options, block
     end
     
diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb
@@ -26,7 +26,7 @@ def self.number! output, mode = :table, options = {}
               "<a href=\"##{anchor}\" name=\"#{anchor}\">#{line}</a>"
             end
           else
-            proc { |line| line.to_s }  # :to_s.to_proc in Ruby 1.8.7+
+            :to_s.to_proc
           end
         
         bold_every = options[:bold_every]
@@ -75,7 +75,7 @@ def self.number! output, mode = :table, options = {}
           line_number = start
           output.gsub!(/^.*$\n?/) do |line|
             line_number_text = bolding.call line_number
-            indent = ' ' * (max_width - line_number.to_s.size)  # TODO: Optimize (10^x)
+            indent = ' ' * (max_width - line_number.to_s.size)
             line_number += 1
             "<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}"
           end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
@@ -67,7 +67,6 @@ def text_token text, kind
       @type_stats['TOTAL'].count += 1
     end
     
-    # TODO Hierarchy handling
     def begin_group kind
       block_token ':begin_group', kind
     end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
@@ -44,7 +44,7 @@ module CodeRay
     :important           => 'important',          # CSS, Taskpaper
     :include             => 'include',            # C, Groovy, Java, Python, Sass
     :inline              => 'inline',             # nested code, eg. inline string evaluation; lots of scanners
-    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why?
+    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why use inline_delimiter?
     :instance_variable   => 'instance-variable',  # Ruby
     :integer             => 'integer',            # most scanners
     :key                 => 'key',                # lots of scanners, used together with :value
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
@@ -1,24 +1,22 @@
 module CodeRay
   
-  # = Tokens  TODO: Rewrite!
+  # The Tokens class represents a list of tokens returned from
+  # a Scanner. It's actually just an Array with a few helper methods.
   #
-  # The Tokens class represents a list of tokens returnd from
-  # a Scanner.
-  #
-  # A token is not a special object, just a two-element Array
+  # A token itself is not a special object, just a two-element Array
   # consisting of
   # * the _token_ _text_ (the original source of the token in a String) or
   #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
   #
-  # A token looks like this:
+  # It looks like this:
   #
   #   ['# It looks like this', :comment]
   #   ['3.1415926', :float]
   #   ['$^', :error]
   #
   # Some scanners also yield sub-tokens, represented by special
-  # token actions, namely begin_group and end_group.
+  # token actions, for example :begin_group and :end_group.
   #
   # The Ruby scanner, for example, splits "a string" into:
   #
@@ -30,22 +28,17 @@ module CodeRay
   #   [:end_group, :string]
   #  ]
   #
-  # Tokens is the interface between Scanners and Encoders:
-  # The input is split and saved into a Tokens object. The Encoder
-  # then builds the output from this object.
-  #
-  # Thus, the syntax below becomes clear:
-  #
-  #   CodeRay.scan('price = 2.59', :ruby).html
-  #   # the Tokens object is here -------^
+  # Tokens can be used to save the output of a Scanners in a simple
+  # Ruby object that can be send to an Encoder later:
   #
-  # See how small it is? ;)
+  #   tokens = CodeRay.scan('price = 2.59', :ruby).tokens
+  #   tokens.encode(:html)
+  #   tokens.html
+  #   CodeRay.encoder(:html).encode_tokens(tokens)
   #
   # Tokens gives you the power to handle pre-scanned code very easily:
-  # You can convert it to a webpage, a YAML file, or a .raydebug representation.
-  # 
-  # It also allows you to generate tokens directly (without using a scanner),
-  # to load them from a file, and still use any Encoder that CodeRay provides.
+  # You can serialize it to a JSON string and store it in a database, pass it
+  # around to encode it more than once, send it to other algorithms...
   class Tokens < Array
     
     # The Scanner instance that created the tokens.
@@ -54,8 +47,7 @@ class Tokens < Array
     # Encode the tokens using encoder.
     #
     # encoder can be
-    # * a symbol like :html oder :statistic
-    # * an Encoder class
+    # * a plugin name like :html oder 'statistic'
     # * an Encoder object
     #
     # options are passed to the encoder.

Original file line number	Diff line number	Diff line change
`@@ -166,7 +166,6 @@ class << self`
`166`	`166`	`#`
`167`	`167`	`# See also demo/demo_simple.`
`168`	`168`	`def scan code, lang, options = {}, &block`
`169`		`- # FIXME: return a proxy for direct-stream encoding`
`170`	`169`	`TokensProxy.new code, lang, options, block`
`171`	`170`	`end`
`172`	`171`