rubychan · korny · Jun 23, 2013 · Jun 22, 2013 · Jun 22, 2013 · Jun 22, 2013
diff --git a/Changes.textile b/Changes.textile
@@ -12,6 +12,8 @@ h2. Changes in 1.1
 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126]
 * Remove double-click toggle handler from HTML table output
 * Fixes to CSS scanner (floats, pseudoclasses)
+* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144]
+* Added @:keep_state@ functionality to more scanners [#116]
 * CSS scanner uses @:id@ and @:tag@ now [#27]
 * Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
 * Add .xaml file type [#121, thanks to Kozman Bálint]
@@ -20,10 +22,13 @@ h2. Changes in 1.1
 * New token type @:done@ for Taskpaper [#39]
 * New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany]
 * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
-* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
+* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
 * Fixed @:docstring@ token type style
 * @Plugin@ does not warn about fallback when default is defined
+* @HTML@ encoder will not warn about unclosed token groups at the end of the stream
 * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
+* @Debug@ encoder will not warn about errors in the token stream
+* New @DebugLint@ encoder that checks for empty tokens and correct nesting
 
 h2. Changes in 1.0.9
 

diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
@@ -19,11 +19,6 @@ class DebugLint < Debug
     EmptyToken = Class.new InvalidTokenStream
     IncorrectTokenGroupNesting = Class.new InvalidTokenStream
 
-    def initialize options = {}
-      super
-      @opened = []
-    end
-
     def text_token text, kind
       raise EmptyToken, 'empty token' if text.empty?
       super
@@ -35,7 +30,8 @@ def begin_group kind
     end
 
     def end_group kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
       super
     end
 
@@ -45,7 +41,20 @@ def begin_line kind
     end
 
     def end_line kind
-      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+      super
+    end
+
+    protected
+
+    def setup options
+      super
+      @opened = []
+    end
+
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
       super
     end
 

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
@@ -193,7 +193,6 @@ def setup options
 
     def finish options
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
         @out << '</span>' while @opened.pop
         @last_opened = nil
       end

diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
@@ -145,10 +145,10 @@ def scan_tokens encoder, options
           start = match[/^\w+\(/]
           encoder.text_token start, :delimiter
           if match[-1] == ?)
-            encoder.text_token match[start.size..-2], :content
+            encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if match.size > start.size
           end
           encoder.end_group :function
 

diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
@@ -69,7 +69,7 @@ def scan_tokens encoder, options
             state = :added
           elsif match = scan(/\\ .*/)
             encoder.text_token match, :comment
-          elsif match = scan(/@@(?>[^@\n]*)@@/)
+          elsif match = scan(/@@(?>[^@\n]+)@@/)
             content_scanner.state = :initial unless match?(/\n\+/)
             content_scanner_entry_state = nil
             if check(/\n|$/)

diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
@@ -36,9 +36,12 @@ class Groovy < Java
 
   protected
 
+    def setup
+      @state = :initial
+    end
+
     def scan_tokens encoder, options
-
-      state = :initial
+      state = options[:state] || @state
       inline_block_stack = []
       inline_block_paren_depth = nil
       string_delimiter = nil
@@ -223,7 +226,7 @@ def scan_tokens encoder, options
             encoder.text_token match, :content  # TODO: Shouldn't this be :error?
 
           elsif match = scan(/ \\ | \n /x)
-            encoder.end_group state
+            encoder.end_group state == :regexp ? :regexp : :string
             encoder.text_token match, :error
             after_def = value_expected = false
             state = :initial
@@ -243,7 +246,17 @@ def scan_tokens encoder, options
       end
 
       if [:multiline_string, :string, :regexp].include? state
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
+      end
+
+      if options[:keep_state]
+        @state = state
+      end
+
+      until inline_block_stack.empty?
+        state, = *inline_block_stack.pop
+        encoder.end_group :inline
+        encoder.end_group state == :regexp ? :regexp : :string
       end
 
       encoder

diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb
@@ -59,13 +59,15 @@ def setup
     # CodeRay entry hook. Starts parsing.
     def scan_tokens(encoder, options)
       state = options[:state] || @state
+      brace_depth = @brace_depth
+      num_equals = nil
 
       until eos?
         case state
 
         when :initial
           if match = scan(/\-\-\[\=*\[/)   #--[[ long (possibly multiline) comment ]]
-            @num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for comment end
             encoder.begin_group(:comment)
             encoder.text_token(match, :delimiter)
             state = :long_comment
@@ -74,7 +76,7 @@ def scan_tokens(encoder, options)
             encoder.text_token(match, :comment)
 
           elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
-            @num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for comment end
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
             state = :long_string
@@ -101,19 +103,19 @@ def scan_tokens(encoder, options)
 
           elsif match = scan(/\{/) # Opening table brace {
             encoder.begin_group(:map)
-            encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter)
-            @brace_depth += 1
+            encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter)
+            brace_depth += 1
             state        = :map
 
           elsif match = scan(/\}/) # Closing table brace }
-            if @brace_depth == 1
-              @brace_depth = 0
+            if brace_depth == 1
+              brace_depth = 0
               encoder.text_token(match, :delimiter)
               encoder.end_group(:map)
-            elsif @brace_depth == 0 # Mismatched brace
+            elsif brace_depth == 0 # Mismatched brace
               encoder.text_token(match, :error)
             else
-              @brace_depth -= 1
+              brace_depth -= 1
               encoder.text_token(match, :inline_delimiter)
               encoder.end_group(:map)
               state = :map
@@ -122,7 +124,7 @@ def scan_tokens(encoder, options)
           elsif match = scan(/["']/) # String delimiters " and '
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
-            @start_delim = match
+            start_delim = match
             state       = :string
 
                             # ↓Prefix                hex number ←|→ decimal number
@@ -146,7 +148,7 @@ def scan_tokens(encoder, options)
           # It may be that we’re scanning a full-blown subexpression of a table
           # (tables can contain full expressions in parts).
           # If this is the case, return to :map scanning state.
-          state = :map if state == :initial && @brace_depth >= 1
+          state = :map if state == :initial && brace_depth >= 1
 
         when :function_expected
           if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name
@@ -198,10 +200,10 @@ def scan_tokens(encoder, options)
           end
 
         when :long_comment
-          if match = scan(/.*?(?=\]={#@num_equals}\])/m)
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m)
             encoder.text_token(match, :content)
 
-            delim = scan(/\]={#@num_equals}\]/)
+            delim = scan(/\]={#{num_equals}}\]/)
             encoder.text_token(delim, :delimiter)
           else # No terminator found till EOF
             encoder.text_token(rest, :error)
@@ -211,10 +213,10 @@ def scan_tokens(encoder, options)
           state = :initial
 
         when :long_string
-          if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences
+          if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences
             encoder.text_token(match, :content)
 
-            delim = scan(/\]={#@num_equals}\]/)
+            delim = scan(/\]={#{num_equals}}\]/)
             encoder.text_token(delim, :delimiter)
           else # No terminator found till EOF
             encoder.text_token(rest, :error)
@@ -224,11 +226,11 @@ def scan_tokens(encoder, options)
           state = :initial
 
         when :string
-          if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
+          if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z)
             encoder.text_token(match, :content)
           elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m)
             encoder.text_token(match, :char)
-          elsif match = scan(Regexp.compile(@start_delim))
+          elsif match = scan(Regexp.compile(start_delim))
             encoder.text_token(match, :delimiter)
             encoder.end_group(:string)
             state = :initial
@@ -266,6 +268,9 @@ def scan_tokens(encoder, options)
         @state = state
       end
 
+      encoder.end_group :string if [:string].include? state
+      brace_depth.times { encoder.end_group :map }
+
       encoder
     end
 

diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
@@ -265,7 +265,7 @@ def scan_tokens encoder, options
             @html_scanner.tokenize match unless match.empty?
           end
 
-        when :php
+        when :php, :php_inline
           if match = scan(/\s+/)
             encoder.text_token match, :space
 
@@ -332,14 +332,15 @@ def scan_tokens encoder, options
             if states.size == 1
               encoder.text_token match, :error
             else
-              states.pop
+              state = states.pop
               if states.last.is_a?(::Array)
                 delimiter = states.last[1]
                 states[-1] = states.last[0]
                 encoder.text_token match, :delimiter
                 encoder.end_group :inline
               else
                 encoder.text_token match, :operator
+                encoder.end_group :inline if state == :php_inline
                 label_expected = true
               end
             end
@@ -350,7 +351,14 @@ def scan_tokens encoder, options
 
           elsif match = scan(RE::PHP_END)
             encoder.text_token match, :inline_delimiter
-            states = [:initial]
+            while state = states.pop
+              encoder.end_group :string if [:sqstring, :dqstring].include? state
+              if state.is_a? Array
+                encoder.end_group :inline
+                encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+              end
+            end
+            states << :initial
 
           elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
             encoder.begin_group :string
@@ -400,6 +408,7 @@ def scan_tokens encoder, options
           elsif match = scan(/\\/)
             encoder.text_token match, :error
           else
+            encoder.end_group :string
             states.pop
           end
 
@@ -459,7 +468,7 @@ def scan_tokens encoder, options
               encoder.begin_group :inline
               states[-1] = [states.last, delimiter]
               delimiter = nil
-              states.push :php
+              states.push :php_inline
               encoder.text_token match, :delimiter
             else
               encoder.text_token match, :content
@@ -469,6 +478,7 @@ def scan_tokens encoder, options
           elsif match = scan(/\$/)
             encoder.text_token match, :content
           else
+            encoder.end_group :string
             states.pop
           end
 
@@ -500,6 +510,14 @@ def scan_tokens encoder, options
 
       end
 
+      while state = states.pop
+        encoder.end_group :string if [:sqstring, :dqstring].include? state
+        if state.is_a? Array
+          encoder.end_group :inline
+          encoder.end_group :string if [:sqstring, :dqstring].include? state.first
+        end
+      end
+
       encoder
     end
 

diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
@@ -1,23 +1,23 @@
 module CodeRay
 module Scanners
-
+  
   # = Debug Scanner
   # 
   # Parses the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
-
+    
     register_for :raydebug
     file_extension 'raydebug'
     title 'CodeRay Token Dump'
 
   protected
 
     def scan_tokens encoder, options
-
+      
       opened_tokens = []
-
+      
       until eos?
-
+        
         if match = scan(/\s+/)
           encoder.text_token match, :space
 
@@ -26,7 +26,7 @@ def scan_tokens encoder, options
           encoder.text_token kind, :class
           encoder.text_token '(', :operator
           match = self[2]
-          encoder.text_token match, kind.to_sym
+          encoder.text_token match, kind.to_sym unless match.empty?
           encoder.text_token match, :operator if match = scan(/\)/)
 
         elsif match = scan(/ (\w+) ([<\[]) /x)
@@ -59,8 +59,8 @@ def scan_tokens encoder, options
 
       encoder
     end
-
+    
   end
-
+  
 end
 end