tweak Debug scanners again, introduce :unknown token kind

korny · korny · commit 5d6bee7f5cac · 2013-07-21T20:14:21.000+02:00
diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
@@ -5,7 +5,7 @@ module Scanners
   
   # = Debug Scanner
   # 
-  # Interprets the output of the Encoders::Debug encoder.
+  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
   class Debug < Scanner
     
     register_for :debug
@@ -31,21 +31,24 @@ def scan_tokens encoder, options
           if @known_token_kinds.include? self[1]
             encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
           else
-            encoder.text_token matched, :error
+            encoder.text_token matched, :unknown
           end
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
           if @known_token_kinds.include? self[1]
             kind = self[1].to_sym
-            opened_tokens << kind
-            case self[2]
-            when '<'
-              encoder.begin_group kind
-            when '['
-              encoder.begin_line kind
-            else
-              raise 'CodeRay bug: This case should not be reached.'
-            end
+          else
+            kind = :unknown
+          end
+          
+          opened_tokens << kind
+          case self[2]
+          when '<'
+            encoder.begin_group kind
+          when '['
+            encoder.begin_line kind
+          else
+            raise 'CodeRay bug: This case should not be reached.'
           end
           
         elsif !opened_tokens.empty? && match = scan(/ > /x)
diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb
@@ -3,9 +3,9 @@
 module CodeRay
 module Scanners
   
-  # = Debug Scanner
+  # = Raydebug Scanner
   # 
-  # Parses the output of the Encoders::Debug encoder.
+  # Highlights the output of the Encoders::Debug encoder.
   class Raydebug < Scanner
     
     register_for :raydebug
@@ -43,17 +43,12 @@ def scan_tokens encoder, options
           encoder.text_token match, :operator if match = scan(/\)/)
           
         elsif match = scan(/ (\w+) ([<\[]) /x)
-          kind = self[1]
-          case self[2]
-          when '<'
-            encoder.text_token kind, :class
-          when '['
-            encoder.text_token kind, :class
+          encoder.text_token self[1], :class
+          if @known_token_kinds.include? self[1]
+            kind = self[1].to_sym
           else
-            raise 'CodeRay bug: This case should not be reached.'
+            kind = :unknown
           end
-          # FIXME: cache attack
-          kind = kind.to_sym
           opened_tokens << kind
           encoder.begin_group kind
           encoder.text_token self[2], :operator
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
@@ -80,5 +80,6 @@ module CodeRay
     :plain               => false                 # almost all scanners
   )
   
-  TokenKinds[:method] = TokenKinds[:function]
+  TokenKinds[:method]  = TokenKinds[:function]
+  TokenKinds[:unknown] = TokenKinds[:plain]
 end
diff --git a/test/unit/debug.rb b/test/unit/debug.rb
@@ -51,7 +51,7 @@ def test_creation
   end
   
   TEST_INPUT = <<-'DEBUG'.chomp
-integer(10)operator((\\\))string<content(test)>head[
+integer(10)operator((\\\))string<content(test)>test[
 
   	   
 method([])]
@@ -62,10 +62,10 @@ def test_creation
     [:begin_group, :string],
     ['test', :content],
     [:end_group, :string],
-    [:begin_line, :head],
+    [:begin_line, :unknown],
     ["\n\n  \t   \n", :space],
     ["[]", :method],
-    [:end_line, :head],
+    [:end_line, :unknown],
   ].flatten
   
   def test_filtering_text_tokens

Original file line number	Diff line number	Diff line change
`@@ -80,5 +80,6 @@ module CodeRay`
`80`	`80`	`:plain => false # almost all scanners`
`81`	`81`	`)`
`82`	`82`
`83`		`- TokenKinds[:method] = TokenKinds[:function]`
	`83`	`+ TokenKinds[:method] = TokenKinds[:function]`
	`84`	`+ TokenKinds[:unknown] = TokenKinds[:plain]`
`84`	`85`	`end`