fix Tokens documentation

korny · korny · commit 546b489e4b38 · 2013-06-23T01:09:14.000+02:00
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
@@ -3,29 +3,28 @@ module CodeRay
   # The Tokens class represents a list of tokens returned from
   # a Scanner. It's actually just an Array with a few helper methods.
   #
-  # A token itself is not a special object, just a two-element Array
-  # consisting of
+  # A token itself is not a special object, just two elements in an Array:
   # * the _token_ _text_ (the original source of the token in a String) or
   #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
   #
   # It looks like this:
   #
-  #   ['# It looks like this', :comment]
-  #   ['3.1415926', :float]
-  #   ['$^', :error]
+  #   ..., '# It looks like this', :comment, ...
+  #   ..., '3.1415926', :float, ...
+  #   ..., '$^', :error, ...
   #
   # Some scanners also yield sub-tokens, represented by special
   # token actions, for example :begin_group and :end_group.
   #
   # The Ruby scanner, for example, splits "a string" into:
   #
   #  [
-  #   [:begin_group, :string],
-  #   ['"', :delimiter],
-  #   ['a string', :content],
-  #   ['"', :delimiter],
-  #   [:end_group, :string]
+  #   :begin_group, :string,
+  #   '"',          :delimiter,
+  #   'a string',   :content,
+  #   '"',          :delimiter,
+  #   :end_group,   :string
   #  ]
   #
   # Tokens can be used to save the output of a Scanners in a simple

Original file line number	Diff line number	Diff line change
`@@ -3,29 +3,28 @@ module CodeRay`
`3`	`3`	`# The Tokens class represents a list of tokens returned from`
`4`	`4`	`# a Scanner. It's actually just an Array with a few helper methods.`
`5`	`5`	`#`
`6`		`- # A token itself is not a special object, just a two-element Array`
`7`		`- # consisting of`
	`6`	`+ # A token itself is not a special object, just two elements in an Array:`
`8`	`7`	`# * the _token_ _text_ (the original source of the token in a String) or`
`9`	`8`	`# a _token_ _action_ (begin_group, end_group, begin_line, end_line)`
`10`	`9`	`# * the _token_ _kind_ (a Symbol representing the type of the token)`
`11`	`10`	`#`
`12`	`11`	`# It looks like this:`
`13`	`12`	`#`
`14`		`- # ['# It looks like this', :comment]`
`15`		`- # ['3.1415926', :float]`
`16`		`- # ['$^', :error]`
	`13`	`+ # ..., '# It looks like this', :comment, ...`
	`14`	`+ # ..., '3.1415926', :float, ...`
	`15`	`+ # ..., '$^', :error, ...`
`17`	`16`	`#`
`18`	`17`	`# Some scanners also yield sub-tokens, represented by special`
`19`	`18`	`# token actions, for example :begin_group and :end_group.`
`20`	`19`	`#`
`21`	`20`	`# The Ruby scanner, for example, splits "a string" into:`
`22`	`21`	`#`
`23`	`22`	`# [`
`24`		`- # [:begin_group, :string],`
`25`		`- # ['"', :delimiter],`
`26`		`- # ['a string', :content],`
`27`		`- # ['"', :delimiter],`
`28`		`- # [:end_group, :string]`
	`23`	`+ # :begin_group, :string,`
	`24`	`+ # '"', :delimiter,`
	`25`	`+ # 'a string', :content,`
	`26`	`+ # '"', :delimiter,`
	`27`	`+ # :end_group, :string`
`29`	`28`	`# ]`
`30`	`29`	`#`
`31`	`30`	`# Tokens can be used to save the output of a Scanners in a simple`