From a92f44c57a67cb083da32ff7385bd618bd9bdfaa Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Fri, 31 May 2013 23:31:18 -0400 Subject: [PATCH 01/11] Quick-n-dirty Lua scanner --- lib/coderay/scanners/lua.rb | 111 ++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 lib/coderay/scanners/lua.rb diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb new file mode 100644 index 00000000..5b762dd0 --- /dev/null +++ b/lib/coderay/scanners/lua.rb @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +# Scanner for the Lua[http://lua.org] programming lanuage. +# +# The language’s complete syntax is defined in +# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], +# which is what this scanner tries to conform to. +class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + KEYWORDS = %w[ + and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + PREDEFINED_CONSTANTS = %w[false true nil] + + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + SCANNER = / + (?[\d\D]*?) # eat content up until something we want + (?: + \b(?#{KEYWORDS.join('|')})\b + | + (? + --\[(?=*)\[[\d\D]*?\]\k\] + ) + | + (?: + (?")(?(?:[^\\"\n]|\\[abfnrtvz\\"']|\\\n|\\\d{1,3}|\\x[\da-fA-F]{2})*)(?") + | + (?')(?(?:[^\\'\n]|\\[abfnrtvz\\"']|\\\n|\\\d{1,3}|\\x[\da-fA-F]{2})*)(?') + | + (?\[(?=*)\[)(?[\d\D]*?)(?\]\k\]) + ) + | + (? + --(?!\[).+ + ) + | + (? + -? # Allows -2 to be properly highlighted, but makes 10-5 show -5 as a single number + (?: + 0[xX][\da-fA-F]+ + | + (?: + \d+\.?\d* + | + \d*\.?\d+ + ) + (?:[eE][-+]?\d+)? + ) + ) + | + \b(?#{PREDEFINED_CONSTANTS.join('|')})\b + | + \b(?#{PREDEFINED_EXPRESSIONS.join('|')})\b + | + (? + ::[a-zA-Z_]\w*:: + ) + | + (? + \b_[A-Z]+\b + ) + ) + /x + + CAPTURE_KINDS = { + fluff: :content, + reserved: :reserved, + comment: :comment, + blockcomment: :comment, + keyword: :keyword, + number: :float, + constant: :"predefined-constant", + library: :predefined, + s1q1: :delimiter, + s1: :string, + s1q2: :delimiter, + s2q1: :delimiter, + s2: :string, + s2q2: :delimiter, + s3q1: :delimiter, + s3: :string, + s3q2: :delimiter, + gotolabel: :label, + } + + protected + + def scan_tokens(tokens, options) + string.gsub(SCANNER) do + CAPTURE_KINDS.each do |capture,kind| + tokens.text_token( $~[capture], kind ) if $~[capture] && !$~[capture].empty? + end + end + tokens + end + +end \ No newline at end of file From 8b41364210d162af9a43905835393d0448c92a94 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Sat, 1 Jun 2013 00:18:03 -0400 Subject: [PATCH 02/11] Don't treat whitespace as content; numbers in variable names are not considered numbers --- lib/coderay/scanners/lua.rb | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 5b762dd0..029a0b62 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -28,6 +28,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] SCANNER = / + (?\s*) # eat leading whitespace, just to make iteration of fluff easier (?[\d\D]*?) # eat content up until something we want (?: \b(?#{KEYWORDS.join('|')})\b @@ -48,19 +49,25 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner --(?!\[).+ ) | - (? + \b(? -? # Allows -2 to be properly highlighted, but makes 10-5 show -5 as a single number (?: - 0[xX][\da-fA-F]+ + 0[xX] + (?: + [\da-fA-F]+\.?[\da-fA-F]* + | + [\da-fA-F]*\.[\da-fA-F]+ + ) + (?:[pP][-+]?\d+)? | (?: \d+\.?\d* | - \d*\.?\d+ + \d*\.\d+ ) (?:[eE][-+]?\d+)? ) - ) + )\b | \b(?#{PREDEFINED_CONSTANTS.join('|')})\b | @@ -77,7 +84,6 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner /x CAPTURE_KINDS = { - fluff: :content, reserved: :reserved, comment: :comment, blockcomment: :comment, @@ -101,8 +107,17 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(tokens, options) string.gsub(SCANNER) do + match = $~ + tokens.text_token( match[:space], :space ) unless match[:space].empty? + unless match[:fluff].empty? + space = false + match[:fluff].split(/(\s+)/).each do |piece| + tokens.text_token(piece, space ? :space : :content) + space = !space + end + end CAPTURE_KINDS.each do |capture,kind| - tokens.text_token( $~[capture], kind ) if $~[capture] && !$~[capture].empty? + tokens.text_token( match[capture], kind ) if match[capture] && !match[capture].empty? end end tokens From e89c84123daf75ecf24f08b0c6d4dfd69fe8d8d4 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Sat, 1 Jun 2013 10:24:17 -0400 Subject: [PATCH 03/11] Recognize operators; add reference link comments and more regex comments --- lib/coderay/scanners/lua.rb | 85 +++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 029a0b62..14a62c51 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 -# Scanner for the Lua[http://lua.org] programming lanuage. -# -# The language’s complete syntax is defined in -# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], -# which is what this scanner tries to conform to. +# Scanner for the Lua programming lanuage. +# This scanner attempts to mimic the syntax defined at: +# http://www.lua.org/manual/5.2/manual.html class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner register_for :lua file_extension "lua" title "Lua" + # http://www.lua.org/manual/5.2/manual.html#3.1 KEYWORDS = %w[ and break do else elseif end for function goto if in @@ -18,9 +17,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner then until while ] - PREDEFINED_CONSTANTS = %w[false true nil] + # http://www.lua.org/manual/5.2/manual.html#3.1 + CONSTANTS = %w[false true nil] - PREDEFINED_EXPRESSIONS = %w[ + # http://www.lua.org/manual/5.2/manual.html#6.1 + LIBRARY = %w[ assert collectgarbage dofile error getmetatable ipairs load loadfile next pairs pcall print rawequal rawget rawlen rawset select setmetatable @@ -33,20 +34,18 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner (?: \b(?#{KEYWORDS.join('|')})\b | - (? - --\[(?=*)\[[\d\D]*?\]\k\] - ) - | - (?: - (?")(?(?:[^\\"\n]|\\[abfnrtvz\\"']|\\\n|\\\d{1,3}|\\x[\da-fA-F]{2})*)(?") + (?: # strings + (?") + (?(?:[^\\"\n]|\\[abfnrtvz\\"']|\\\n|\\\d{1,3}|\\x[\da-fA-F]{2})*) + (?") | - (?')(?(?:[^\\'\n]|\\[abfnrtvz\\"']|\\\n|\\\d{1,3}|\\x[\da-fA-F]{2})*)(?') + (?') + (?(?:[^\\'\n]|\\[abfnrtvz\\"']|\\\n|\\\d{1,3}|\\x[\da-fA-F]{2})*) + (?') | - (?\[(?=*)\[)(?[\d\D]*?)(?\]\k\]) - ) - | - (? - --(?!\[).+ + (?\[(?=*)\[) + (?[\d\D]*?) # Not using multiline mode due to single-line comments + (?\]\k\]) ) | \b(? @@ -54,31 +53,53 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner (?: 0[xX] (?: - [\da-fA-F]+\.?[\da-fA-F]* + [\da-fA-F]+\.?[\da-fA-F]* # 0xA and 0xA. and 0xA.1 | - [\da-fA-F]*\.[\da-fA-F]+ + \.[\da-fA-F]+ # 0x.A ) - (?:[pP][-+]?\d+)? + (?:[pP][-+]?\d+)? # 0xA.1p-3 | (?: - \d+\.?\d* + \d+\.?\d* # 3 and 3. and 3.14 | - \d*\.\d+ + \.\d+ # .3 ) - (?:[eE][-+]?\d+)? + (?:[eE][-+]?\d+)? # 3.1e-7 ) )\b | - \b(?#{PREDEFINED_CONSTANTS.join('|')})\b + (? + --(?!\[).+ + ) | - \b(?#{PREDEFINED_EXPRESSIONS.join('|')})\b + \b(?#{CONSTANTS.join('|')})\b | - (? - ::[a-zA-Z_]\w*:: + \b(?#{LIBRARY.join('|')})\b + | + (? + (?]=? + ) + | + (? + --\[(?=*) + \[[\d\D]*? # Not using multiline mode due to single-line comments + \]\k\] ) | (? - \b_[A-Z]+\b + \b_[A-Z]+\b # _VERSION + ) + | + (? + ::[a-zA-Z_]\w*:: ) ) /x @@ -101,11 +122,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner s3: :string, s3q2: :delimiter, gotolabel: :label, + operators: :operator, } protected def scan_tokens(tokens, options) + # We use the block form of gsub instead of the StringScanner capabilities because StringScanner does not support named captures in 1.9 string.gsub(SCANNER) do match = $~ tokens.text_token( match[:space], :space ) unless match[:space].empty? From ef83c13bcfe9adaaeb67def384e0f1ce0e6337c2 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Sat, 1 Jun 2013 10:45:36 -0400 Subject: [PATCH 04/11] Wrap strings and block comment delimiters in groups --- lib/coderay/scanners/lua.rb | 57 ++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 14a62c51..97e6dcc3 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -68,6 +68,12 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ) )\b | + (?: + (?--\[(?=*)\[) + (?[\d\D]*?) # Not using multiline mode due to single-line comments + (?\]\k\]) + ) + | (? --(?!\[).+ ) @@ -88,12 +94,6 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner [<>]=? ) | - (? - --\[(?=*) - \[[\d\D]*? # Not using multiline mode due to single-line comments - \]\k\] - ) - | (? \b_[A-Z]+\b # _VERSION ) @@ -107,20 +107,34 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner CAPTURE_KINDS = { reserved: :reserved, comment: :comment, - blockcomment: :comment, + blockcommentstart: { + _group: :comment, + blockcommentstart: :delimiter, + blockcommentmain: :content, + blockcommentclose: :delimiter + }, keyword: :keyword, number: :float, constant: :"predefined-constant", library: :predefined, - s1q1: :delimiter, - s1: :string, - s1q2: :delimiter, - s2q1: :delimiter, - s2: :string, - s2q2: :delimiter, - s3q1: :delimiter, - s3: :string, - s3q2: :delimiter, + s1q1: { + _group: :string, + s1q1: :delimiter, + s1: :content, + s1q2: :delimiter, + }, + s2q1: { + _group: :string, + s2q1: :delimiter, + s2: :content, + s2q2: :delimiter, + }, + s3q1: { + _group: :string, + s3q1: :delimiter, + s3: :content, + s3q2: :delimiter, + }, gotolabel: :label, operators: :operator, } @@ -140,7 +154,16 @@ def scan_tokens(tokens, options) end end CAPTURE_KINDS.each do |capture,kind| - tokens.text_token( match[capture], kind ) if match[capture] && !match[capture].empty? + next unless match[capture] && !match[capture].empty? + if kind.is_a? Hash + tokens.begin_group(kind[:_group]) + kind.each do |c,k| + tokens.text_token( match[c], k ) unless c==:_group + end + tokens.end_group(kind[:_group]) + else + tokens.text_token( match[capture], kind ) + end end end tokens From 4a9f1fea697e05e472bc32c8666853177fe432a9 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Sat, 1 Jun 2013 13:38:55 -0400 Subject: [PATCH 05/11] Use inline option switching for simpler regex --- lib/coderay/scanners/lua.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 97e6dcc3..1cab6b2b 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -30,7 +30,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner SCANNER = / (?\s*) # eat leading whitespace, just to make iteration of fluff easier - (?[\d\D]*?) # eat content up until something we want + (?(?m:.*?)) # eat content up until something we want (?: \b(?#{KEYWORDS.join('|')})\b | @@ -44,34 +44,34 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner (?') | (?\[(?=*)\[) - (?[\d\D]*?) # Not using multiline mode due to single-line comments + (?(?m:.*?)) (?\]\k\]) ) | \b(? -? # Allows -2 to be properly highlighted, but makes 10-5 show -5 as a single number - (?: - 0[xX] + (?i: + 0x (?: - [\da-fA-F]+\.?[\da-fA-F]* # 0xA and 0xA. and 0xA.1 + [\da-f]+\.?[\da-f]* # 0xA and 0xA. and 0xA.1 | - \.[\da-fA-F]+ # 0x.A + \.[\da-f]+ # 0x.A ) - (?:[pP][-+]?\d+)? # 0xA.1p-3 + (?:p[-+]?\d+)? # 0xA.1p-3 | (?: - \d+\.?\d* # 3 and 3. and 3.14 + \d+\.?\d* # 3 and 3. and 3.14 | - \.\d+ # .3 + \.\d+ # .3 ) - (?:[eE][-+]?\d+)? # 3.1e-7 + (?:e[-+]?\d+)? # 3.1e-7 ) )\b | (?: - (?--\[(?=*)\[) - (?[\d\D]*?) # Not using multiline mode due to single-line comments - (?\]\k\]) + (?--\[(?=*)\[) + (?(?m:.*?)) + (?\]\k\]) ) | (? @@ -99,7 +99,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ) | (? - ::[a-zA-Z_]\w*:: + (?i:::[a-z_]\w*::) ) ) /x @@ -107,11 +107,11 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner CAPTURE_KINDS = { reserved: :reserved, comment: :comment, - blockcommentstart: { + blockstart: { _group: :comment, - blockcommentstart: :delimiter, - blockcommentmain: :content, - blockcommentclose: :delimiter + blockstart: :delimiter, + blockmain: :content, + blockclose: :delimiter }, keyword: :keyword, number: :float, From 7b862c56140629a14b3400a500f357c3ca040817 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Sat, 1 Jun 2013 15:25:48 -0400 Subject: [PATCH 06/11] Simplify option switching; add function name matching --- lib/coderay/scanners/lua.rb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 1cab6b2b..3793524f 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -30,8 +30,12 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner SCANNER = / (?\s*) # eat leading whitespace, just to make iteration of fluff easier - (?(?m:.*?)) # eat content up until something we want + (?(?m).*?) # eat content up until something we want (?: + (? + \bfunction\s+[^\(]+ + ) + | \b(?#{KEYWORDS.join('|')})\b | (?: # strings @@ -44,7 +48,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner (?') | (?\[(?=*)\[) - (?(?m:.*?)) + (?(?m).*?) (?\]\k\]) ) | @@ -64,13 +68,13 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner | \.\d+ # .3 ) - (?:e[-+]?\d+)? # 3.1e-7 + (?:e[-+]?\d+)? # 3.1e-7 ) )\b | (?: (?--\[(?=*)\[) - (?(?m:.*?)) + (?(?m).*?) (?\]\k\]) ) | @@ -153,6 +157,12 @@ def scan_tokens(tokens, options) space = !space end end + if match[:funcname] && !match[:funcname].empty? + f,s,n = match[:funcname].split(/(\s+)/) + tokens.text_token(f,:keyword) + tokens.text_token(s,:space) + tokens.text_token(n,:function) + end CAPTURE_KINDS.each do |capture,kind| next unless match[capture] && !match[capture].empty? if kind.is_a? Hash From 3c078ec8fb955d3634dcc8954cb8234d2112213d Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Tue, 4 Jun 2013 17:45:13 -0600 Subject: [PATCH 07/11] Don't drop the end of the string when it isn't special --- lib/coderay/scanners/lua.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 3793524f..78ac757c 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -147,6 +147,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(tokens, options) # We use the block form of gsub instead of the StringScanner capabilities because StringScanner does not support named captures in 1.9 + match = nil string.gsub(SCANNER) do match = $~ tokens.text_token( match[:space], :space ) unless match[:space].empty? @@ -176,6 +177,7 @@ def scan_tokens(tokens, options) end end end + tokens.text_token(match.post_match,:content) unless match.post_match.empty? tokens end From 162ccf95780c8f85211b17991aec30c77252e38c Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Tue, 4 Jun 2013 20:45:06 -0600 Subject: [PATCH 08/11] Prevent error when styling Lua with no content; separate out whitespace in trailing fluff --- lib/coderay/scanners/lua.rb | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 78ac757c..bc1ea5de 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -29,7 +29,6 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] SCANNER = / - (?\s*) # eat leading whitespace, just to make iteration of fluff easier (?(?m).*?) # eat content up until something we want (?: (? @@ -147,17 +146,19 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner def scan_tokens(tokens, options) # We use the block form of gsub instead of the StringScanner capabilities because StringScanner does not support named captures in 1.9 - match = nil + remainder_index = 0 + + boring_kinds = [:space,:content] + add_boring = ->(fluff) do + fluff.split(/(\S+)/).each.with_index do |text,i| + tokens.text_token(text, boring_kinds[i%2]) unless text.empty? + end + end + string.gsub(SCANNER) do match = $~ - tokens.text_token( match[:space], :space ) unless match[:space].empty? - unless match[:fluff].empty? - space = false - match[:fluff].split(/(\s+)/).each do |piece| - tokens.text_token(piece, space ? :space : :content) - space = !space - end - end + remainder_index = match.offset(0).last + add_boring[match[:fluff]] if match[:funcname] && !match[:funcname].empty? f,s,n = match[:funcname].split(/(\s+)/) tokens.text_token(f,:keyword) @@ -177,7 +178,9 @@ def scan_tokens(tokens, options) end end end - tokens.text_token(match.post_match,:content) unless match.post_match.empty? + unless remainder_index >= string.length + add_boring[string[remainder_index..-1]] + end tokens end From 864a7b2de5659731926e4b185e4c76de6afe9192 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Wed, 5 Jun 2013 06:59:54 -0600 Subject: [PATCH 09/11] Cleaner way to separate whitespace runs from fluff --- lib/coderay/scanners/lua.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index bc1ea5de..1f9c1a8c 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -148,10 +148,9 @@ def scan_tokens(tokens, options) # We use the block form of gsub instead of the StringScanner capabilities because StringScanner does not support named captures in 1.9 remainder_index = 0 - boring_kinds = [:space,:content] add_boring = ->(fluff) do - fluff.split(/(\S+)/).each.with_index do |text,i| - tokens.text_token(text, boring_kinds[i%2]) unless text.empty? + fluff.scan(/((\s+)|(\S+))/).each do |text,ws,nws| + tokens.text_token(text, ws ? :space : :content) end end From 9cfd255be57f26e3e00a47cb2d3812cd54605b68 Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Thu, 6 Jun 2013 10:43:19 -0600 Subject: [PATCH 10/11] Support CDATA blocks in HTML/XML --- lib/coderay/scanners/html.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b795..fcf249a1 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -99,7 +99,17 @@ def scan_tokens encoder, options case state when :initial - if match = scan(/|.*)/m) + if match = scan(//m) + encoder.text_token match[0..-4], :content + encoder.text_token ']]>', :delimiter + else + encoder.text_token scan(/.*/m), :error + end + encoder.end_group :string + elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) encoder.text_token match, :doctype From 243eb063a0515bd6dee0862d0804beb80ee5fbdc Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Thu, 6 Jun 2013 10:43:58 -0600 Subject: [PATCH 11/11] Plain content is :plain, not :content --- lib/coderay/scanners/lua.rb | 2 +- lib/coderay/token_kinds.rb | 2 +- lib/coderay/version.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 1f9c1a8c..7c4adc37 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -150,7 +150,7 @@ def scan_tokens(tokens, options) add_boring = ->(fluff) do fluff.scan(/((\s+)|(\S+))/).each do |text,ws,nws| - tokens.text_token(text, ws ? :space : :content) + tokens.text_token(text, ws ? :space : :plain) end end diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e4..d478820c 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -76,7 +76,7 @@ module CodeRay :eyecatcher => 'eyecatcher', :ident => false, - :operator => false, + :operator => 'operator', :space => false, :plain => false diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index bfb5f243..4f8384d0 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.0.9' + VERSION = '1.0.9.1' end