From 6d205f124ab268907fcc950fc4aebff684dfcc4a Mon Sep 17 00:00:00 2001 From: Sean Kellogg Date: Thu, 3 Jan 2013 18:05:14 -0800 Subject: [PATCH 1/8] add new liquid scanner that actually works --- lib/coderay/helpers/file_type.rb | 37 ++++++++-------- lib/coderay/scanners/liquid.rb | 76 ++++++++++++++++++++------------ 2 files changed, 66 insertions(+), 47 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 59bc73fc..f7a75fc1 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ def [] filename, read_shebang = false name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ def [] filename, read_shebang = false TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ def fetch filename, default = nil, read_shebang = false if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ def fetch filename, default = nil, read_shebang = false raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ def shebang filename end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -125,20 +125,21 @@ def shebang filename 'xml' => :xml, 'yaml' => :yaml, 'yml' => :yaml, + 'liquid' => :liquid, } for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index a0255712..318cd3c0 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -3,53 +3,71 @@ module Scanners load :html + # Scanner for Handlebars templates. class Liquid < Scanner - + register_for :liquid title 'Liquid Template' KINDS_NOT_LOC = HTML::KINDS_NOT_LOC - LIQUID_BLOCK = / - ({[{|%]) - \s + LIQUID_OUTPUT_BLOCK = / + ({{) + (.*?) + (}}) + /mx # :nodoc: + + LIQUID_DIRECTIVE_BLOCK = / + ({%) (.*?) - \s - ([%|}]}) - / - - START_OF_LIQUID = /{{|{%/ - - protected - + (%}) + /mx # :nodoc: + + START_OF_LIQUID = /{{|{%/ # :nodoc: + + protected + def setup - @html_scanner = CodeRay.scanner :html, tokens: @tokens, keep_tokens: true, keep_state: true - @liquid_attribute_scanner = CodeRay.scanner :html, tokens: @tokens, keep_tokens: true, keep_stat: true + @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true + @liquid_attribute_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => false + end + + def reset_instance + super + @html_scanner.reset + @liquid_attribute_scanner.reset end - - def scan_tokens(tokens, options) + + def scan_tokens encoder, options until eos? if (match = scan_until(/(?=#{START_OF_LIQUID})/o) || scan_rest) and not match.empty? - @html_scanner.tokenize match, tokens: tokens - elsif match = scan(/#{LIQUID_BLOCK}/o) + @html_scanner.tokenize match, :tokens => encoder + + elsif match = scan(/#{LIQUID_OUTPUT_BLOCK}/o) || match = scan(/#{LIQUID_DIRECTIVE_BLOCK}/o) + start_tag = self[1] code = self[2] end_tag = self[3] - - tokens.begin_group :inline - tokens.text_token start_tag, :inline_delimiter - + + encoder.begin_group :inline + encoder.text_token start_tag, :inline_delimiter + unless code.empty? - @liquid_attribute_scanner.tokenize code, tokens: tokens, state: :attribute + @liquid_attribute_scanner.tokenize code, :tokens => encoder, :state => :attribute end - - tokens.text_token end_tag, :inline_delimiter unless end_tag.empty? - tokens.end_group :inline - else - raise_inspect 'else-case reached!', tokens - end + + encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? + encoder.end_group :inline + + else + raise_inspect 'else-case reached!', encoder + end end + encoder end + end + end end + From 2dffd7eab2f214bb2d2740f8deaf2ccf2be9e0f2 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Mon, 11 Mar 2013 13:17:15 -0700 Subject: [PATCH 2/8] working directive highlighting, partial output --- "lib/coderay/scanners/\\" | 1 + lib/coderay/scanners/] | 67 +++++++++++++ lib/coderay/scanners/liquid.rb | 169 ++++++++++++++++++++++++--------- 3 files changed, 190 insertions(+), 47 deletions(-) create mode 100644 "lib/coderay/scanners/\\" create mode 100644 lib/coderay/scanners/] diff --git "a/lib/coderay/scanners/\\" "b/lib/coderay/scanners/\\" new file mode 100644 index 00000000..0bd9b6b2 --- /dev/null +++ "b/lib/coderay/scanners/\\" @@ -0,0 +1 @@ +0: html diff --git a/lib/coderay/scanners/] b/lib/coderay/scanners/] new file mode 100644 index 00000000..7e97581d --- /dev/null +++ b/lib/coderay/scanners/] @@ -0,0 +1,67 @@ +module CodeRay +module Scanners + + class Liquid < Scanner + + register_for :liquid + + KEYWORDS = %w[ list endlist for endfor wrap endwrap if endif unless endunless elsif assign cycle capture endcapture fill iflist endiflist else ] + + KEYWORDS_REGEX = /( + list| + endlist| + for| + endfor| + wrap| + endwrap| + if| + endif| + unless| + endunless| + elsif| + assign| + cycle| + capture| + end| + capture| + fill| + iflist| + endiflist| + else + )/ + + LIQUID_DIRECTIVE_BLOCK = / + ({%\s) + (.*?) + (\s%}) + /mx # :nodoc: + + def setup + #@liquid_scanner = Scanners[:liquid].new(keep_tokens: true, keep_state: true) + end + + def scan_tokens(encoder, options) + until eos? + + if match = scan(/ \s+ /x) + encoder.text_token match, :space + elsif match = scan(/#{LIQUID_DIRECTIVE_BLOCK}/) + start_tag = self[1] + directive = self[2] + end_tag = self[3] + raise start_tag + + encoder.begin_group :directive + encoder.text_token start_tag, :delimiter + + encoder.text_token end_tag, :delimiter + encoder.end_group :directive + else + raise 'else case reached!' + end + end + encoder + end + end +end +end diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index 318cd3c0..c42128ff 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -1,73 +1,148 @@ module CodeRay module Scanners - load :html - - # Scanner for Handlebars templates. class Liquid < Scanner - + register_for :liquid - title 'Liquid Template' - - KINDS_NOT_LOC = HTML::KINDS_NOT_LOC - - LIQUID_OUTPUT_BLOCK = / - ({{) - (.*?) - (}}) - /mx # :nodoc: + + DIRECTIVE_KEYWORDS = /( + list| + endlist| + for| + endfor| + wrap| + endwrap| + if| + endif| + unless| + endunless| + elsif| + assign| + cycle| + capture| + end| + capture| + fill| + iflist| + endiflist| + else| + =| + ==| + !=| + >| + <| + <=| + >= + )/ + + FILTER_KEYWORDS = /( + date| + capitalize| + downcase| + upcase| + first| + last| + join| + sort| + map| + size| + escape| + escape_once| + strip_html| + strip_newlines| + newline_to_br| + replace| + replace_first| + remove| + remove_first| + truncate| + truncatewords| + prepend| + append| + minus| + plus| + times| + divided_by| + split| + modulo + )/ LIQUID_DIRECTIVE_BLOCK = / - ({%) + {% (.*?) - (%}) - /mx # :nodoc: - - START_OF_LIQUID = /{{|{%/ # :nodoc: - - protected + %} + / def setup - @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true - @liquid_attribute_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => false + @html_scanner = CodeRay.scanner(:html, tokens: @tokens, keep_tokens: true, keep_state: false) end - def reset_instance - super - @html_scanner.reset - @liquid_attribute_scanner.reset + def debug(match, debug_cycle, state) + raise "Match: #{match}, left to scan: '#{post_match}', cycle: #{debug_cycle.to_s}, state: #{state.to_s}." end - def scan_tokens encoder, options - until eos? - if (match = scan_until(/(?=#{START_OF_LIQUID})/o) || scan_rest) and not match.empty? - @html_scanner.tokenize match, :tokens => encoder - - elsif match = scan(/#{LIQUID_OUTPUT_BLOCK}/o) || match = scan(/#{LIQUID_DIRECTIVE_BLOCK}/o) - - start_tag = self[1] - code = self[2] - end_tag = self[3] + def scan_spaces(encoder) + if match = scan(/\s+/) + encoder.text_token match, :space + end + end - encoder.begin_group :inline - encoder.text_token start_tag, :inline_delimiter + def scan_directive(encoder, options, match) + encoder.text_token match, :key + state = :liquid + scan_spaces(encoder) + #This should use the DIRECTIVE_KEYWORDS regex, not sure why it doesn't work + if match = scan(/(wrap|endwrap)/) + encoder.text_token match, :directive + if match = scan(/(\w+)(:)('\D+')/) + selector = match[0] + delimiter = match[1] + variable = match[2] + + encoder.text_token selector, :operator + encoder.text_token delimiter, :delimiter + encoder.text_token variable, :variable + end + end + scan_spaces(encoder) + if match = scan(/(%}|}})/) + encoder.text_token match, :key + state = :initial + end + end - unless code.empty? - @liquid_attribute_scanner.tokenize code, :tokens => encoder, :state => :attribute - end + def scan_output(encoder, options, match) + encoder.text_token match, :key + state = :liquid + scan_spaces(encoder) + if match = scan(/\D+/) + encoder.text_token match, :variable + end + if scan(/\|/) + end + end - encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? - encoder.end_group :inline + def scan_tokens(encoder, options) + state = :initial + debug_cycle = 0 + until eos? + if (match = scan_until(/(?=({{|{%))/) || scan_rest) and not match.empty? and state != :liquid + @html_scanner.tokenize(match, tokens: encoder) + debug match, debug_cycle, state if debug_cycle == 1 + state = :initial + scan_spaces(encoder) + elsif match = scan(/({%|{{)/) + scan_directive(encoder, options, match) + elsif match = scan(/({{|{{)/) + scan_output(encoder, options, match) else - raise_inspect 'else-case reached!', encoder + raise "Else-case reached. #{debug_cycle.to_s} cycles run. State: #{state.to_s}." end + debug_cycle += 1 end encoder end - end - end end - From 1b4bc58b27019457c6188aee47f066c2a8a6e311 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Mon, 11 Mar 2013 15:58:27 -0700 Subject: [PATCH 3/8] working scanner --- lib/coderay/scanners/liquid.rb | 64 ++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index c42128ff..aecee51e 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -26,13 +26,18 @@ class Liquid < Scanner iflist| endiflist| else| + )/ + + DIRECTIVE_OPERATORS = /( =| ==| !=| >| <| <=| - >= + >=| + contains| + with )/ FILTER_KEYWORDS = /( @@ -67,6 +72,8 @@ class Liquid < Scanner modulo )/ + SELECTORS = + LIQUID_DIRECTIVE_BLOCK = / {% (.*?) @@ -94,32 +101,60 @@ def scan_directive(encoder, options, match) #This should use the DIRECTIVE_KEYWORDS regex, not sure why it doesn't work if match = scan(/(wrap|endwrap)/) encoder.text_token match, :directive - if match = scan(/(\w+)(:)('\D+')/) - selector = match[0] - delimiter = match[1] - variable = match[2] - - encoder.text_token selector, :operator - encoder.text_token delimiter, :delimiter - encoder.text_token variable, :variable + scan_spaces(encoder) + #Replace with DIRECTIVE_OPERATORS + if match = scan(/with/) + encoder.text_token match, :operator + if delimiter = scan(/:/) + encoder.text_token delimiter, :delimiter + scan_spaces(encoder) + end + if variable = scan(/(\w+)|('\S+')|("\w+")/) + encoder.text_token variable, :variable + end end end scan_spaces(encoder) - if match = scan(/(%}|}})/) + if match = scan(/%}/) encoder.text_token match, :key state = :initial end end + def scan_output_filters(encoder, options, match) + encoder.text_token match, :delimiter + scan_spaces(encoder) + #Replace with OUTPUT_KEYWORDS regex + if directive = scan(/prepend|replace_first/) + encoder.text_token directive, :directive + end + if delimiter = scan(/:/) + encoder.text_token delimiter, :delimiter + end + scan_spaces(encoder) + if variable = scan(/(\w+)|('\S+')|(".+")/) + encoder.text_token variable, :variable + end + if next_filter = scan(/\s\|\s/) + scan_output_filters(encoder, options, next_filter) + end + end + def scan_output(encoder, options, match) encoder.text_token match, :key state = :liquid scan_spaces(encoder) - if match = scan(/\D+/) + if match = scan(/(\w+)|('\S+')|("\w+")/) encoder.text_token match, :variable end - if scan(/\|/) + if match = scan(/(\s\|\s)/) + scan_output_filters(encoder, options, match) end + scan_spaces(encoder) + if match = scan(/}}/) + encoder.text_token match, :key + end + state = :initial end def scan_tokens(encoder, options) @@ -129,12 +164,11 @@ def scan_tokens(encoder, options) until eos? if (match = scan_until(/(?=({{|{%))/) || scan_rest) and not match.empty? and state != :liquid @html_scanner.tokenize(match, tokens: encoder) - debug match, debug_cycle, state if debug_cycle == 1 state = :initial scan_spaces(encoder) - elsif match = scan(/({%|{{)/) + elsif match = scan(/{%/) scan_directive(encoder, options, match) - elsif match = scan(/({{|{{)/) + elsif match = scan(/{{/) scan_output(encoder, options, match) else raise "Else-case reached. #{debug_cycle.to_s} cycles run. State: #{state.to_s}." From 799227ac7c4bc9b96f4946802d352dfe5fb01028 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Thu, 14 Mar 2013 12:39:58 -0700 Subject: [PATCH 4/8] finish out scanner (still needs some fixes) --- "lib/coderay/scanners/\\" | 1 - lib/coderay/scanners/] | 67 --------------------------- lib/coderay/scanners/liquid.rb | 84 ++++++++++++++++++++++------------ 3 files changed, 56 insertions(+), 96 deletions(-) delete mode 100644 "lib/coderay/scanners/\\" delete mode 100644 lib/coderay/scanners/] diff --git "a/lib/coderay/scanners/\\" "b/lib/coderay/scanners/\\" deleted file mode 100644 index 0bd9b6b2..00000000 --- "a/lib/coderay/scanners/\\" +++ /dev/null @@ -1 +0,0 @@ -0: html diff --git a/lib/coderay/scanners/] b/lib/coderay/scanners/] deleted file mode 100644 index 7e97581d..00000000 --- a/lib/coderay/scanners/] +++ /dev/null @@ -1,67 +0,0 @@ -module CodeRay -module Scanners - - class Liquid < Scanner - - register_for :liquid - - KEYWORDS = %w[ list endlist for endfor wrap endwrap if endif unless endunless elsif assign cycle capture endcapture fill iflist endiflist else ] - - KEYWORDS_REGEX = /( - list| - endlist| - for| - endfor| - wrap| - endwrap| - if| - endif| - unless| - endunless| - elsif| - assign| - cycle| - capture| - end| - capture| - fill| - iflist| - endiflist| - else - )/ - - LIQUID_DIRECTIVE_BLOCK = / - ({%\s) - (.*?) - (\s%}) - /mx # :nodoc: - - def setup - #@liquid_scanner = Scanners[:liquid].new(keep_tokens: true, keep_state: true) - end - - def scan_tokens(encoder, options) - until eos? - - if match = scan(/ \s+ /x) - encoder.text_token match, :space - elsif match = scan(/#{LIQUID_DIRECTIVE_BLOCK}/) - start_tag = self[1] - directive = self[2] - end_tag = self[3] - raise start_tag - - encoder.begin_group :directive - encoder.text_token start_tag, :delimiter - - encoder.text_token end_tag, :delimiter - encoder.end_group :directive - else - raise 'else case reached!' - end - end - encoder - end - end -end -end diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index aecee51e..6e688512 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -5,7 +5,7 @@ class Liquid < Scanner register_for :liquid - DIRECTIVE_KEYWORDS = /( + DIRECTIVE_KEYWORDS = / list| endlist| for| @@ -26,9 +26,9 @@ class Liquid < Scanner iflist| endiflist| else| - )/ + /x - DIRECTIVE_OPERATORS = /( + DIRECTIVE_OPERATORS = / =| ==| !=| @@ -37,10 +37,19 @@ class Liquid < Scanner <=| >=| contains| - with - )/ + /x - FILTER_KEYWORDS = /( + MATH = / + =| + ==| + !=| + >| + <| + <=| + >| + /x + + FILTER_KEYWORDS = / date| capitalize| downcase| @@ -70,50 +79,67 @@ class Liquid < Scanner divided_by| split| modulo - )/ - - SELECTORS = + /x LIQUID_DIRECTIVE_BLOCK = / {% (.*?) %} - / + /x def setup @html_scanner = CodeRay.scanner(:html, tokens: @tokens, keep_tokens: true, keep_state: false) end - def debug(match, debug_cycle, state) - raise "Match: #{match}, left to scan: '#{post_match}', cycle: #{debug_cycle.to_s}, state: #{state.to_s}." - end - def scan_spaces(encoder) if match = scan(/\s+/) encoder.text_token match, :space end end + def scan_selector(encoder, options, match) + scan_spaces(encoder) + if match = scan(/in|with/) + Rails.logger.debug 'DEBUG: Scanning selector' + scan_spaces(encoder) + encoder.text_token match, :type + if delimiter = scan(/:/) + encoder.text_token delimiter, :delimiter + scan_spaces(encoder) + end + if variable = scan(/(\w+)|('\S+')|("\w+")/) + encoder.text_token variable, :variable + end + scan_selector(encoder, options, match) + end + end + def scan_directive(encoder, options, match) + Rails.logger.debug 'DEBUG: Scanning directive' encoder.text_token match, :key state = :liquid scan_spaces(encoder) - #This should use the DIRECTIVE_KEYWORDS regex, not sure why it doesn't work - if match = scan(/(wrap|endwrap)/) + #Replace with DIRECTIVES_KEYWORDS regex + if match = scan(/wrap|if|endif|endwrap/) encoder.text_token match, :directive scan_spaces(encoder) - #Replace with DIRECTIVE_OPERATORS - if match = scan(/with/) - encoder.text_token match, :operator - if delimiter = scan(/:/) - encoder.text_token delimiter, :delimiter + if match =~ /if/ + if match = scan(/\w+\.?\w*/) + encoder.text_token match, :variable + end + scan_spaces(encoder) + #Replace with MATH regex + if match = scan(/!=/) + encoder.text_token match, :char scan_spaces(encoder) end - if variable = scan(/(\w+)|('\S+')|("\w+")/) - encoder.text_token variable, :variable + if match = scan(/(\w+)|('\S+')|(".+")/) + encoder.text_token match, :variable + scan_spaces(encoder) end end end + scan_selector(encoder, options, match) scan_spaces(encoder) if match = scan(/%}/) encoder.text_token match, :key @@ -124,8 +150,10 @@ def scan_directive(encoder, options, match) def scan_output_filters(encoder, options, match) encoder.text_token match, :delimiter scan_spaces(encoder) - #Replace with OUTPUT_KEYWORDS regex - if directive = scan(/prepend|replace_first/) + #Replace with FILTER_KEYWORDS regex + testx = /replace_first|prepend/ + if directive = scan(/#{testx}/) + #if directive = scan(/#{FILTER_KEYWORDS}/) encoder.text_token directive, :directive end if delimiter = scan(/:/) @@ -141,6 +169,7 @@ def scan_output_filters(encoder, options, match) end def scan_output(encoder, options, match) + Rails.logger.debug 'DEBUG: Scanning output' encoder.text_token match, :key state = :liquid scan_spaces(encoder) @@ -158,8 +187,8 @@ def scan_output(encoder, options, match) end def scan_tokens(encoder, options) + Rails.logger.debug "DEBUG: Scan started: #{self.string}" state = :initial - debug_cycle = 0 until eos? if (match = scan_until(/(?=({{|{%))/) || scan_rest) and not match.empty? and state != :liquid @@ -171,9 +200,8 @@ def scan_tokens(encoder, options) elsif match = scan(/{{/) scan_output(encoder, options, match) else - raise "Else-case reached. #{debug_cycle.to_s} cycles run. State: #{state.to_s}." + raise "Else-case reached. State: #{state.to_s}." end - debug_cycle += 1 end encoder end From 253fca4da75785a0b3da53bf0bb264b8a9d5cab2 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Fri, 15 Mar 2013 12:09:08 -0700 Subject: [PATCH 5/8] change regexes constants to strings, still broken --- lib/coderay/scanners/liquid.rb | 88 ++++------------------------------ 1 file changed, 8 insertions(+), 80 deletions(-) diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index 6e688512..5c95cce6 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -5,81 +5,13 @@ class Liquid < Scanner register_for :liquid - DIRECTIVE_KEYWORDS = / - list| - endlist| - for| - endfor| - wrap| - endwrap| - if| - endif| - unless| - endunless| - elsif| - assign| - cycle| - capture| - end| - capture| - fill| - iflist| - endiflist| - else| - /x + DIRECTIVE_KEYWORDS = "list|endlist|for|endfor|wrap|endwrap|if|endif|unless|endunless|elsif|assign|cycle|capture|end|capture|fill|iflist|endiflist|else" - DIRECTIVE_OPERATORS = / - =| - ==| - !=| - >| - <| - <=| - >=| - contains| - /x + DIRECTIVE_OPERATORS = "=|==|!=|>|<|<=|>=|contains|" - MATH = / - =| - ==| - !=| - >| - <| - <=| - >| - /x + MATH = "=|==|!=|>|<|<=|>|" - FILTER_KEYWORDS = / - date| - capitalize| - downcase| - upcase| - first| - last| - join| - sort| - map| - size| - escape| - escape_once| - strip_html| - strip_newlines| - newline_to_br| - replace| - replace_first| - remove| - remove_first| - truncate| - truncatewords| - prepend| - append| - minus| - plus| - times| - divided_by| - split| - modulo - /x + FILTER_KEYWORDS = "date|capitalize|downcase|upcase|first|last|join|sort|map|size|escape|escape_once|strip_html|strip_newlines|newline_to_br|replace|replace_first|remove|remove_first|truncate|truncatewords|prepend|append|minus|plus|times|divided_by|split|modulo" LIQUID_DIRECTIVE_BLOCK = / {% @@ -119,8 +51,8 @@ def scan_directive(encoder, options, match) encoder.text_token match, :key state = :liquid scan_spaces(encoder) - #Replace with DIRECTIVES_KEYWORDS regex - if match = scan(/wrap|if|endif|endwrap/) + #This regex doesn't work and I don't know why + if match = scan(/#{DIRECTIVES_KEYWORDS}/) encoder.text_token match, :directive scan_spaces(encoder) if match =~ /if/ @@ -128,8 +60,7 @@ def scan_directive(encoder, options, match) encoder.text_token match, :variable end scan_spaces(encoder) - #Replace with MATH regex - if match = scan(/!=/) + if match = scan(/#{MATH}/) encoder.text_token match, :char scan_spaces(encoder) end @@ -150,10 +81,7 @@ def scan_directive(encoder, options, match) def scan_output_filters(encoder, options, match) encoder.text_token match, :delimiter scan_spaces(encoder) - #Replace with FILTER_KEYWORDS regex - testx = /replace_first|prepend/ - if directive = scan(/#{testx}/) - #if directive = scan(/#{FILTER_KEYWORDS}/) + if directive = scan(/#{FILTER_KEYWORDS}/) encoder.text_token directive, :directive end if delimiter = scan(/:/) From 1e8f0e6c5a7d297bf9b26cc9a6861c1d89667c5f Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Thu, 21 Mar 2013 16:50:50 -0700 Subject: [PATCH 6/8] fix typo in var name --- lib/coderay/scanners/liquid.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index 5c95cce6..ceb81dde 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -52,7 +52,7 @@ def scan_directive(encoder, options, match) state = :liquid scan_spaces(encoder) #This regex doesn't work and I don't know why - if match = scan(/#{DIRECTIVES_KEYWORDS}/) + if match = scan(/#{DIRECTIVE_KEYWORDS}/) encoder.text_token match, :directive scan_spaces(encoder) if match =~ /if/ From 7a19124113f7d827da0e340ad70b6af6aeaf5730 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Thu, 21 Mar 2013 17:25:51 -0700 Subject: [PATCH 7/8] last fixes --- lib/coderay/scanners/liquid.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index ceb81dde..c6cca777 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -7,11 +7,11 @@ class Liquid < Scanner DIRECTIVE_KEYWORDS = "list|endlist|for|endfor|wrap|endwrap|if|endif|unless|endunless|elsif|assign|cycle|capture|end|capture|fill|iflist|endiflist|else" - DIRECTIVE_OPERATORS = "=|==|!=|>|<|<=|>=|contains|" + DIRECTIVE_OPERATORS = "=|==|!=|>|<|<=|>=|contains" - MATH = "=|==|!=|>|<|<=|>|" + MATH = "=|==|!=|>|<|<=|>" - FILTER_KEYWORDS = "date|capitalize|downcase|upcase|first|last|join|sort|map|size|escape|escape_once|strip_html|strip_newlines|newline_to_br|replace|replace_first|remove|remove_first|truncate|truncatewords|prepend|append|minus|plus|times|divided_by|split|modulo" + FILTER_KEYWORDS = "date|capitalize|downcase|upcase|first|last|join|sort|map|size|escape_once|escape|strip_html|strip_newlines|newline_to_br|replace_first|replace|remove_first|remove|truncate|truncatewords|prepend|append|minus|plus|times|divided_by|split|modulo" LIQUID_DIRECTIVE_BLOCK = / {% From 19d04e45f2af075c08f2715ebd7a169eab611f65 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Thu, 28 Mar 2013 11:51:52 -0700 Subject: [PATCH 8/8] remove debug statements --- lib/coderay/scanners/liquid.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index c6cca777..93a546e7 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -32,7 +32,6 @@ def scan_spaces(encoder) def scan_selector(encoder, options, match) scan_spaces(encoder) if match = scan(/in|with/) - Rails.logger.debug 'DEBUG: Scanning selector' scan_spaces(encoder) encoder.text_token match, :type if delimiter = scan(/:/) @@ -47,7 +46,6 @@ def scan_selector(encoder, options, match) end def scan_directive(encoder, options, match) - Rails.logger.debug 'DEBUG: Scanning directive' encoder.text_token match, :key state = :liquid scan_spaces(encoder) @@ -97,7 +95,6 @@ def scan_output_filters(encoder, options, match) end def scan_output(encoder, options, match) - Rails.logger.debug 'DEBUG: Scanning output' encoder.text_token match, :key state = :liquid scan_spaces(encoder) @@ -115,7 +112,6 @@ def scan_output(encoder, options, match) end def scan_tokens(encoder, options) - Rails.logger.debug "DEBUG: Scan started: #{self.string}" state = :initial until eos?