From 501df7613c67bcd743eaa00071f0dbb179607394 Mon Sep 17 00:00:00 2001 From: Quintus Date: Sun, 22 Apr 2012 22:10:42 +0200 Subject: [PATCH 001/236] Lua scanner for CodeRay. Meta-commit. This commit is a super-commit containing all the subcommits for implementing the Lua scanner. --- lib/coderay/scanners/lua.rb | 267 ++++++++++++++++++++++++++++++++++++ lib/coderay/styles/alpha.rb | 3 + lib/coderay/token_kinds.rb | 1 + 3 files changed, 271 insertions(+) create mode 100644 lib/coderay/scanners/lua.rb diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb new file mode 100644 index 00000000..e7706fc1 --- /dev/null +++ b/lib/coderay/scanners/lua.rb @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- + +# Scanner for the Lua[http://lua.org] programming lanuage. +# +# The language’s complete syntax is defined in +# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], +# which is what this scanner tries to conform to. +class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident) + .add(KEYWORDS, :keyword) + .add(PREDEFINED_CONSTANTS, :predefined_constant) + .add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + @encoder = encoder + @options = options + + send(:"handle_state_#@state") until eos? + + @encoder + end + + def handle_state_initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:comment) + @encoder.text_token(match, :delimiter) + @state = :long_comment + + elsif match = scan(/--.*?$/) # --Lua comment + @encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + @encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + @encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + @state = :function_expected + elsif kind == :keyword and match == "goto" + @state = :goto_label_expected + elsif kind == :keyword and match == "local" + @state = :local_var_expected + end + + @encoder.text_token(match, kind) + + elsif match = scan(/{/) # Opening table brace { + @encoder.begin_group(:table) + @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + @state = :table + + elsif match = scan(/}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + @encoder.text_token(match, :delimiter) + elsif @brace_depth == 0 # Mismatched brace + @encoder.text_token(match, :error) + else + @brace_depth -= 1 + @encoder.text_token(match, :inline_delimiter) + @state = :table + end + @encoder.end_group(:table) + + elsif match = scan(/["']/) # String delimiters " and ' + @encoder.begin_group(:string) + @encoder.text_token(match, :delimiter) + @start_delim = match + @state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + @encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + @encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + @encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + @encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :table scanning state. + @state = :table if @state == :initial && @brace_depth >= 1 + end + + def handle_state_function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + @encoder.text_token(match, :operator) + @state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + @encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + @encoder.text_token(match, :function) + @state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + @state = :initial + end + end + + def handle_state_goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :label) + @state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_local_var_expected + if match = scan(/function/) # local function ... + @encoder.text_token(match, :keyword) + @state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + @encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + @encoder.text_token(match, :operator) + elsif match = scan(/=/) + @encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + @state = :initial + elsif match = scan(/\n/) + @encoder.text_token(match, :space) + @state = :initial + elsif match = scan(/\s+/) + @encoder.text_token(match, :space) + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:comment) + @state = :initial + end + + def handle_state_long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + @encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + @encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + @encoder.text_token(rest, :error) + terminate + end + @encoder.end_group(:string) + @state = :initial + end + + def handle_state_string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + @encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + @encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + @encoder.text_token(match, :delimiter) + @encoder.end_group(:string) + @state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + @encoder.end_group(:string) + @state = :initial + else + @encoder.text_token(getch, :error) + end + end + + def handle_state_table + if match = scan(/[,;]/) + @encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + @encoder.text_token(match, :key) + @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + @encoder.text_token(scan(/=/), :operator) + @state = :initial + elsif match = scan(/\s+/m) + @encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + @state = :initial + end + end + +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 8506d103..257083e5 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -116,6 +116,9 @@ class Alpha < Style .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } +.table .content { color:#808 } +.table .delimiter { color:#40A} +.table { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 3b8d07e4..e2456235 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -63,6 +63,7 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', + :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', From 8979cc621431248fded86c341e2102a67c5344bb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 17:45:49 +0200 Subject: [PATCH 002/236] use case+when instead of send and methods --- lib/coderay/scanners/lua.rb | 64 ++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc1..e7120612 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -57,13 +57,11 @@ def setup def scan_tokens(encoder, options) @encoder = encoder @options = options - - send(:"handle_state_#@state") until eos? - - @encoder - end - - def handle_state_initial + + until eos? + case state + + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @encoder.begin_group(:comment) @@ -99,13 +97,13 @@ def handle_state_initial @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) @@ -146,9 +144,8 @@ def handle_state_initial # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - end - - def handle_state_function_expected + + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @state = :initial @@ -163,9 +160,8 @@ def handle_state_function_expected @encoder.text_token(getch, :error) @state = :initial end - end - def handle_state_goto_label_expected + when :goto_label_expected if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) @encoder.text_token(match, :label) @state = :initial @@ -174,9 +170,8 @@ def handle_state_goto_label_expected else @encoder.text_token(getch, :error) end - end - - def handle_state_local_var_expected + + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @state = :function_expected @@ -198,9 +193,8 @@ def handle_state_local_var_expected else @encoder.text_token(getch, :error) end - end - - def handle_state_long_comment + + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -212,9 +206,8 @@ def handle_state_long_comment end @encoder.end_group(:comment) @state = :initial - end - - def handle_state_long_string + + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -226,9 +219,8 @@ def handle_state_long_string end @encoder.end_group(:string) @state = :initial - end - - def handle_state_string + + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) @@ -244,9 +236,8 @@ def handle_state_string else @encoder.text_token(getch, :error) end - end - - def handle_state_table + + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @@ -262,6 +253,13 @@ def handle_state_table # advances the pointer). @state = :initial end + else + raise + end + + end + + @encoder end end From 2b16d115f7d6caf21864934df763556e126d6357 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 19 Jun 2012 18:05:04 +0200 Subject: [PATCH 003/236] fix for Ruby 1.8, escape { and } in regexps --- lib/coderay/scanners/lua.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7706fc1..8464ef7d 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -40,10 +40,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner ] # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident) - .add(KEYWORDS, :keyword) - .add(PREDEFINED_CONSTANTS, :predefined_constant) - .add(PREDEFINED_EXPRESSIONS, :predefined) + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) protected @@ -99,13 +99,13 @@ def handle_state_initial @encoder.text_token(match, kind) - elsif match = scan(/{/) # Opening table brace { + elsif match = scan(/\{/) # Opening table brace { @encoder.begin_group(:table) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 @state = :table - elsif match = scan(/}/) # Closing table brace } + elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @brace_depth = 0 @encoder.text_token(match, :delimiter) From abb92f30b12e11781afa76f43a344627520b5b34 Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Sun, 8 Jul 2012 14:32:28 +0800 Subject: [PATCH 004/236] New: *Go Encoder* Draft version, copy from c --- lib/coderay/helpers/file_type.rb | 37 +++--- lib/coderay/scanners/go.rb | 195 +++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 lib/coderay/scanners/go.rb diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 637001b8..51590544 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ def [] filename, read_shebang = false name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ def [] filename, read_shebang = false TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ def fetch filename, default = nil, read_shebang = false if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ def fetch filename, default = nil, read_shebang = false raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ def shebang filename end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -86,6 +86,7 @@ def shebang filename 'dpr' => :delphi, 'erb' => :erb, 'gemspec' => :ruby, + 'go' => :go, 'groovy' => :groovy, 'gvy' => :groovy, 'h' => :c, @@ -128,16 +129,16 @@ def shebang filename for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb new file mode 100644 index 00000000..4431ef2a --- /dev/null +++ b/lib/coderay/scanners/go.rb @@ -0,0 +1,195 @@ +module CodeRay +module Scanners + + # Scanner for Go, copy from c + class Go < Scanner + + register_for :go + file_extension 'go' + + # http://golang.org/ref/spec#Keywords + KEYWORDS = [ + 'break', 'default', 'func', 'interface', 'select', + 'case', 'defer', 'go', 'map', 'struct', + 'chan', 'else', 'goto', 'package', 'switch', + 'const', 'fallthrough', 'if', 'range', 'type', + 'continue', 'for', 'import', 'return', 'var', + ] # :nodoc: + + # http://golang.org/ref/spec#Types + PREDEFINED_TYPES = [ + 'bool', + 'uint8', 'uint16', 'uint32', 'uint64', + 'int8', 'int16', 'int32', 'int64', + 'float32', 'float64', + 'complex64', 'complex128', + 'byte', 'rune', + 'uint', 'int', 'uintptr', + ] # :nodoc: + + PREDEFINED_CONSTANTS = [ + 'nil', 'iota', + 'true', 'false', + ] # :nodoc: + + DIRECTIVES = [ + 'go_no_directive', # Seems no directive concept in Go? + ] # :nodoc: + + IDENT_KIND = WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_TYPES, :predefined_type). + add(DIRECTIVES, :directive). + add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + + protected + + def scan_tokens encoder, options + + state = :initial + label_expected = true + case_expected = false + label_expected_before_preproc_line = nil + in_preproc_line = false + + until eos? + + case state + + when :initial + + if match = scan(/ \s+ | \\\n /x) + if in_preproc_line && match != "\\\n" && match.index(?\n) + in_preproc_line = false + label_expected = label_expected_before_preproc_line + end + encoder.text_token match, :space + + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + encoder.text_token match, :comment + + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + label_expected = match =~ /[;\{\}]/ + if case_expected + label_expected = true if match == ':' + case_expected = false + end + encoder.text_token match, :operator + + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) + kind = IDENT_KIND[match] + if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) + kind = :label + match << matched + else + label_expected = false + if kind == :keyword + case match + when 'case', 'default' + case_expected = true + end + end + end + encoder.text_token match, kind + + elsif match = scan(/L?"/) + encoder.begin_group :string + if match[0] == ?L + encoder.text_token 'L', :modifier + match = '"' + end + encoder.text_token match, :delimiter + state = :string + + elsif match = scan(/ \# \s* if \s* 0 /x) + match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? + encoder.text_token match, :comment + + elsif match = scan(/#[ \t]*(\w*)/) + encoder.text_token match, :preprocessor + in_preproc_line = true + label_expected_before_preproc_line = label_expected + state = :include_expected if self[1] == 'include' + + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + label_expected = false + encoder.text_token match, :char + + elsif match = scan(/\$/) + encoder.text_token match, :ident + + elsif match = scan(/0[xX][0-9A-Fa-f]+/) + label_expected = false + encoder.text_token match, :hex + + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + label_expected = false + encoder.text_token match, :octal + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer + + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + label_expected = false + encoder.text_token match, :float + + else + encoder.text_token getch, :error + + end + + when :string + if match = scan(/[^\\\n"]+/) + encoder.text_token match, :content + elsif match = scan(/"/) + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + label_expected = false + elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) + encoder.text_token match, :char + elsif match = scan(/ \\ | $ /x) + encoder.end_group :string + encoder.text_token match, :error + state = :initial + label_expected = false + else + raise_inspect "else case \" reached; %p not handled." % peek(1), encoder + end + + when :include_expected + if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) + encoder.text_token match, :include + state = :initial + + elsif match = scan(/\s+/) + encoder.text_token match, :space + state = :initial if match.index ?\n + + else + state = :initial + + end + + else + raise_inspect 'Unknown state', encoder + + end + + end + + if state == :string + encoder.end_group :string + end + + encoder + end + + end + +end +end From ec39f80865cac97a49a6fec138dedc22e5971bad Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 28 Jul 2012 13:12:37 +0200 Subject: [PATCH 005/236] fix whitespace --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 51590544..5cc74467 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ def [] filename, read_shebang = false name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ def [] filename, read_shebang = false TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ def fetch filename, default = nil, read_shebang = false if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ def fetch filename, default = nil, read_shebang = false raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ def shebang filename end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -129,16 +129,16 @@ def shebang filename for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, } - + end - + end From b081c1f3ab670301199005c04ecbc11614593285 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Tue, 11 Sep 2012 15:04:45 -0600 Subject: [PATCH 006/236] escape = in regex (my editor gets confused:-) --- lib/coderay/scanners/lua.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e7120612..e6403971 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -57,10 +57,10 @@ def setup def scan_tokens(encoder, options) @encoder = encoder @options = options - + until eos? case state - + when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] @num_equals = match.count("=") # Number must match for comment end @@ -144,7 +144,7 @@ def scan_tokens(encoder, options) # (tables can contain full expressions in parts). # If this is the case, return to :table scanning state. @state = :table if @state == :initial && @brace_depth >= 1 - + when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @encoder.text_token(match, :operator) @@ -170,7 +170,7 @@ def scan_tokens(encoder, options) else @encoder.text_token(getch, :error) end - + when :local_var_expected if match = scan(/function/) # local function ... @encoder.text_token(match, :keyword) @@ -179,7 +179,7 @@ def scan_tokens(encoder, options) @encoder.text_token(match, :local_variable) elsif match = scan(/,/) @encoder.text_token(match, :operator) - elsif match = scan(/=/) + elsif match = scan(/\=/) @encoder.text_token(match, :operator) # After encountering the equal sign, arbitrary expressions are # allowed again, so just return to the main state for further @@ -193,7 +193,7 @@ def scan_tokens(encoder, options) else @encoder.text_token(getch, :error) end - + when :long_comment if match = scan(/.*?(?=\]={#@num_equals}\])/m) @encoder.text_token(match, :content) @@ -206,7 +206,7 @@ def scan_tokens(encoder, options) end @encoder.end_group(:comment) @state = :initial - + when :long_string if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences @encoder.text_token(match, :content) @@ -219,7 +219,7 @@ def scan_tokens(encoder, options) end @encoder.end_group(:string) @state = :initial - + when :string if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) @encoder.text_token(match, :content) @@ -236,14 +236,14 @@ def scan_tokens(encoder, options) else @encoder.text_token(getch, :error) end - + when :table if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) @encoder.text_token(match, :key) @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/=/), :operator) + @encoder.text_token(scan(/\=/), :operator) @state = :initial elsif match = scan(/\s+/m) @encoder.text_token(match, :space) From 279348d3c2159df4ce6ac5949ada0177aa7c7159 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sat, 27 Oct 2012 23:41:52 -0600 Subject: [PATCH 007/236] :map token kind Use :map instead of :table. It's more generic, and won't be confused with the :table rendering style. --- lib/coderay/scanners/lua.rb | 14 +++++++------- lib/coderay/styles/alpha.rb | 8 ++++---- lib/coderay/token_kinds.rb | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index e6403971..64763dcd 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -98,10 +98,10 @@ def scan_tokens(encoder, options) @encoder.text_token(match, kind) elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:table) + @encoder.begin_group(:map) @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) @brace_depth += 1 - @state = :table + @state = :map elsif match = scan(/\}/) # Closing table brace } if @brace_depth == 1 @@ -112,9 +112,9 @@ def scan_tokens(encoder, options) else @brace_depth -= 1 @encoder.text_token(match, :inline_delimiter) - @state = :table + @state = :map end - @encoder.end_group(:table) + @encoder.end_group(:map) elsif match = scan(/["']/) # String delimiters " and ' @encoder.begin_group(:string) @@ -142,8 +142,8 @@ def scan_tokens(encoder, options) # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). - # If this is the case, return to :table scanning state. - @state = :table if @state == :initial && @brace_depth >= 1 + # If this is the case, return to :map scanning state. + @state = :map if @state == :initial && @brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -237,7 +237,7 @@ def scan_tokens(encoder, options) @encoder.text_token(getch, :error) end - when :table + when :map if match = scan(/[,;]/) @encoder.text_token(match, :operator) elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 257083e5..a60725b1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style @@ -116,9 +116,9 @@ class Alpha < Style .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } .symbol { color:#A60 } -.table .content { color:#808 } -.table .delimiter { color:#40A} -.table { background-color:hsla(200,100%,50%,0.06); } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070 } .type { color:#339; font-weight:bold } .value { color: #088; } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index e2456235..a0ad5e4b 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :annotation => 'annotation', :attribute_name => 'attribute-name', @@ -50,6 +50,7 @@ module CodeRay :keyword => 'keyword', :label => 'label', :local_variable => 'local-variable', + :map => 'map', :modifier => 'modifier', :namespace => 'namespace', :octal => 'octal', @@ -63,29 +64,28 @@ module CodeRay :shell => 'shell', :string => 'string', :symbol => 'symbol', - :table => 'table', :tag => 'tag', :type => 'type', :value => 'value', :variable => 'variable', - + :change => 'change', :delete => 'delete', :head => 'head', :insert => 'insert', - + :eyecatcher => 'eyecatcher', - + :ident => false, :operator => false, - + :space => false, :plain => false ) - + TokenKinds[:method] = TokenKinds[:function] TokenKinds[:escape] = TokenKinds[:delimiter] TokenKinds[:docstring] = TokenKinds[:comment] - + TokenKinds.freeze end From d9d6dd5f4a73363ea5d353ecda142f77ed4eba5a Mon Sep 17 00:00:00 2001 From: Gavin Kistner Date: Thu, 6 Jun 2013 10:43:19 -0600 Subject: [PATCH 008/236] Support CDATA blocks in HTML/XML --- lib/coderay/scanners/html.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b795..fcf249a1 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -99,7 +99,17 @@ def scan_tokens encoder, options case state when :initial - if match = scan(/|.*)/m) + if match = scan(//m) + encoder.text_token match[0..-4], :content + encoder.text_token ']]>', :delimiter + else + encoder.text_token scan(/.*/m), :error + end + encoder.end_group :string + elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) encoder.text_token match, :doctype From d0d1b129f09dbae8fc25c093c09f72c6396b122a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Jun 2013 23:53:32 +0200 Subject: [PATCH 009/236] whitespace --- lib/coderay/styles/alpha.rb | 2 +- lib/coderay/token_kinds.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c05ffd59..c2314230 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -1,6 +1,6 @@ module CodeRay module Styles - + # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index f1696dfe..9137a49f 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,14 +1,14 @@ module CodeRay - + # A Hash of all known token kinds and their associated CSS classes. TokenKinds = Hash.new do |h, k| warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG false end - + # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity - + TokenKinds.update( # :nodoc: :debug => 'debug', # highlight for debugging (white on blue background) From 90adb394e8b4da3a733941fc764c308b34482293 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 00:15:59 +0200 Subject: [PATCH 010/236] add lua file type --- lib/coderay/helpers/file_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 19f27acd..6d4fa926 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -96,6 +96,7 @@ def shebang filename 'java' => :java, 'js' => :java_script, 'json' => :json, + 'lua' => :lua, 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, From fbaf55d69c20897655c80e9723973e2f9490484d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:04:53 +0200 Subject: [PATCH 011/236] more helpful error messages from DebugLint --- lib/coderay/encoders/debug_lint.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index eeb2a92b..0ac89eff 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -35,7 +35,7 @@ def begin_group kind end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind super end @@ -45,7 +45,7 @@ def begin_line kind end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind super end From c0dc9f2b82b84e6a7511ff9bfe3e80aaa1fef1d1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:05:41 +0200 Subject: [PATCH 012/236] move .map styles before .string --- lib/coderay/styles/alpha.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index c2314230..a869d9e1 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -99,6 +99,9 @@ class Alpha < Style .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } .local-variable { color:#963 } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .namespace { color:#707; font-weight:bold } .octal { color:#40E } .operator { } @@ -123,9 +126,6 @@ class Alpha < Style .symbol { color:#A60 } .symbol .content { color:#A60 } .symbol .delimiter { color:#630 } -.map .content { color:#808 } -.map .delimiter { color:#40A} -.map { background-color:hsla(200,100%,50%,0.06); } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } From da1425058e9b9e04bd988ddf606331d9cd0c827b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:06:33 +0200 Subject: [PATCH 013/236] cleanup Lua scanner, fix end_group(:map) issue - use local instead of instance variables --- lib/coderay/scanners/lua.rb | 524 ++++++++++++++++++------------------ 1 file changed, 267 insertions(+), 257 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 64763dcd..3bee2755 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -1,265 +1,275 @@ -# -*- coding: utf-8 -*- +# encoding: utf-8 -# Scanner for the Lua[http://lua.org] programming lanuage. -# -# The language’s complete syntax is defined in -# {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], -# which is what this scanner tries to conform to. -class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner +module CodeRay +module Scanners - register_for :lua - file_extension "lua" - title "Lua" - - # Keywords used in Lua. - KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while - ] - - # Constants set by the Lua core. - PREDEFINED_CONSTANTS = %w[false true nil] - - # The expressions contained in this array are parts of Lua’s `basic' - # library. Although it’s not entirely necessary to load that library, - # it is highly recommended and one would have to provide own implementations - # of some of these expressions if one does not do so. They however aren’t - # keywords, neither are they constants, but nearly predefined, so they - # get tagged as `predefined' rather than anything else. + # Scanner for the Lua[http://lua.org] programming lanuage. # - # This list excludes values of form `_UPPERCASE' because the Lua manual - # requires such identifiers to be reserved by Lua anyway and they are - # highlighted directly accordingly, without the need for specific - # identifiers to be listed here. - PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall - ] - - # Automatic token kind selection for normal words. - IDENT_KIND = CodeRay::WordList.new(:ident). - add(KEYWORDS, :keyword). - add(PREDEFINED_CONSTANTS, :predefined_constant). - add(PREDEFINED_EXPRESSIONS, :predefined) - - protected - - # Scanner initialization. - def setup - @state = :initial - @brace_depth = 0 - end - - # CodeRay entry hook. Starts parsing. - def scan_tokens(encoder, options) - @encoder = encoder - @options = options - - until eos? - case state - - when :initial - if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:comment) - @encoder.text_token(match, :delimiter) - @state = :long_comment - - elsif match = scan(/--.*?$/) # --Lua comment - @encoder.text_token(match, :comment) - - elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @state = :long_string - - elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: - @encoder.text_token(match, :label) - - elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua - @encoder.text_token(match, :predefined) - - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) - kind = IDENT_KIND[match] - - # Extra highlighting for entities following certain keywords - if kind == :keyword and match == "function" - @state = :function_expected - elsif kind == :keyword and match == "goto" - @state = :goto_label_expected - elsif kind == :keyword and match == "local" - @state = :local_var_expected + # The language’s complete syntax is defined in + # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], + # which is what this scanner tries to conform to. + class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + + register_for :lua + file_extension "lua" + title "Lua" + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + state = options[:state] || @state + + until eos? + case state + + when :initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:comment) + encoder.text_token(match, :delimiter) + state = :long_comment + + elsif match = scan(/--.*$/) # --Lua comment + encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + @num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + state = :function_expected + elsif kind == :keyword and match == "goto" + state = :goto_label_expected + elsif kind == :keyword and match == "local" + state = :local_var_expected + end + + encoder.text_token(match, kind) + + elsif match = scan(/\{/) # Opening table brace { + encoder.begin_group(:map) + encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) + @brace_depth += 1 + state = :map + + elsif match = scan(/\}/) # Closing table brace } + if @brace_depth == 1 + @brace_depth = 0 + encoder.text_token(match, :delimiter) + encoder.end_group(:map) + elsif @brace_depth == 0 # Mismatched brace + encoder.text_token(match, :error) + else + @brace_depth -= 1 + encoder.text_token(match, :inline_delimiter) + encoder.end_group(:map) + state = :map + end + + elsif match = scan(/["']/) # String delimiters " and ' + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + @start_delim = match + state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :map scanning state. + state = :map if state == :initial && @brace_depth >= 1 + + when :function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + encoder.text_token(match, :operator) + state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + encoder.text_token(match, :function) + state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + state = :initial + end + + when :goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :label) + state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :local_var_expected + if match = scan(/function/) # local function ... + encoder.text_token(match, :keyword) + state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + encoder.text_token(match, :operator) + elsif match = scan(/\=/) + encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + state = :initial + elsif match = scan(/\n/) + encoder.text_token(match, :space) + state = :initial + elsif match = scan(/\s+/) + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :long_comment + if match = scan(/.*?(?=\]={#@num_equals}\])/m) + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:comment) + state = :initial + + when :long_string + if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + encoder.text_token(match, :content) + + delim = scan(/\]={#@num_equals}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:string) + state = :initial + + when :string + if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(@start_delim)) + encoder.text_token(match, :delimiter) + encoder.end_group(:string) + state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + encoder.end_group(:string) + state = :initial + else + encoder.text_token(getch, :error) + end + + when :map + if match = scan(/[,;]/) + encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + encoder.text_token(match, :key) + encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + encoder.text_token(scan(/\=/), :operator) + state = :initial + elsif match = scan(/\s+/m) + encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + state = :initial + end + else + raise + end + end - - @encoder.text_token(match, kind) - - elsif match = scan(/\{/) # Opening table brace { - @encoder.begin_group(:map) - @encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 - @state = :map - - elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 - @encoder.text_token(match, :delimiter) - elsif @brace_depth == 0 # Mismatched brace - @encoder.text_token(match, :error) - else - @brace_depth -= 1 - @encoder.text_token(match, :inline_delimiter) - @state = :map + + if options[:keep_state] + @state = state end - @encoder.end_group(:map) - - elsif match = scan(/["']/) # String delimiters " and ' - @encoder.begin_group(:string) - @encoder.text_token(match, :delimiter) - @start_delim = match - @state = :string - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :float) - - # ↓Prefix hex number ←|→ decimal number - elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power - @encoder.text_token(match, :integer) - - elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators - @encoder.text_token(match, :operator) - - elsif match = scan(/\s+/) # Space - @encoder.text_token(match, :space) - - else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. - @encoder.text_token(getch, :error) - end - - # It may be that we’re scanning a full-blown subexpression of a table - # (tables can contain full expressions in parts). - # If this is the case, return to :map scanning state. - @state = :map if @state == :initial && @brace_depth >= 1 - - when :function_expected - if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name - @encoder.text_token(match, :operator) - @state = :initial - elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator - @encoder.text_token(match, :ident) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() - @encoder.text_token(match, :function) - @state = :initial - elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - @state = :initial - end - - when :goto_label_expected - if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :label) - @state = :initial - elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :local_var_expected - if match = scan(/function/) # local function ... - @encoder.text_token(match, :keyword) - @state = :function_expected - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) - @encoder.text_token(match, :local_variable) - elsif match = scan(/,/) - @encoder.text_token(match, :operator) - elsif match = scan(/\=/) - @encoder.text_token(match, :operator) - # After encountering the equal sign, arbitrary expressions are - # allowed again, so just return to the main state for further - # parsing. - @state = :initial - elsif match = scan(/\n/) - @encoder.text_token(match, :space) - @state = :initial - elsif match = scan(/\s+/) - @encoder.text_token(match, :space) - else - @encoder.text_token(getch, :error) - end - - when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:comment) - @state = :initial - - when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences - @encoder.text_token(match, :content) - - delim = scan(/\]={#@num_equals}\]/) - @encoder.text_token(delim, :delimiter) - else # No terminator found till EOF - @encoder.text_token(rest, :error) - terminate - end - @encoder.end_group(:string) - @state = :initial - - when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) - @encoder.text_token(match, :content) - elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) - @encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) - @encoder.text_token(match, :delimiter) - @encoder.end_group(:string) - @state = :initial - elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings - @encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings - @encoder.end_group(:string) - @state = :initial - else - @encoder.text_token(getch, :error) - end - - when :map - if match = scan(/[,;]/) - @encoder.text_token(match, :operator) - elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) - @encoder.text_token(match, :key) - @encoder.text_token(scan(/\s+/), :space) if check(/\s+/) - @encoder.text_token(scan(/\=/), :operator) - @state = :initial - elsif match = scan(/\s+/m) - @encoder.text_token(match, :space) - else - # Note this clause doesn’t advance the scan pointer, it’s a kind of - # "retry with other options" (the :initial state then of course - # advances the pointer). - @state = :initial + + encoder end - else - raise + end - - end - - @encoder - end - + +end end From 546b489e4b3856d361a480de5f1f02cb55c7f002 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:09:14 +0200 Subject: [PATCH 014/236] fix Tokens documentation --- lib/coderay/tokens.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 54358d4e..e7bffce2 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,17 +3,16 @@ module CodeRay # The Tokens class represents a list of tokens returned from # a Scanner. It's actually just an Array with a few helper methods. # - # A token itself is not a special object, just a two-element Array - # consisting of + # A token itself is not a special object, just two elements in an Array: # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # # It looks like this: # - # ['# It looks like this', :comment] - # ['3.1415926', :float] - # ['$^', :error] + # ..., '# It looks like this', :comment, ... + # ..., '3.1415926', :float, ... + # ..., '$^', :error, ... # # Some scanners also yield sub-tokens, represented by special # token actions, for example :begin_group and :end_group. @@ -21,11 +20,11 @@ module CodeRay # The Ruby scanner, for example, splits "a string" into: # # [ - # [:begin_group, :string], - # ['"', :delimiter], - # ['a string', :content], - # ['"', :delimiter], - # [:end_group, :string] + # :begin_group, :string, + # '"', :delimiter, + # 'a string', :content, + # '"', :delimiter, + # :end_group, :string # ] # # Tokens can be used to save the output of a Scanners in a simple From 06cf0bab5975bc9069f6859cf58eed117fd9f474 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:16:30 +0200 Subject: [PATCH 015/236] changelog --- Changes.textile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes.textile b/Changes.textile index fcee29bc..930fdbca 100644 --- a/Changes.textile +++ b/Changes.textile @@ -4,6 +4,7 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1 +* New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] @@ -17,6 +18,7 @@ h2. Changes in 1.1 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] * New token type @:id@ for CSS/Sass [#27] * New token type @:done@ for Taskpaper [#39] +* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] * Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style From 90c401c9001e0a670360a55d1189e9f814ae7592 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:17:49 +0200 Subject: [PATCH 016/236] some more code cleanups before merge --- lib/coderay/scanners/lua.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 3bee2755..25bebbec 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -8,17 +8,17 @@ module Scanners # The language’s complete syntax is defined in # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], # which is what this scanner tries to conform to. - class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner + class Lua < Scanner register_for :lua - file_extension "lua" - title "Lua" + file_extension 'lua' + title 'Lua' # Keywords used in Lua. KEYWORDS = %w[and break do else elseif end - for function goto if in - local not or repeat return - then until while + for function goto if in + local not or repeat return + then until while ] # Constants set by the Lua core. @@ -36,10 +36,10 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # highlighted directly accordingly, without the need for specific # identifiers to be listed here. PREDEFINED_EXPRESSIONS = %w[ - assert collectgarbage dofile error getmetatable - ipairs load loadfile next pairs pcall print - rawequal rawget rawlen rawset select setmetatable - tonumber tostring type xpcall + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall ] # Automatic token kind selection for normal words. @@ -52,7 +52,7 @@ class CodeRay::Scanners::Lua < CodeRay::Scanners::Scanner # Scanner initialization. def setup - @state = :initial + @state = :initial @brace_depth = 0 end From fa9848b6dbd95a4a97a3e63cb99ad2d4c26516df Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:37:06 +0200 Subject: [PATCH 017/236] avoid empty tokens in CSS --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 732f9c59..6c52c8af 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -148,7 +148,7 @@ def scan_tokens encoder, options encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 167051d0..65d40b0d 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -176,7 +176,7 @@ def scan_tokens encoder, options encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function From bedd4e4fb5a18270a742e71b65f88fb8e3182050 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:40:56 +0200 Subject: [PATCH 018/236] avoid empty tokens in Raydebug --- lib/coderay/scanners/raydebug.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 7a21354c..d39d9626 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,11 +1,11 @@ module CodeRay module Scanners - + # = Debug Scanner # # Parses the output of the Encoders::Debug encoder. class Raydebug < Scanner - + register_for :raydebug file_extension 'raydebug' title 'CodeRay Token Dump' @@ -13,11 +13,11 @@ class Raydebug < Scanner protected def scan_tokens encoder, options - + opened_tokens = [] - + until eos? - + if match = scan(/\s+/) encoder.text_token match, :space @@ -26,7 +26,7 @@ def scan_tokens encoder, options encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - encoder.text_token match, kind.to_sym + encoder.text_token match, kind.to_sym unless match.empty? encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) @@ -59,8 +59,8 @@ def scan_tokens encoder, options encoder end - + end - + end end From 39c074f171eeca1c91aa89757a3f3a12fc78323b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 01:42:32 +0200 Subject: [PATCH 019/236] avoid empty tokens in YAML --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 96f4e93f..358a3f19 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -47,7 +47,7 @@ def scan_tokens encoder, options when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/) encoder.begin_group :string encoder.text_token match, :delimiter - encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx) + encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty? encoder.text_token match, :delimiter if match = scan(/"/) encoder.end_group :string next From e7df328cb49fd09a4e5d0f02e877e3270a8703ed Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:53:53 +0200 Subject: [PATCH 020/236] DebugLint check open tokens at the end --- lib/coderay/encoders/debug_lint.rb | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 0ac89eff..17a07955 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -19,11 +19,6 @@ class DebugLint < Debug EmptyToken = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def initialize options = {} - super - @opened = [] - end - def text_token text, kind raise EmptyToken, 'empty token' if text.empty? super @@ -35,7 +30,8 @@ def begin_group kind end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_group)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop super end @@ -45,7 +41,20 @@ def begin_line kind end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind} (end_line)" if @opened.pop != kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + super + end + + protected + + def setup options + super + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? super end From d6cc5767c9a9bfb4c08bcd5127145b38cb1d29de Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:31 +0200 Subject: [PATCH 021/236] HTML encoder shouldn't warn about open tokens --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index b897f5e2..20f24095 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -193,7 +193,6 @@ def setup options def finish options unless @opened.empty? - warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG @out << '' while @opened.pop @last_opened = nil end From b059a63e7ff3cd7c26dd5664bf048aa42338fbe1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:54:56 +0200 Subject: [PATCH 022/236] close open tokens in Groovy scanner --- lib/coderay/scanners/groovy.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index cf55daf2..d04a5354 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -36,9 +36,12 @@ class Groovy < Java protected + def setup + @state = :initial + end + def scan_tokens encoder, options - - state = :initial + state = options[:state] || @state inline_block_stack = [] inline_block_paren_depth = nil string_delimiter = nil @@ -246,6 +249,16 @@ def scan_tokens encoder, options encoder.end_group state end + if options[:keep_state] + @state = state + end + + until inline_block_stack.empty? + state, = *inline_block_stack.pop + encoder.end_group :inline + encoder.end_group state + end + encoder end From 3c3060acaf2422c23784539b270e365decc9cb2e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 05:55:23 +0200 Subject: [PATCH 023/236] close open tokens in PHP scanner --- lib/coderay/scanners/php.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 6c688343..40037f95 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -500,6 +500,10 @@ def scan_tokens encoder, options end + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + end + encoder end From 7f45168efd1abe679b261a7a91f3641984091d12 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:16 +0200 Subject: [PATCH 024/236] remove instance variables from Lua scanner --- lib/coderay/scanners/lua.rb | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 25bebbec..185c133c 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -59,13 +59,15 @@ def setup # CodeRay entry hook. Starts parsing. def scan_tokens(encoder, options) state = options[:state] || @state + brace_depth = @brace_depth + num_equals = nil until eos? case state when :initial if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:comment) encoder.text_token(match, :delimiter) state = :long_comment @@ -74,7 +76,7 @@ def scan_tokens(encoder, options) encoder.text_token(match, :comment) elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - @num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for comment end encoder.begin_group(:string) encoder.text_token(match, :delimiter) state = :long_string @@ -101,19 +103,19 @@ def scan_tokens(encoder, options) elsif match = scan(/\{/) # Opening table brace { encoder.begin_group(:map) - encoder.text_token(match, @brace_depth >= 1 ? :inline_delimiter : :delimiter) - @brace_depth += 1 + encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter) + brace_depth += 1 state = :map elsif match = scan(/\}/) # Closing table brace } - if @brace_depth == 1 - @brace_depth = 0 + if brace_depth == 1 + brace_depth = 0 encoder.text_token(match, :delimiter) encoder.end_group(:map) - elsif @brace_depth == 0 # Mismatched brace + elsif brace_depth == 0 # Mismatched brace encoder.text_token(match, :error) else - @brace_depth -= 1 + brace_depth -= 1 encoder.text_token(match, :inline_delimiter) encoder.end_group(:map) state = :map @@ -122,7 +124,7 @@ def scan_tokens(encoder, options) elsif match = scan(/["']/) # String delimiters " and ' encoder.begin_group(:string) encoder.text_token(match, :delimiter) - @start_delim = match + start_delim = match state = :string # ↓Prefix hex number ←|→ decimal number @@ -146,7 +148,7 @@ def scan_tokens(encoder, options) # It may be that we’re scanning a full-blown subexpression of a table # (tables can contain full expressions in parts). # If this is the case, return to :map scanning state. - state = :map if state == :initial && @brace_depth >= 1 + state = :map if state == :initial && brace_depth >= 1 when :function_expected if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name @@ -198,10 +200,10 @@ def scan_tokens(encoder, options) end when :long_comment - if match = scan(/.*?(?=\]={#@num_equals}\])/m) + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -211,10 +213,10 @@ def scan_tokens(encoder, options) state = :initial when :long_string - if match = scan(/.*?(?=\]={#@num_equals}\])/m) # Long strings do not interpret any escape sequences + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences encoder.text_token(match, :content) - delim = scan(/\]={#@num_equals}\]/) + delim = scan(/\]={#{num_equals}}\]/) encoder.text_token(delim, :delimiter) else # No terminator found till EOF encoder.text_token(rest, :error) @@ -224,11 +226,11 @@ def scan_tokens(encoder, options) state = :initial when :string - if match = scan(/[^\\#@start_delim\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) encoder.text_token(match, :content) elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) encoder.text_token(match, :char) - elsif match = scan(Regexp.compile(@start_delim)) + elsif match = scan(Regexp.compile(start_delim)) encoder.text_token(match, :delimiter) encoder.end_group(:string) state = :initial From 45bb0c576b67b7a3c0dead02078b3a16b5583154 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:06:52 +0200 Subject: [PATCH 025/236] close open token groups in Lua scanner --- lib/coderay/scanners/lua.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index 185c133c..fb1e45a7 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -268,6 +268,9 @@ def scan_tokens(encoder, options) @state = state end + encoder.end_group :string if [:string].include? state + brace_depth.times { encoder.end_group :map } + encoder end From dc6071129cdc1bcd15129147bbc4d92ba870f007 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:13:32 +0200 Subject: [PATCH 026/236] close open token groups in Sass scanner --- lib/coderay/scanners/sass.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 65d40b0d..0ba383fb 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -218,6 +218,14 @@ def scan_tokens encoder, options @state = states end + while state = states.pop + if state == :sass_inline + encoder.end_group :inline + elsif state == :string + encoder.end_group :string + end + end + encoder end From 4327fcbe932ac913f7eb8e92497f97913fb398c5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:15:55 +0200 Subject: [PATCH 027/236] fix Sass regexp modifier --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 0ba383fb..e20bebe9 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -195,7 +195,7 @@ def scan_tokens encoder, options elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) encoder.text_token match, :color - elsif match = scan(/@else if\b|#{RE::AtKeyword}/) + elsif match = scan(/@else if\b|#{RE::AtKeyword}/o) encoder.text_token match, :directive value_expected = true From 755904f7046d8a8d6208fe8367aaa3b9a19ecd65 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:24 +0200 Subject: [PATCH 028/236] avoid empty tokens in Diff scanner --- lib/coderay/scanners/diff.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index af0f7557..fd1aed67 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -69,7 +69,7 @@ def scan_tokens encoder, options state = :added elsif match = scan(/\\ .*/) encoder.text_token match, :comment - elsif match = scan(/@@(?>[^@\n]*)@@/) + elsif match = scan(/@@(?>[^@\n]+)@@/) content_scanner.state = :initial unless match?(/\n\+/) content_scanner_entry_state = nil if check(/\n|$/) From e1d5e111d968639fa03a6074cf90535ecc90d0dd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:30:44 +0200 Subject: [PATCH 029/236] close correct token groups in Groovy scanner --- lib/coderay/scanners/groovy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index d04a5354..c64454f0 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -226,7 +226,7 @@ def scan_tokens encoder, options encoder.text_token match, :content # TODO: Shouldn't this be :error? elsif match = scan(/ \\ | \n /x) - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string encoder.text_token match, :error after_def = value_expected = false state = :initial @@ -246,7 +246,7 @@ def scan_tokens encoder, options end if [:multiline_string, :string, :regexp].include? state - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end if options[:keep_state] @@ -256,7 +256,7 @@ def scan_tokens encoder, options until inline_block_stack.empty? state, = *inline_block_stack.pop encoder.end_group :inline - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string end encoder From 603ff7d0b14521cfd0408aa68e2e1cb6ea9086bc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:36:46 +0200 Subject: [PATCH 030/236] avoid empty tokens in YAML scanner --- lib/coderay/scanners/yaml.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 358a3f19..32c8e2cb 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -84,7 +84,7 @@ def scan_tokens encoder, options when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/) encoder.begin_group :key encoder.text_token match[0,1], :delimiter - encoder.text_token match[1..-2], :content + encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter encoder.end_group :key key_indent = column(pos - match.size) - 1 From 253a616924000cc5d522a2d4b8030f5693c98e43 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 06:39:30 +0200 Subject: [PATCH 031/236] close open string token groups in SQL scanner --- lib/coderay/scanners/sql.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index b7572789..93aeaf39 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -1,8 +1,9 @@ -module CodeRay module Scanners +module CodeRay +module Scanners # by Josh Goebel class SQL < Scanner - + register_for :sql KEYWORDS = %w( @@ -149,6 +150,7 @@ def scan_tokens encoder, options string_content = '' end encoder.text_token match, :error unless match.empty? + encoder.end_group :string state = :initial else raise "else case \" reached; %p not handled." % peek(1), encoder @@ -171,4 +173,5 @@ def scan_tokens encoder, options end -end end \ No newline at end of file +end +end From ad5efeb8c96c976804fcfcd3f71e0d02cf73cc10 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:00:13 +0200 Subject: [PATCH 032/236] changelog --- Changes.textile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 930fdbca..581e485a 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,6 +12,8 @@ h2. Changes in 1.1 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) +* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] +* Added @:keep_state@ functionality to more scanners [#116] * CSS scanner uses @:id@ and @:tag@ now [#27] * Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right? * Add .xaml file type [#121, thanks to Kozman Bálint] @@ -20,10 +22,13 @@ h2. Changes in 1.1 * New token type @:done@ for Taskpaper [#39] * New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] -* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] +* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style * @Plugin@ does not warn about fallback when default is defined +* @HTML@ encoder will not warn about unclosed token groups at the end of the stream * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now +* @Debug@ encoder will not warn about errors in the token stream +* New @DebugLint@ encoder that checks for empty tokens and correct nesting h2. Changes in 1.0.9 From c8751fbc09d4a8f43b0b037e3a7dc7ddbfbcacb1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:15:10 +0200 Subject: [PATCH 033/236] fix unclosed token group in Ruby scanner --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index c282f31b..80165cae 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -269,7 +269,7 @@ def scan_tokens encoder, options end if last_state - state = last_state + state = last_state unless state.is_a?(StringState) # otherwise, a simple 'def"' results in unclosed tokens last_state = nil end From c34ad739bc7342aa40830945a5e19e90e9814e04 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 13:27:43 +0200 Subject: [PATCH 034/236] fix token nesting in PHP scanner --- lib/coderay/scanners/php.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 40037f95..7a8d75d9 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -265,7 +265,7 @@ def scan_tokens encoder, options @html_scanner.tokenize match unless match.empty? end - when :php + when :php, :php_inline if match = scan(/\s+/) encoder.text_token match, :space @@ -332,7 +332,7 @@ def scan_tokens encoder, options if states.size == 1 encoder.text_token match, :error else - states.pop + state = states.pop if states.last.is_a?(::Array) delimiter = states.last[1] states[-1] = states.last[0] @@ -340,6 +340,7 @@ def scan_tokens encoder, options encoder.end_group :inline else encoder.text_token match, :operator + encoder.end_group :inline if state == :php_inline label_expected = true end end @@ -350,7 +351,14 @@ def scan_tokens encoder, options elsif match = scan(RE::PHP_END) encoder.text_token match, :inline_delimiter - states = [:initial] + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end + end + states << :initial elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o) encoder.begin_group :string @@ -400,6 +408,7 @@ def scan_tokens encoder, options elsif match = scan(/\\/) encoder.text_token match, :error else + encoder.end_group :string states.pop end @@ -459,7 +468,7 @@ def scan_tokens encoder, options encoder.begin_group :inline states[-1] = [states.last, delimiter] delimiter = nil - states.push :php + states.push :php_inline encoder.text_token match, :delimiter else encoder.text_token match, :content @@ -469,6 +478,7 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :content else + encoder.end_group :string states.pop end @@ -502,6 +512,10 @@ def scan_tokens encoder, options while state = states.pop encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end end encoder From 14f3a4f28341237503a51bfa0764c922bb02bdfa Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:13:15 +0200 Subject: [PATCH 035/236] tweak HTML CDATA token kinds --- lib/coderay/scanners/html.rb | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index fcf249a1..530ce447 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -1,13 +1,13 @@ module CodeRay module Scanners - + # HTML Scanner # # Alias: +xhtml+ # # See also: Scanners::XML class HTML < Scanner - + register_for :html KINDS_NOT_LOC = [ @@ -100,15 +100,13 @@ def scan_tokens encoder, options when :initial if match = scan(//m) - encoder.text_token match[0..-4], :content - encoder.text_token ']]>', :delimiter - else - encoder.text_token scan(/.*/m), :error + encoder.text_token match[0..-4], :plain + encoder.text_token ']]>', :inline_delimiter + elsif match = scan(/.+/) + encoder.text_token match, :error end - encoder.end_group :string elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) From 5b0dc0f35090949d7512a85dadc8cf2871b91aac Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:15:01 +0200 Subject: [PATCH 036/236] scan_css in HTML scanner (tags) --- lib/coderay/scanners/html.rb | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b795..06728e44 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -75,9 +75,14 @@ def setup def scan_java_script encoder, code if code && !code.empty? @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true - # encoder.begin_group :inline @java_script_scanner.tokenize code, :tokens => encoder - # encoder.end_group :inline + end + end + + def scan_css encoder, code + if code && !code.empty? + @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true + @css_scanner.tokenize code, :tokens => encoder end end @@ -110,7 +115,7 @@ def scan_tokens encoder, options elsif match = scan(/<\/[-\w.:]*>?/m) in_tag = nil encoder.text_token match, :tag - elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m) + elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m) encoder.text_token match, :tag in_tag = self[1] if self[2] @@ -206,19 +211,23 @@ def scan_tokens encoder, options when :in_special_tag case in_tag - when 'script' + when 'script', 'style' encoder.text_token match, :space if match = scan(/[ \t]*\n/) if scan(/(\s*)|(.*))/m) code = self[2] || self[4] closing = self[3] encoder.text_token self[1], :comment else - code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/) + code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/) closing = false end unless code.empty? encoder.begin_group :inline - scan_java_script encoder, code + if in_tag == 'script' + scan_java_script encoder, code + else + scan_css encoder, code + end encoder.end_group :inline end encoder.text_token closing, :comment if closing From 7ada74c8f0f77815393e887db83e1d2c36ce7235 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:32:14 +0200 Subject: [PATCH 037/236] scan_css in HTML scanner (arguments), change token kind from :inline to :string --- Changes.textile | 1 + lib/coderay/scanners/html.rb | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Changes.textile b/Changes.textile index 930fdbca..9509b15e 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,6 +10,7 @@ h2. Changes in 1.1 * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] +* HTML scanner displays style tags and attributes now * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * CSS scanner uses @:id@ and @:tag@ now [#27] diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 06728e44..f1dfba07 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -33,7 +33,8 @@ class HTML < Scanner ) IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil). - add(EVENT_ATTRIBUTES, :script) + add(EVENT_ATTRIBUTES, :script). + add(['style'], :style) ATTR_NAME = /[\w.:-]+/ # :nodoc: TAG_END = /\/?>/ # :nodoc: @@ -79,10 +80,10 @@ def scan_java_script encoder, code end end - def scan_css encoder, code + def scan_css encoder, code, state = [:initial] if code && !code.empty? @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true - @css_scanner.tokenize code, :tokens => encoder + @css_scanner.tokenize code, :tokens => encoder, :state => state end end @@ -166,17 +167,21 @@ def scan_tokens encoder, options encoder.text_token match, :attribute_value state = :attribute elsif match = scan(/["']/) - if in_attribute == :script - encoder.begin_group :inline - encoder.text_token match, :inline_delimiter + if in_attribute == :script || in_attribute == :style + encoder.begin_group :string + encoder.text_token match, :delimiter if scan(/javascript:[ \t]*/) encoder.text_token matched, :comment end code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/) - scan_java_script encoder, code + if in_attribute == :script + scan_java_script encoder, code + else + scan_css encoder, code, [:block] + end match = scan(/["']/) - encoder.text_token match, :inline_delimiter if match - encoder.end_group :inline + encoder.text_token match, :delimiter if match + encoder.end_group :string state = :attribute in_attribute = nil else From 56f4163e99689912c3797e5d3b2b97244ce65782 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:33:22 +0200 Subject: [PATCH 038/236] fix another CSS empty token issue --- lib/coderay/scanners/css.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 6c52c8af..9ed4618b 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -145,10 +145,10 @@ def scan_tokens encoder, options start = match[/^\w+\(/] encoder.text_token start, :delimiter if match[-1] == ?) - encoder.text_token match[start.size..-2], :content + encoder.text_token match[start.size..-2], :content if match.size > start.size + 1 encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content if start.size < match.size + encoder.text_token match[start.size..-1], :content if match.size > start.size end encoder.end_group :function From 6a2f1a2ec7d6136f1ac9dd5723bad84541dafb26 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:38:37 +0200 Subject: [PATCH 039/236] changelog --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 53f4109a..80f1d108 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,7 +10,7 @@ h2. Changes in 1.1 * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] -* HTML scanner displays style tags and attributes now +* HTML scanner displays style tags and attributes now [#145] * Remove double-click toggle handler from HTML table output * Fixes to CSS scanner (floats, pseudoclasses) * Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] From 9fb2fd9fa3ba4b92a440f271adbb9584b236c34e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:45:31 +0200 Subject: [PATCH 040/236] tweak :local_variable token color --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index a869d9e1..b05ae535 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -98,7 +98,7 @@ class Alpha < Style .key .delimiter { color: #404 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } -.local-variable { color:#963 } +.local-variable { color:#950 } .map .content { color:#808 } .map .delimiter { color:#40A} .map { background-color:hsla(200,100%,50%,0.06); } From 844975b53233b4697d1385548c1295ce18a7be72 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:46:45 +0200 Subject: [PATCH 041/236] whitespace --- lib/coderay/styles/alpha.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index b05ae535..ff85eccf 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -3,14 +3,14 @@ module Styles # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style - + register_for :alpha - + code_background = 'hsl(0,0%,95%)' numbers_background = 'hsl(180,65%,90%)' border_color = 'silver' normal_color = 'black' - + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; @@ -145,8 +145,8 @@ class Alpha < Style .change .change { color: #88f } .head .head { color: #f4f } TOKENS - + end - + end end From b89caf96d1cfc304c2114d8734ebe8b91337c799 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 14:53:14 +0200 Subject: [PATCH 042/236] fix diff/json token nesting issue --- lib/coderay/scanners/json.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 3754a9b4..b09970c2 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -26,6 +26,10 @@ def setup def scan_tokens encoder, options state = options[:state] || @state + if [:string, :key].include? state + encoder.begin_group state + end + until eos? case state From 965b8f4d1be88e1ae75ef11e6270a7fbfb2d86bd Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:03:57 +0200 Subject: [PATCH 043/236] use -w in executable tests --- test/executable/suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index 997405ca..ac0ff1d4 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - "#{RUBY_COMMAND}--ng -I%s %s" + "#{RUBY_COMMAND}--ng -w -I%s %s" else - "#{RUBY_COMMAND} -I%s %s" + "#{RUBY_COMMAND} -w -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} From a3e79c794f0998034b26ff6baa10bc801918be9d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:07:57 +0200 Subject: [PATCH 044/236] trying to fix TravisCI error --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f0850..4115d092 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.0' unless defined? VERSION end From 416243353e0b4b1c449476b269a50b0f60c13bda Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:15:24 +0200 Subject: [PATCH 045/236] trying different ways to load CodeRay version --- lib/coderay.rb | 2 +- lib/coderay/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay.rb b/lib/coderay.rb index 24ae5a2c..0c66f49d 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -134,7 +134,7 @@ def self.coderay_path *path File.join CODERAY_PATH, *path end - require coderay_path('version') + require 'coderay/version' # helpers autoload :FileType, coderay_path('helpers', 'file_type') diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4115d092..4b4f0850 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' unless defined? VERSION + VERSION = '1.1.0' end From a112fca72ce802b4b5fd25fdc7d76b6360cdc718 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:22:47 +0200 Subject: [PATCH 046/236] allow [j]ruby-head failures in Travis --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 59bb791d..b3b8cf72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ branches: - master matrix: allow_failures: - - rvm: rbx-18mode + - rvm: ruby-head + - rvm: jruby-head + - rvm: rbx-19mode - rvm: rbx-19mode script: "rake test" # test:scanners" From 64ca2ae8ad5130bdcf652aa7aa08298de00f20f4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 15:27:29 +0200 Subject: [PATCH 047/236] how did this happen? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3b8cf72..6d926f32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,6 @@ matrix: allow_failures: - rvm: ruby-head - rvm: jruby-head - - rvm: rbx-19mode + - rvm: rbx-18mode - rvm: rbx-19mode script: "rake test" # test:scanners" From 31c252ae9fd4b7e2f1ea2fd0009e7808d7691bcc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 16:04:26 +0200 Subject: [PATCH 048/236] tweaking colors for :function tokens --- lib/coderay/styles/alpha.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index ff85eccf..7d013723 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -82,7 +82,8 @@ class Alpha < Style .exception { color:#C00; font-weight:bold } .float { color:#60E } .function { color:#06B; font-weight:bold } -.function .delimiter { color:#024; font-weight:bold } +.function .delimiter { color:#059 } +.function .content { color:#037 } .global-variable { color:#d70 } .hex { color:#02b } .id { color:#33D; font-weight:bold } From e1e6a6af97ef710cfc02a1380180b772f1c34672 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:23:54 -0600 Subject: [PATCH 049/236] additional Ruby files types Ruby seems to have no shortage of these specially named files. --- lib/coderay/helpers/file_type.rb | 41 +++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d4fa926..6d928d64 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ def [] filename, read_shebang = false name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ def [] filename, read_shebang = false TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ def fetch filename, default = nil, read_shebang = false if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ def fetch filename, default = nil, read_shebang = false raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ def shebang filename end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -116,7 +116,7 @@ def shebang filename 'rhtml' => :erb, 'rjs' => :ruby, 'rpdf' => :ruby, - 'ru' => :ruby, + 'ru' => :ruby, # config.ru 'rxml' => :ruby, 'sass' => :sass, 'sql' => :sql, @@ -132,16 +132,19 @@ def shebang filename for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, + 'Guardfile' => :ruby, + 'Vagrantfile' => :ruby, + 'Appraisals' => :ruby } - + end - + end From fd8c81f5c338e4f1448007c9100d15d912fadd27 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 09:43:54 -0600 Subject: [PATCH 050/236] additional types: string, error http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2a..a66a5e35 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ class Go < Scanner 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'error', 'uint', 'int', 'uintptr', ] # :nodoc: From 004d0c83222113ff732101a3e25785ce0625bfa2 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:14:29 +0200 Subject: [PATCH 051/236] whitespace --- lib/coderay/scanners/go.rb | 82 +++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 4431ef2a..d4a3598d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,12 +1,12 @@ module CodeRay module Scanners - + # Scanner for Go, copy from c class Go < Scanner - + register_for :go file_extension 'go' - + # http://golang.org/ref/spec#Keywords KEYWORDS = [ 'break', 'default', 'func', 'interface', 'select', @@ -15,7 +15,7 @@ class Go < Scanner 'const', 'fallthrough', 'if', 'range', 'type', 'continue', 'for', 'import', 'return', 'var', ] # :nodoc: - + # http://golang.org/ref/spec#Types PREDEFINED_TYPES = [ 'bool', @@ -26,51 +26,51 @@ class Go < Scanner 'byte', 'rune', 'uint', 'int', 'uintptr', ] # :nodoc: - + PREDEFINED_CONSTANTS = [ 'nil', 'iota', 'true', 'false', ] # :nodoc: - + DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: - + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - - protected - + + protected + def scan_tokens encoder, options - + state = :initial label_expected = true case_expected = false label_expected_before_preproc_line = nil in_preproc_line = false - + until eos? - + case state - + when :initial - + if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space - + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - + elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) label_expected = match =~ /[;\{\}]/ if case_expected @@ -78,7 +78,7 @@ def scan_tokens encoder, options case_expected = false end encoder.text_token match, :operator - + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) @@ -94,7 +94,7 @@ def scan_tokens encoder, options end end encoder.text_token match, kind - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -107,41 +107,41 @@ def scan_tokens encoder, options elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment - + elsif match = scan(/#[ \t]*(\w*)/) encoder.text_token match, :preprocessor in_preproc_line = true label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) label_expected = false encoder.text_token match, :char - + elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - + elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer - + elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) label_expected = false encoder.text_token match, :float - + else encoder.text_token getch, :error - + end - + when :string if match = scan(/[^\\\n"]+/) encoder.text_token match, :content @@ -160,36 +160,36 @@ def scan_tokens encoder, options else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder end - + when :include_expected if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) encoder.text_token match, :include state = :initial - + elsif match = scan(/\s+/) encoder.text_token match, :space state = :initial if match.index ?\n - + else state = :initial - + end - + else raise_inspect 'Unknown state', encoder - + end - + end - + if state == :string encoder.end_group :string end - + encoder end - + end - + end end From afa2be73c03d033056024354787abd66f2ff7fa0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:05 +0200 Subject: [PATCH 052/236] add string as predefined type --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index d4a3598d..dbf9595e 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -23,7 +23,7 @@ class Go < Scanner 'int8', 'int16', 'int32', 'int64', 'float32', 'float64', 'complex64', 'complex128', - 'byte', 'rune', + 'byte', 'rune', 'string', 'uint', 'int', 'uintptr', ] # :nodoc: From 7ef6f7783d788bf656a3c59600cd45cd5b694376 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:22 +0200 Subject: [PATCH 053/236] add support for raw strings in Go --- lib/coderay/scanners/go.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index dbf9595e..c0c602d8 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -103,7 +103,14 @@ def scan_tokens encoder, options end encoder.text_token match, :delimiter state = :string - + + elsif match = scan(/ ` ([^`]+)? (`)? /x) + encoder.begin_group :shell + encoder.text_token '`', :delimiter + encoder.text_token self[1], :content if self[1] + encoder.text_token self[2], :delimiter if self[2] + encoder.end_group :shell + elsif match = scan(/ \# \s* if \s* 0 /x) match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? encoder.text_token match, :comment From a24c207a7e547408de0f6157d495b0dd2e8d5a40 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:15:36 +0200 Subject: [PATCH 054/236] fix empty token in Go scanner --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index c0c602d8..76a0820a 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -161,7 +161,7 @@ def scan_tokens encoder, options encoder.text_token match, :char elsif match = scan(/ \\ | $ /x) encoder.end_group :string - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else From 4669b7086136647f2eece41a122fa204eb16e1e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:16:47 +0200 Subject: [PATCH 055/236] fix label_expected (test case?) --- lib/coderay/scanners/go.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 76a0820a..23cbfe64 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -64,6 +64,7 @@ def scan_tokens encoder, options if match = scan(/ \s+ | \\\n /x) if in_preproc_line && match != "\\\n" && match.index(?\n) in_preproc_line = false + case_expected = false label_expected = label_expected_before_preproc_line end encoder.text_token match, :space @@ -72,7 +73,6 @@ def scan_tokens encoder, options encoder.text_token match, :comment elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) - label_expected = match =~ /[;\{\}]/ if case_expected label_expected = true if match == ':' case_expected = false @@ -83,6 +83,7 @@ def scan_tokens encoder, options kind = IDENT_KIND[match] if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) kind = :label + label_expected = false match << matched else label_expected = false From c0d028010b081020352444078a685ee60bffe209 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:17:14 +0200 Subject: [PATCH 056/236] make predefined-type a bit more bright/blue --- lib/coderay/styles/alpha.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index ff85eccf..f4e9d7de 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -107,7 +107,7 @@ class Alpha < Style .operator { } .predefined { color:#369; font-weight:bold } .predefined-constant { color:#069 } -.predefined-type { color:#0a5; font-weight:bold } +.predefined-type { color:#0a8; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } .regexp { background-color:hsla(300,100%,50%,0.06); } From 17946d7146390ee296264e66e6de31b6421e5231 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 23 Jun 2013 18:29:02 +0200 Subject: [PATCH 057/236] changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index 80f1d108..89b66c88 100644 --- a/Changes.textile +++ b/Changes.textile @@ -6,6 +6,7 @@ h2. Changes in 1.1 * New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] +* New scanner: Go [#28, thanks to Eric Guo] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly From 85275cf21e7d15459abc11fd76606bd7d38fb8b5 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:11:49 -0600 Subject: [PATCH 058/236] Go doesn't have a "f" suffix for floats like C. --- lib/coderay/scanners/go.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 49d24c22..afcfca5f 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -141,7 +141,7 @@ def scan_tokens encoder, options label_expected = false encoder.text_token match, :integer - elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/) + elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float From cad9a00e28d61781d5a12a0556be7126eb790725 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:35:26 -0600 Subject: [PATCH 059/236] add imaginary numbers to Go scanner --- lib/coderay/scanners/go.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index afcfca5f..04504ab6 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -128,7 +128,11 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :ident - + + elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + label_expected = false + encoder.text_token match, :imaginary + elsif match = scan(/0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex @@ -137,13 +141,13 @@ def scan_tokens encoder, options label_expected = false encoder.text_token match, :octal - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) - label_expected = false - encoder.text_token match, :integer - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float + + elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer else encoder.text_token getch, :error From 4c877bf0eb655cf2c04a0efb24b4e5f83b97e160 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:43:50 -0600 Subject: [PATCH 060/236] predeclared identifiers http://golang.org/ref/spec#Predeclared_identifiers --- lib/coderay/scanners/go.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 04504ab6..ae50aa4d 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,6 +31,11 @@ class Go < Scanner 'nil', 'iota', 'true', 'false', ] # :nodoc: + + PREDEFINED_FUNCTIONS = %w[ + append cap close complex copy delete imag len + make new panic print println real recover + ] # :nodoc: DIRECTIVES = [ 'go_no_directive', # Seems no directive concept in Go? @@ -40,7 +45,8 @@ class Go < Scanner add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). add(DIRECTIVES, :directive). - add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: From dd9ec43a3ea4cfce8eb1e8dd38504a9d8f24df54 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 12:46:15 -0600 Subject: [PATCH 061/236] yup, no C-style directives (auto extern static) --- lib/coderay/scanners/go.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index ae50aa4d..eb06fb06 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -37,14 +37,9 @@ class Go < Scanner make new panic print println real recover ] # :nodoc: - DIRECTIVES = [ - 'go_no_directive', # Seems no directive concept in Go? - ] # :nodoc: - IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). - add(DIRECTIVES, :directive). add(PREDEFINED_CONSTANTS, :predefined_constant). add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: From ae1969b1809365c87a9cb485b671364c502236a0 Mon Sep 17 00:00:00 2001 From: Nathan Youngman Date: Sun, 23 Jun 2013 13:00:17 -0600 Subject: [PATCH 062/236] revert trimming spaces --- lib/coderay/helpers/file_type.rb | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 6d928d64..a0f9fd61 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -1,5 +1,5 @@ module CodeRay - + # = FileType # # A simple filetype recognizer. @@ -8,18 +8,18 @@ module CodeRay # # # determine the type of the given # lang = FileType[file_name] - # + # # # return :text if the file type is unknown # lang = FileType.fetch file_name, :text - # + # # # try the shebang line, too # lang = FileType.fetch file_name, :text, true module FileType - + UnknownFileType = Class.new Exception - + class << self - + # Try to determine the file type of the file. # # +filename+ is a relative or absolute path to a file. @@ -30,7 +30,7 @@ def [] filename, read_shebang = false name = File.basename filename ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot ext2 = filename.to_s[/\.(.*)/, 1] # from first dot - + type = TypeFromExt[ext] || TypeFromExt[ext.downcase] || @@ -39,10 +39,10 @@ def [] filename, read_shebang = false TypeFromName[name] || TypeFromName[name.downcase] type ||= shebang(filename) if read_shebang - + type end - + # This works like Hash#fetch. # # If the filetype cannot be found, the +default+ value @@ -51,7 +51,7 @@ def fetch filename, default = nil, read_shebang = false if default && block_given? warn 'Block supersedes default value argument; use either.' end - + if type = self[filename, read_shebang] type else @@ -60,9 +60,9 @@ def fetch filename, default = nil, read_shebang = false raise UnknownFileType, 'Could not determine type of %p.' % filename end end - + protected - + def shebang filename return unless File.exist? filename File.open filename, 'r' do |f| @@ -73,9 +73,9 @@ def shebang filename end end end - + end - + TypeFromExt = { 'c' => :c, 'cfc' => :xml, @@ -132,9 +132,9 @@ def shebang filename for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu] TypeFromExt[cpp_alias] = :cpp end - + TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/ - + TypeFromName = { 'Capfile' => :ruby, 'Rakefile' => :ruby, @@ -144,7 +144,7 @@ def shebang filename 'Vagrantfile' => :ruby, 'Appraisals' => :ruby } - + end - + end From e75fecef1998836a4e209db4974b9cb9ec470e95 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 03:50:23 +0200 Subject: [PATCH 063/236] don't use -w flag in executable tests --- test/executable/suite.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index ac0ff1d4..997405ca 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -18,9 +18,9 @@ class TestCodeRayExecutable < Test::Unit::TestCase EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - "#{RUBY_COMMAND}--ng -w -I%s %s" + "#{RUBY_COMMAND}--ng -I%s %s" else - "#{RUBY_COMMAND} -w -I%s %s" + "#{RUBY_COMMAND} -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} From fb8c0dbfd065e19419b30213efb8176a1e968945 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:34 +0200 Subject: [PATCH 064/236] cleanup benchmark script (finally!) --- .gitignore | 1 - bench/bench.rb | 158 ++++++++------------------------------ rake_tasks/benchmark.rake | 3 +- 3 files changed, 32 insertions(+), 130 deletions(-) diff --git a/.gitignore b/.gitignore index dd001c8a..deed1a27 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,4 @@ Gemfile.lock test/executable/source.rb.html test/executable/source.rb.json test/scanners -bench/test.div.html old-stuff diff --git a/bench/bench.rb b/bench/bench.rb index 1958c73a..92f9d07f 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -1,142 +1,46 @@ -# The most ugly test script I've ever written! -# Shame on me! - -require 'pathname' -require 'profile' if ARGV.include? '-p' - -MYDIR = File.dirname(__FILE__) -LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s -$:.unshift MYDIR, LIBDIR +require 'benchmark' +$: << File.expand_path('../../lib', __FILE__) require 'coderay' -@size = ARGV.fetch(2, 100).to_i * 1000 - -lang = ARGV.fetch(0) do - puts <<-HELP -Usage: - ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream] - - SIZE defaults to 100 kB (= 100,000 bytes). - SIZE = 0 means the whole input. - --p generates a profile (slow! use with SIZE = 1) --o shows the output -stream enabled streaming mode - -Sorry for the strange interface. I will improve it in the next release. - HELP +if ARGV.include? '-h' + puts DATA.read exit end -format = ARGV.fetch(1, 'html').downcase - -$stream = ARGV.include? 'stream' -$optimize = ARGV.include? 'opt' -$style = ARGV.include? 'style' - -require 'benchmark' -require 'fileutils' +lang = ARGV.fetch(0, 'ruby') +data = nil +File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read } +raise 'Example file is empty.' if data.empty? -if format == 'comp' - format = 'page' - begin - require 'syntax' - require 'syntax/convertors/html.rb' - rescue LoadError - puts 'Syntax no found!! (Try % gem install syntax)' - end -end +format = ARGV.fetch(1, 'html').downcase +encoder = CodeRay.encoder(format) -def here fn = nil - return MYDIR unless fn - File.join here, fn +size = ARGV.fetch(2, 1000).to_i * 1000 +unless size.zero? + data += data until data.size >= size + data = data[0, size] end +size = data.size +puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] -n = ARGV.find { |a| a[/^N/] } -N = if n then n[/\d+/].to_i else 1 end -$filename = ARGV.include?('strange') ? 'strange' : 'example' - -Benchmark.bm(20) do |bm| -N.times do - - data = nil - File.open(here("#$filename." + lang), 'rb') { |f| data = f.read } - raise 'Example file is empty.' if data.empty? - unless @size.zero? - data += data until data.size >= @size - data = data[0, @size] - end - @size = data.size - - options = { - :tab_width => 2, - # :line_numbers => :inline, - :css => $style ? :style : :class, - } - $hl = CodeRay.encoder(format, options) - time = bm.report('CodeRay') do - if $stream || true - $o = $hl.encode(data, lang, options) - else - tokens = CodeRay.scan(data, lang) - tokens.optimize! if $optimize - $o = tokens.encode($hl) - end - end - $file_created = here('test.' + $hl.file_extension) - File.open($file_created, 'wb') do |f| - # f.write $o - end - - time_real = time.real - - puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000] - puts $o if ARGV.include? '-o' - -end +n = ARGV.fetch(3, 5).to_s[/\d+/].to_i +require 'profile' if ARGV.include? '-p' +n.times do |i| + time = Benchmark.realtime { encoder.encode(data, lang) } + puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] end -puts "Files created: #$file_created" -STDIN.gets if ARGV.include? 'wait' +STDIN.gets if ARGV.include? '-w' __END__ -.ruby .normal {} -.ruby .comment { color: #005; font-style: italic; } -.ruby .keyword { color: #A00; font-weight: bold; } -.ruby .method { color: #077; } -.ruby .class { color: #074; } -.ruby .module { color: #050; } -.ruby .punct { color: #447; font-weight: bold; } -.ruby .symbol { color: #099; } -.ruby .string { color: #944; background: #FFE; } -.ruby .char { color: #F07; } -.ruby .ident { color: #004; } -.ruby .constant { color: #07F; } -.ruby .regex { color: #B66; background: #FEF; } -.ruby .number { color: #F99; } -.ruby .attribute { color: #7BB; } -.ruby .global { color: #7FB; } -.ruby .expr { color: #227; } -.ruby .escape { color: #277; } +Usage: + ruby bench.rb [lang] [format] [size in kB] [number of runs] -.xml .normal {} -.xml .namespace { color: #B66; font-weight: bold; } -.xml .tag { color: #F88; } -.xml .comment { color: #005; font-style: italic; } -.xml .punct { color: #447; font-weight: bold; } -.xml .string { color: #944; } -.xml .number { color: #F99; } -.xml .attribute { color: #BB7; } + - lang defaults to ruby. + - format defaults to html. + - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input. + - number of runs defaults to 5. -.yaml .normal {} -.yaml .document { font-weight: bold; color: #07F; } -.yaml .type { font-weight: bold; color: #05C; } -.yaml .key { color: #F88; } -.yaml .comment { color: #005; font-style: italic; } -.yaml .punct { color: #447; font-weight: bold; } -.yaml .string { color: #944; } -.yaml .number { color: #F99; } -.yaml .time { color: #F99; } -.yaml .date { color: #F99; } -.yaml .ref { color: #944; } -.yaml .anchor { color: #944; } +-h prints this help +-p generates a profile (slow, use with SIZE = 1) +-w waits after the benchmark (for debugging memory usw) diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake index 040951b5..2e38b577 100644 --- a/rake_tasks/benchmark.rake +++ b/rake_tasks/benchmark.rake @@ -1,7 +1,6 @@ desc 'Do a benchmark' task :benchmark do - ruby "-v" - ruby "-wIlib bench/bench.rb ruby div 3000 N5" + ruby 'bench/bench.rb ruby html 3000' end task :bench => :benchmark From dc57601571af8024700991b6a80129285a980e9e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 05:57:38 +0200 Subject: [PATCH 065/236] add Lint encoder; do we still need DebugLint? --- lib/coderay/encoders/debug_lint.rb | 12 +++---- lib/coderay/encoders/lint.rb | 57 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 lib/coderay/encoders/lint.rb diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 17a07955..2c141863 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -1,6 +1,8 @@ module CodeRay module Encoders + load :lint + # = Debug Lint Encoder # # Debug encoder with additional checks for: @@ -15,12 +17,8 @@ class DebugLint < Debug register_for :debug_lint - InvalidTokenStream = Class.new StandardError - EmptyToken = Class.new InvalidTokenStream - IncorrectTokenGroupNesting = Class.new InvalidTokenStream - def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token' if text.empty? super end @@ -30,7 +28,7 @@ def begin_group kind end def end_group kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end @@ -41,7 +39,7 @@ def begin_line kind end def end_line kind - raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind @opened.pop super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb new file mode 100644 index 00000000..4601e902 --- /dev/null +++ b/lib/coderay/encoders/lint.rb @@ -0,0 +1,57 @@ +module CodeRay +module Encoders + + # = Lint Encoder + # + # Checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::DebugLint + class Lint < Debug + + register_for :lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def text_token text, kind + raise EmptyToken, 'empty token' if text.empty? + end + + def begin_group kind + @opened << kind + end + + def end_group kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + def begin_line kind + @opened << kind + end + + def end_line kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + protected + + def setup options + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? + end + + end + +end +end From ab1bb26c29cc69a93da2d808ae8cd7b3cad6ea25 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 30 Jun 2013 16:55:15 +0200 Subject: [PATCH 066/236] use File.expand_path instead of File.join --- lib/coderay.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay.rb b/lib/coderay.rb index 0c66f49d..f759ed63 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -127,7 +127,7 @@ module CodeRay $CODERAY_DEBUG ||= false - CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay' + CODERAY_PATH = File.expand_path('../coderay', __FILE__) # Assuming the path is a subpath of lib/coderay/ def self.coderay_path *path From 94e4bb3366537def28ed257fcf2b70a634711c9f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 12 Jul 2013 14:57:46 +0200 Subject: [PATCH 067/236] don't change value of objects you don't own --- lib/coderay/scanners/css.rb | 2 +- lib/coderay/scanners/sass.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 9ed4618b..5977b9ce 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -53,7 +53,7 @@ def setup end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup value_expected = @value_expected until eos? diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index e20bebe9..7ba9bf5c 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -19,7 +19,7 @@ def setup end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup string_delimiter = nil until eos? @@ -119,7 +119,7 @@ def scan_tokens encoder, options else #:nocov: - raise_inspect 'Unknown state', encoder + raise_inspect 'Unknown state: %p' % [states.last], encoder #:nocov: end @@ -215,7 +215,7 @@ def scan_tokens encoder, options end if options[:keep_state] - @state = states + @state = states.dup end while state = states.pop From 990ed25fd1bf350dabae4bed031e07ee485beb79 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:20:59 +0200 Subject: [PATCH 068/236] split '" string states in Sass scanner (edge case) --- lib/coderay/scanners/sass.rb | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 7ba9bf5c..85b4711b 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -7,11 +7,6 @@ class Sass < CSS register_for :sass file_extension 'sass' - STRING_CONTENT_PATTERN = { - "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - } - protected def setup @@ -20,7 +15,8 @@ def setup def scan_tokens encoder, options states = Array(options[:state] || @state).dup - string_delimiter = nil + + encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring until eos? @@ -91,24 +87,23 @@ def scan_tokens encoder, options next end - when :string - if match = scan(STRING_CONTENT_PATTERN[string_delimiter]) + when :sqstring, :dqstring + if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o) encoder.text_token match, :content elsif match = scan(/['"]/) encoder.text_token match, :delimiter encoder.end_group :string - string_delimiter = nil states.pop elsif match = scan(/#\{/) encoder.begin_group :inline encoder.text_token match, :inline_delimiter states.push :sass_inline elsif match = scan(/ \\ | $ /x) - encoder.end_group :string + encoder.end_group states.last encoder.text_token match, :error unless match.empty? states.pop else - raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder + raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder end when :include @@ -157,15 +152,15 @@ def scan_tokens encoder, options elsif match = scan(/['"]/) encoder.begin_group :string - string_delimiter = match encoder.text_token match, :delimiter if states.include? :sass_inline - content = scan_until(/(?=#{string_delimiter}|\}|\z)/) + # no nesting, just scan the string until delimiter + content = scan_until(/(?=#{match}|\}|\z)/) encoder.text_token content, :content unless content.empty? - encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/) + encoder.text_token match, :delimiter if scan(/#{match}/) encoder.end_group :string else - states.push :string + states.push match == "'" ? :sqstring : :dqstring end elsif match = scan(/#{RE::Function}/o) @@ -221,7 +216,7 @@ def scan_tokens encoder, options while state = states.pop if state == :sass_inline encoder.end_group :inline - elsif state == :string + elsif state == :sqstring || state == :dqstring encoder.end_group :string end end From def7e09db1963368e20bfd53c72532d6a631e0e8 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:21:42 +0200 Subject: [PATCH 069/236] fix #139: don't scan for :include after eos --- lib/coderay/scanners/sass.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 85b4711b..1bbd5349 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -209,6 +209,8 @@ def scan_tokens encoder, options end + states.pop if states.last == :include + if options[:keep_state] @state = states.dup end From 62a0be9509f8814902a4a97df4ad84913728d059 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:34:17 +0200 Subject: [PATCH 070/236] fix #143 (Sass scanner key vs tag heuristic) --- lib/coderay/scanners/sass.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 1bbd5349..e3296b90 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -44,7 +44,7 @@ def scan_tokens encoder, options elsif case states.last when :initial, :media, :sass_inline if match = scan(/(?>#{RE::Ident})(?!\()/ox) - encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag) + encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag) next elsif !value_expected && (match = scan(/\*/)) encoder.text_token match, :tag From 70c9ba896e1bba5ac727fb6fdfc3ba94510e652d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:34:41 +0200 Subject: [PATCH 071/236] fix CSS scanner for things like "nth-child(2n)" --- lib/coderay/scanners/css.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 5977b9ce..55d52397 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -25,7 +25,7 @@ module RE # :nodoc: HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/ - Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/ + Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/ Name = /#{NMChar}+/ Ident = /-?#{NMStart}#{NMChar}*/ AtKeyword = /@#{Ident}/ From 028ea15c5ee983bfa1b045081e68c2c178d5f126 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:36:06 +0200 Subject: [PATCH 072/236] changelog --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 80f1d108..7ea4bfe2 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,7 +12,7 @@ h2. Changes in 1.1 * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] * HTML scanner displays style tags and attributes now [#145] * Remove double-click toggle handler from HTML table output -* Fixes to CSS scanner (floats, pseudoclasses) +* Fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143] * Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] * Added @:keep_state@ functionality to more scanners [#116] * CSS scanner uses @:id@ and @:tag@ now [#27] From ee72fe95fcfca2848c3cff51b13ee78b662ba50f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 16:56:51 +0200 Subject: [PATCH 073/236] use bundler rake tasks --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index ba6c34ef..55770098 100644 --- a/Rakefile +++ b/Rakefile @@ -1,3 +1,5 @@ +require 'bundler/gem_tasks' + $:.unshift File.dirname(__FILE__) unless $:.include? '.' ROOT = '.' From 4c2486353a2e5e7e393ac97e556e4e29cac6bcc3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:29:49 +0200 Subject: [PATCH 074/236] create nathany for Go scanner, too --- Changes.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index cc401e9d..58427aaf 100644 --- a/Changes.textile +++ b/Changes.textile @@ -6,7 +6,7 @@ h2. Changes in 1.1 * New scanner: Lua [#21, #22, thanks to Quintus] * New scanner: Sass [#93] -* New scanner: Go [#28, thanks to Eric Guo] +* New scanner: Go [#28, thanks to Eric Guo and Nathan Youngman] * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly From bbe4d72ba785f1bd6fd703d63b096a907da1b09f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:31:34 +0200 Subject: [PATCH 075/236] tweak numeral tokens handling (#147) --- lib/coderay/scanners/go.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index eb06fb06..938da9da 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -31,7 +31,7 @@ class Go < Scanner 'nil', 'iota', 'true', 'false', ] # :nodoc: - + PREDEFINED_FUNCTIONS = %w[ append cap close complex copy delete imag len make new panic print println real recover @@ -73,7 +73,7 @@ def scan_tokens encoder, options elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) encoder.text_token match, :comment - elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + elsif match = scan(/ ?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) if case_expected label_expected = true if match == ':' case_expected = false @@ -129,24 +129,24 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :ident - - elsif match = scan(/\d*(\.\d*)?([eE][+-]?\d+)?i/) + + elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/) label_expected = false encoder.text_token match, :imaginary - - elsif match = scan(/0[xX][0-9A-Fa-f]+/) + + elsif match = scan(/-?0[xX][0-9A-Fa-f]+/) label_expected = false encoder.text_token match, :hex - elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/) + elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/) label_expected = false encoder.text_token match, :octal - elsif match = scan(/\d|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) + elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) label_expected = false encoder.text_token match, :float - - elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/) + + elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/) label_expected = false encoder.text_token match, :integer From 6dd14ef018ee4417771504117819121bd4b8520d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:34:10 +0200 Subject: [PATCH 076/236] be a bit more graceful with buggy Go strings --- lib/coderay/scanners/go.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 938da9da..5034adc9 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -165,9 +165,10 @@ def scan_tokens encoder, options label_expected = false elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) encoder.text_token match, :char - elsif match = scan(/ \\ | $ /x) + elsif match = scan(/ \\ /x) + encoder.text_token match, :error + elsif match = scan(/$/) encoder.end_group :string - encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else From 82864efabda9dc17fa6a28b0740ffc8f58706126 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Jul 2013 20:36:27 +0200 Subject: [PATCH 077/236] allow unicode characters in char literals --- lib/coderay/scanners/go.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 5034adc9..59473f6e 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -44,7 +44,7 @@ class Go < Scanner add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected @@ -123,7 +123,7 @@ def scan_tokens encoder, options label_expected_before_preproc_line = label_expected state = :include_expected if self[1] == 'include' - elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox) + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox) label_expected = false encoder.text_token match, :char From 59ca07b0d1a1710ab729636ea00de4b638f56110 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 20 Jul 2013 11:17:30 +0200 Subject: [PATCH 078/236] =?UTF-8?q?add=20Ruby=202=20syntax:=20%i(=E2=80=A6?= =?UTF-8?q?)=20and=20%I(=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Changes.textile | 3 ++- lib/coderay/scanners/ruby/patterns.rb | 5 ++++- lib/coderay/styles/alpha.rb | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Changes.textile b/Changes.textile index 58427aaf..f57faf5f 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,6 +10,7 @@ h2. Changes in 1.1 * New scanner: Taskpaper [#39, thanks to shimomura] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly +* Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman] * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] * HTML scanner displays style tags and attributes now [#145] * Remove double-click toggle handler from HTML table output @@ -22,7 +23,7 @@ h2. Changes in 1.1 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] * New token type @:id@ for CSS/Sass [#27] * New token type @:done@ for Taskpaper [#39] -* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and nathany] +* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] * Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index ed071d2b..0b36e13b 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -157,13 +157,16 @@ module Ruby::Patterns # :nodoc: all yield ]) - FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x + FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x FANCY_STRING_KIND = Hash.new(:string).merge({ + 'i' => :symbol, + 'I' => :symbol, 'r' => :regexp, 's' => :symbol, 'x' => :shell, }) FANCY_STRING_INTERPRETED = Hash.new(true).merge({ + 'i' => false, 'q' => false, 's' => false, 'w' => false, diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f4e9d7de..d304dc4a 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -125,7 +125,7 @@ class Alpha < Style .string .modifier { color: #E40 } .symbol { color:#A60 } .symbol .content { color:#A60 } -.symbol .delimiter { color:#630 } +.symbol .delimiter { color:#740 } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } From 5c23a731ca55729fc65630eca3b37a5b1a71e5b1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 20 Jul 2013 18:12:04 +0200 Subject: [PATCH 079/236] mark possibly problematic spots with FIXME --- lib/coderay/encoders/html.rb | 2 ++ lib/coderay/encoders/html/css.rb | 2 ++ lib/coderay/helpers/file_type.rb | 1 + lib/coderay/helpers/plugin.rb | 1 + lib/coderay/scanners/debug.rb | 2 ++ lib/coderay/scanners/diff.rb | 1 + lib/coderay/scanners/python.rb | 3 +++ lib/coderay/scanners/raydebug.rb | 2 ++ lib/coderay/scanners/ruby/string_state.rb | 1 + 9 files changed, 15 insertions(+) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 20f24095..6dd231ae 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -142,6 +142,7 @@ def self.make_html_escape_hash HTML_ESCAPE = make_html_escape_hash HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ + # FIXME: cache attack TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end @@ -284,6 +285,7 @@ def style_for_kinds kinds end def make_span_for_kinds method, hint + # FIXME: cache attack Hash.new do |h, kinds| h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin css_class = css_class_for_kinds(kinds) diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index 164d7f85..de98f0ed 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -21,6 +21,7 @@ def initialize style = :default end def get_style_for_css_classes css_classes + # FIXME: cache attack cl = @styles[css_classes.first] return '' unless cl style = '' @@ -52,6 +53,7 @@ def parse stylesheet for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop + # FIXME: cache attack @styles[cl] ||= Hash.new @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5e3a1e75..e8a7b755 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -68,6 +68,7 @@ def shebang filename File.open filename, 'r' do |f| if first_line = f.gets if type = first_line[TypeFromShebang] + # FIXME: cache attack type.to_sym end end diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index d14c5a94..3a38bfd3 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -207,6 +207,7 @@ def validate_id id id elsif id.is_a? String if id[/\w+/] == id + # FIXME: cache attack id.downcase.to_sym else raise ArgumentError, "Invalid id given: #{id}" diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 566bfa77..9d108649 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -21,6 +21,7 @@ def scan_tokens encoder, options encoder.text_token match, :space elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) + # FIXME: cache attack kind = self[1].to_sym match = self[2].gsub(/\\(.)/m, '\1') unless TokenKinds.has_key? kind @@ -30,6 +31,7 @@ def scan_tokens encoder, options encoder.text_token match, kind elsif match = scan(/ (\w+) ([<\[]) /x) + # FIXME: cache attack kind = self[1].to_sym opened_tokens << kind case self[2] diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index fd1aed67..836fa416 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -21,6 +21,7 @@ def scan_tokens encoder, options line_kind = nil state = :initial deleted_lines_count = 0 + # FIXME: cache attack scanners = Hash.new do |h, lang| h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true end diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 09c8b6e7..23630f98 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -75,10 +75,12 @@ class Python < Scanner <<=? | >>=? | [<>=]=? | != # comparison and assignment /x # :nodoc: + # FIXME: cache attack STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter| h[delimiter] = Regexp.union delimiter # :nodoc: } + # FIXME: cache attack STRING_CONTENT_REGEXP = Hash.new { |h, delimiter| h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x # :nodoc: } @@ -183,6 +185,7 @@ def scan_tokens encoder, options kind = :ident elsif kind == :keyword state = DEF_NEW_STATE[match] + # FIXME: cache attack from_import_state << match.to_sym if state == :include_expected end encoder.text_token match, kind diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index d39d9626..ca35de02 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -26,6 +26,7 @@ def scan_tokens encoder, options encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] + # FIXME: cache attack encoder.text_token match, kind.to_sym unless match.empty? encoder.text_token match, :operator if match = scan(/\)/) @@ -39,6 +40,7 @@ def scan_tokens encoder, options else raise 'CodeRay bug: This case should not be reached.' end + # FIXME: cache attack kind = kind.to_sym opened_tokens << kind encoder.begin_group kind diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 2f398d1e..fe37d07b 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -14,6 +14,7 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, { } ] ].each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with << + # FIXME: cache attack STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k # delim = delim.dup # workaround for old Ruby From ea107396fdd72cdbbaf4820d09a87bd879ba7e6c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 16:43:19 +0200 Subject: [PATCH 080/236] check token kinds in Lint encoders --- lib/coderay/encoders/debug_lint.rb | 3 ++- lib/coderay/encoders/lint.rb | 4 +++- lib/coderay/token_kinds.rb | 5 +---- test/functional/basic.rb | 4 +--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index 2c141863..a4eba2c7 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -18,7 +18,8 @@ class DebugLint < Debug register_for :debug_lint def text_token text, kind - raise Lint::EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind super end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb index 4601e902..88c8bd1d 100644 --- a/lib/coderay/encoders/lint.rb +++ b/lib/coderay/encoders/lint.rb @@ -17,10 +17,12 @@ class Lint < Debug InvalidTokenStream = Class.new StandardError EmptyToken = Class.new InvalidTokenStream + UnknownTokenKind = Class.new InvalidTokenStream IncorrectTokenGroupNesting = Class.new InvalidTokenStream def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind end def begin_group kind diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 9137a49f..5f49d775 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,10 +1,7 @@ module CodeRay # A Hash of all known token kinds and their associated CSS classes. - TokenKinds = Hash.new do |h, k| - warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG - false - end + TokenKinds = Hash.new(false) # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity diff --git a/test/functional/basic.rb b/test/functional/basic.rb index 3053b543..752d4ba0 100755 --- a/test/functional/basic.rb +++ b/test/functional/basic.rb @@ -164,9 +164,7 @@ def test_token_kinds end end assert_equal 'reserved', CodeRay::TokenKinds[:reserved] - assert_warning 'Undefined Token kind: :shibboleet' do - assert_equal false, CodeRay::TokenKinds[:shibboleet] - end + assert_equal false, CodeRay::TokenKinds[:shibboleet] end class Milk < CodeRay::Encoders::Encoder From 60afd6857c8d0f1c3f9f2d6ca45f01b216d6b4b5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 16:49:40 +0200 Subject: [PATCH 081/236] no attack vector found --- lib/coderay/encoders/html.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 6dd231ae..ee2d91a5 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -142,7 +142,6 @@ def self.make_html_escape_hash HTML_ESCAPE = make_html_escape_hash HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ - # FIXME: cache attack TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } end From 5cd749771379b9832ab1b37936bd98fb7cc80a34 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 17:04:09 +0200 Subject: [PATCH 082/236] don't dup @span_for_kinds hash keys --- lib/coderay/encoders/html.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ee2d91a5..ad5fafcf 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -286,7 +286,7 @@ def style_for_kinds kinds def make_span_for_kinds method, hint # FIXME: cache attack Hash.new do |h, kinds| - h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + h[kinds] = begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint From ee30738b0b0615715321aa4f1ed8c7e4025cb411 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 17:04:23 +0200 Subject: [PATCH 083/236] rename local variable --- lib/coderay/encoders/html.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ad5fafcf..81a6ffa7 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -310,8 +310,8 @@ def check_group_nesting name, kind def break_lines text, style reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '') + @opened.each_with_index do |kind, index| + reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '') end text.gsub("\n", "#{'' * @opened.size}#{'' if style}\n#{reopen}#{style}") end From e2546068d0f16fcba15268e740bbb6d9f4f223e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:28:54 +0200 Subject: [PATCH 084/236] prevent Symbol attack in Raydebug scanner --- lib/coderay/scanners/raydebug.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index ca35de02..6c1c10f7 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,3 +1,5 @@ +require 'set' + module CodeRay module Scanners @@ -12,6 +14,11 @@ class Raydebug < Scanner protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -26,8 +33,13 @@ def scan_tokens encoder, options encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - # FIXME: cache attack - encoder.text_token match, kind.to_sym unless match.empty? + unless match.empty? + if @known_token_kinds.include? kind + encoder.text_token match, kind.to_sym + else + encoder.text_token match, :plain + end + end encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) From 8ee1c8deedc58672aa46f311163c2178a70186ce Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:44:01 +0200 Subject: [PATCH 085/236] cleanup Plugin, don't use #to_sym anymore --- lib/coderay/helpers/plugin.rb | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index 3a38bfd3..9a724fff 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -30,7 +30,7 @@ module PluginHost # * a file could not be found # * the requested Plugin is not registered PluginNotFound = Class.new LoadError - HostNotFound = Class.new LoadError + HostNotFound = Class.new LoadError PLUGIN_HOSTS = [] PLUGIN_HOSTS_BY_ID = {} # dummy hash @@ -49,8 +49,8 @@ def load_all def [] id, *args, &blk plugin = validate_id(id) begin - plugin = plugin_hash.[] plugin, *args, &blk - end while plugin.is_a? Symbol + plugin = plugin_hash.[](plugin, *args, &blk) + end while plugin.is_a? String plugin end @@ -95,7 +95,7 @@ def plugin_path *args def map hash for from, to in hash from = validate_id from - to = validate_id to + to = validate_id to plugin_hash[from] = to unless plugin_hash.has_key? from end end @@ -197,23 +197,22 @@ def path_to plugin_id File.join plugin_path, "#{plugin_id}.rb" end - # Converts +id+ to a Symbol if it is a String, - # or returns +id+ if it already is a Symbol. + # Converts +id+ to a valid plugin ID String, or returns +nil+. # # Raises +ArgumentError+ for all other objects, or if the # given String includes non-alphanumeric characters (\W). def validate_id id - if id.is_a? Symbol or id.nil? - id - elsif id.is_a? String + case id + when Symbol + id.to_s + when String if id[/\w+/] == id - # FIXME: cache attack - id.downcase.to_sym + id.downcase else raise ArgumentError, "Invalid id given: #{id}" end else - raise ArgumentError, "String or Symbol expected, but #{id.class} given." + raise ArgumentError, "Symbol or String expected, but #{id.class} given." end end From 6ef7fa4541230442b6e743042648320619ad6859 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:44:55 +0200 Subject: [PATCH 086/236] no attack vector found --- lib/coderay/helpers/file_type.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index e8a7b755..5e3a1e75 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -68,7 +68,6 @@ def shebang filename File.open filename, 'r' do |f| if first_line = f.gets if type = first_line[TypeFromShebang] - # FIXME: cache attack type.to_sym end end From 2ab42c7b5e674453fac0320fe0c4a40daf6197e1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 18:53:41 +0200 Subject: [PATCH 087/236] prevent Symbol attack in Debug scanner --- lib/coderay/scanners/debug.rb | 39 ++++++++++++++++++++--------------- test/unit/debug.rb | 12 +++++------ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 9d108649..ac12c16c 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -1,3 +1,5 @@ +require 'set' + module CodeRay module Scanners @@ -11,6 +13,11 @@ class Debug < Scanner protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -21,26 +28,24 @@ def scan_tokens encoder, options encoder.text_token match, :space elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) - # FIXME: cache attack - kind = self[1].to_sym - match = self[2].gsub(/\\(.)/m, '\1') - unless TokenKinds.has_key? kind - kind = :error - match = matched + if @known_token_kinds.include? self[1] + encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym + else + encoder.text_token matched, :error end - encoder.text_token match, kind elsif match = scan(/ (\w+) ([<\[]) /x) - # FIXME: cache attack - kind = self[1].to_sym - opened_tokens << kind - case self[2] - when '<' - encoder.begin_group kind - when '[' - encoder.begin_line kind - else - raise 'CodeRay bug: This case should not be reached.' + if @known_token_kinds.include? self[1] + kind = self[1].to_sym + opened_tokens << kind + case self[2] + when '<' + encoder.begin_group kind + when '[' + encoder.begin_line kind + else + raise 'CodeRay bug: This case should not be reached.' + end end elsif !opened_tokens.empty? && match = scan(/ > /x) diff --git a/test/unit/debug.rb b/test/unit/debug.rb index f2b80bd4..616cda5d 100644 --- a/test/unit/debug.rb +++ b/test/unit/debug.rb @@ -18,15 +18,15 @@ def test_creation [:begin_group, :string], ['test', :content], [:end_group, :string], - [:begin_line, :test], + [:begin_line, :head], ["\n", :space], ["\n \t", :space], [" \n", :space], ["[]", :method], - [:end_line, :test], + [:end_line, :head], ].flatten TEST_OUTPUT = <<-'DEBUG'.chomp -integer(10)operator((\\\))stringtest[ +integer(10)operator((\\\))stringhead[ method([])] @@ -51,7 +51,7 @@ def test_creation end TEST_INPUT = <<-'DEBUG'.chomp -integer(10)operator((\\\))stringtest[ +integer(10)operator((\\\))stringhead[ method([])] @@ -62,10 +62,10 @@ def test_creation [:begin_group, :string], ['test', :content], [:end_group, :string], - [:begin_line, :test], + [:begin_line, :head], ["\n\n \t \n", :space], ["[]", :method], - [:end_line, :test], + [:end_line, :head], ].flatten def test_filtering_text_tokens From 5d6bee7f5caced1383e6aac427fb356a4788794b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:14:21 +0200 Subject: [PATCH 088/236] tweak Debug scanners again, introduce :unknown token kind --- lib/coderay/scanners/debug.rb | 25 ++++++++++++++----------- lib/coderay/scanners/raydebug.rb | 17 ++++++----------- lib/coderay/token_kinds.rb | 3 ++- test/unit/debug.rb | 6 +++--- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index ac12c16c..83ede9a5 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -5,7 +5,7 @@ module Scanners # = Debug Scanner # - # Interprets the output of the Encoders::Debug encoder. + # Interprets the output of the Encoders::Debug encoder (basically the inverse function). class Debug < Scanner register_for :debug @@ -31,21 +31,24 @@ def scan_tokens encoder, options if @known_token_kinds.include? self[1] encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym else - encoder.text_token matched, :error + encoder.text_token matched, :unknown end elsif match = scan(/ (\w+) ([<\[]) /x) if @known_token_kinds.include? self[1] kind = self[1].to_sym - opened_tokens << kind - case self[2] - when '<' - encoder.begin_group kind - when '[' - encoder.begin_line kind - else - raise 'CodeRay bug: This case should not be reached.' - end + else + kind = :unknown + end + + opened_tokens << kind + case self[2] + when '<' + encoder.begin_group kind + when '[' + encoder.begin_line kind + else + raise 'CodeRay bug: This case should not be reached.' end elsif !opened_tokens.empty? && match = scan(/ > /x) diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 6c1c10f7..1effdc85 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -3,9 +3,9 @@ module CodeRay module Scanners - # = Debug Scanner + # = Raydebug Scanner # - # Parses the output of the Encoders::Debug encoder. + # Highlights the output of the Encoders::Debug encoder. class Raydebug < Scanner register_for :raydebug @@ -43,17 +43,12 @@ def scan_tokens encoder, options encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) - kind = self[1] - case self[2] - when '<' - encoder.text_token kind, :class - when '[' - encoder.text_token kind, :class + encoder.text_token self[1], :class + if @known_token_kinds.include? self[1] + kind = self[1].to_sym else - raise 'CodeRay bug: This case should not be reached.' + kind = :unknown end - # FIXME: cache attack - kind = kind.to_sym opened_tokens << kind encoder.begin_group kind encoder.text_token self[2], :operator diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index 5f49d775..f9118622 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -80,5 +80,6 @@ module CodeRay :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:unknown] = TokenKinds[:plain] end diff --git a/test/unit/debug.rb b/test/unit/debug.rb index 616cda5d..88baf563 100644 --- a/test/unit/debug.rb +++ b/test/unit/debug.rb @@ -51,7 +51,7 @@ def test_creation end TEST_INPUT = <<-'DEBUG'.chomp -integer(10)operator((\\\))stringhead[ +integer(10)operator((\\\))stringtest[ method([])] @@ -62,10 +62,10 @@ def test_creation [:begin_group, :string], ['test', :content], [:end_group, :string], - [:begin_line, :head], + [:begin_line, :unknown], ["\n\n \t \n", :space], ["[]", :method], - [:end_line, :head], + [:end_line, :unknown], ].flatten def test_filtering_text_tokens From 21d07b305f6293065cf08134cee2c66e727422cf Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:17:47 +0200 Subject: [PATCH 089/236] rename protected method in FileType --- lib/coderay/helpers/file_type.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 5e3a1e75..7de34d58 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -38,7 +38,7 @@ def [] filename, read_shebang = false (TypeFromExt[ext2.downcase] if ext2) || TypeFromName[name] || TypeFromName[name.downcase] - type ||= shebang(filename) if read_shebang + type ||= type_from_shebang(filename) if read_shebang type end @@ -63,7 +63,7 @@ def fetch filename, default = nil, read_shebang = false protected - def shebang filename + def type_from_shebang filename return unless File.exist? filename File.open filename, 'r' do |f| if first_line = f.gets From 368e053880819edc74fdcef38f38b5fd4806a3f4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:18:49 +0200 Subject: [PATCH 090/236] FileType should guard against attacks here --- lib/coderay/scanners/diff.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index 836fa416..fd1aed67 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -21,7 +21,6 @@ def scan_tokens encoder, options line_kind = nil state = :initial deleted_lines_count = 0 - # FIXME: cache attack scanners = Hash.new do |h, lang| h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true end From af04107b8b370452a17fa54e8ea0e8adc8b376b0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:19:57 +0200 Subject: [PATCH 091/236] no attack vector, there are only 4 cases --- lib/coderay/scanners/python.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 23630f98..05e1f5f0 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -75,12 +75,10 @@ class Python < Scanner <<=? | >>=? | [<>=]=? | != # comparison and assignment /x # :nodoc: - # FIXME: cache attack STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter| h[delimiter] = Regexp.union delimiter # :nodoc: } - # FIXME: cache attack STRING_CONTENT_REGEXP = Hash.new { |h, delimiter| h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x # :nodoc: } From e9140073f4dcba5c022a2ad40a1b935a07a6b4c3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:21:45 +0200 Subject: [PATCH 092/236] no attack vector, there are only 2 cases --- lib/coderay/scanners/python.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 05e1f5f0..09c8b6e7 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -183,7 +183,6 @@ def scan_tokens encoder, options kind = :ident elsif kind == :keyword state = DEF_NEW_STATE[match] - # FIXME: cache attack from_import_state << match.to_sym if state == :include_expected end encoder.text_token match, kind From c3c70e0b3497939dbfb1958a0764f4fd18c05a48 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:31:35 +0200 Subject: [PATCH 093/236] cleanup --- lib/coderay/scanners/ruby/string_state.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index fe37d07b..bcc0507d 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -17,7 +17,6 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, # FIXME: cache attack STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k - # delim = delim.dup # workaround for old Ruby delim_pattern = Regexp.escape(delim) if closing_paren = CLOSING_PAREN[delim] delim_pattern << Regexp.escape(closing_paren) From 65983f38eaed758a9901adf9e4e8c4be3e3a6123 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:41:55 +0200 Subject: [PATCH 094/236] avoid cache attack in Ruby scanner (eg. using Unicode-delimited Fancy Strings) --- lib/coderay/scanners/ruby/string_state.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index bcc0507d..28ddd6c6 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -14,7 +14,6 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, { } ] ].each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with << - # FIXME: cache attack STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k delim_pattern = Regexp.escape(delim) @@ -29,12 +28,13 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, # '| [|?*+(){}\[\].^$]' # end - h[k] = - if interpreted && delim != '#' - / (?= [#{delim_pattern}] | \# [{$@] ) /mx - else - / (?= [#{delim_pattern}] ) /mx - end + if interpreted && delim != '#' + / (?= [#{delim_pattern}] | \# [{$@] ) /mx + else + / (?= [#{delim_pattern}] ) /mx + end.tap do |pattern| + h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256 + end end def initialize kind, interpreted, delim, heredoc = false From 05f5a0e270ce2cde4ff242634033c902c58f13ea Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:45:33 +0200 Subject: [PATCH 095/236] no cache attacks possible, static input (CSS) --- lib/coderay/encoders/html/css.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index de98f0ed..164d7f85 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -21,7 +21,6 @@ def initialize style = :default end def get_style_for_css_classes css_classes - # FIXME: cache attack cl = @styles[css_classes.first] return '' unless cl style = '' @@ -53,7 +52,6 @@ def parse stylesheet for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop - # FIXME: cache attack @styles[cl] ||= Hash.new @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end From ee992427810a1cf88b53c12ccf7fda91a30ab33e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 21 Jul 2013 20:58:07 +0200 Subject: [PATCH 096/236] limit HTML encoder span_for_kinds cache size --- lib/coderay/encoders/html.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 81a6ffa7..ffde5d2d 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -284,9 +284,8 @@ def style_for_kinds kinds end def make_span_for_kinds method, hint - # FIXME: cache attack Hash.new do |h, kinds| - h[kinds] = begin + begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint @@ -298,6 +297,9 @@ def make_span_for_kinds method, hint "" end end + end.tap do |span| + h.clear if h.size >= 100 + h[kinds] = span end end end From 4d472e2399fbd169ab40be2e0d72c2dc328b2d7a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 23 Aug 2013 14:57:02 +0200 Subject: [PATCH 097/236] changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index f57faf5f..44d0d937 100644 --- a/Changes.textile +++ b/Changes.textile @@ -27,6 +27,7 @@ h2. Changes in 1.1 * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] * Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style +* Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148] * @Plugin@ does not warn about fallback when default is defined * @HTML@ encoder will not warn about unclosed token groups at the end of the stream * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now From 28c57a5f02ca066e66346a69db1bfe33fc6bfb6e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 23 Aug 2013 15:03:04 +0200 Subject: [PATCH 098/236] more changelog --- Changes.textile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changes.textile b/Changes.textile index 44d0d937..e54970d9 100644 --- a/Changes.textile +++ b/Changes.textile @@ -24,10 +24,14 @@ h2. Changes in 1.1 * New token type @:id@ for CSS/Sass [#27] * New token type @:done@ for Taskpaper [#39] * New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] +* New token type @:unknown@ for Debug scanner * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] * Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style * Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148] +* @PluginHost@ now works with Strings instead of Symbols internally (to avoid using @#to_sym@) +* The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input) +* The @Raydebug@ scanner highlights unknown token kinds as @:plain@ * @Plugin@ does not warn about fallback when default is defined * @HTML@ encoder will not warn about unclosed token groups at the end of the stream * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now From a31b36683834f39c1581add498cce0b016f20fb5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 30 Aug 2013 16:22:19 +0200 Subject: [PATCH 099/236] fix coderay -HTML option --- Changes.textile | 1 + lib/coderay/encoders/html.rb | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Changes.textile b/Changes.textile index e54970d9..cd4443ab 100644 --- a/Changes.textile +++ b/Changes.textile @@ -33,6 +33,7 @@ h2. Changes in 1.1 * The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input) * The @Raydebug@ scanner highlights unknown token kinds as @:plain@ * @Plugin@ does not warn about fallback when default is defined +* Fixed @HTML@ encoder when output is a StringIO (eg. when using @-HTML@) * @HTML@ encoder will not warn about unclosed token groups at the end of the stream * @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now * @Debug@ encoder will not warn about errors in the token stream diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index ffde5d2d..d2ebb5af 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -197,13 +197,15 @@ def finish options @last_opened = nil end - @out.extend Output - @out.css = @css - if options[:line_numbers] - Numbering.number! @out, options[:line_numbers], options + if @out.respond_to? :to_str + @out.extend Output + @out.css = @css + if options[:line_numbers] + Numbering.number! @out, options[:line_numbers], options + end + @out.wrap! options[:wrap] + @out.apply_title! options[:title] end - @out.wrap! options[:wrap] - @out.apply_title! options[:title] if defined?(@real_out) && @real_out @real_out << @out From 718c0ac901eef189a1dad36f57a78f36d9d0ba11 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 30 Aug 2013 17:11:46 +0200 Subject: [PATCH 100/236] cleanup --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 55770098..c9b1e8a3 100644 --- a/Rakefile +++ b/Rakefile @@ -34,4 +34,4 @@ else rd.rdoc_dir = 'doc' end -end \ No newline at end of file +end From 73da367263ac9b59ebdb531d7d4cc9302932099f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 1 Sep 2013 00:28:21 +0200 Subject: [PATCH 101/236] cleanup changelog for 1.1 --- Changes.textile | 69 ++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/Changes.textile b/Changes.textile index cd4443ab..f8081dad 100644 --- a/Changes.textile +++ b/Changes.textile @@ -4,40 +4,57 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1 -* New scanner: Lua [#21, #22, thanks to Quintus] -* New scanner: Sass [#93] -* New scanner: Go [#28, thanks to Eric Guo and Nathan Youngman] -* New scanner: Taskpaper [#39, thanks to shimomura] +New scanners: + +* Go [#28, thanks to Eric Guo and Nathan Youngman] +* Lua [#21, #22, thanks to Quintus] +* Sass [#93] +* Taskpaper [#39, thanks to shimomura] + +More new stuff: + +* Add .xaml file type [#121, thanks to Kozman Bálint] +* New token type @:id@ for CSS/Sass [#27] +* New token type @:done@ for Taskpaper [#39] +* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] +* New token type @:unknown@ for Debug scanner +* New DebugLint encoder that checks for empty tokens and correct nesting + +Improvements: + +* CSS scanner uses @:id@ and @:tag@ now [#27] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly +* HTML scanner displays style tags and attributes now [#145] * Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman] * Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] -* HTML scanner displays style tags and attributes now [#145] -* Remove double-click toggle handler from HTML table output + +Removed: + +* @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency +* Double-click toggle handler from HTML table output + +Fixes: + * Fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143] * Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] -* Added @:keep_state@ functionality to more scanners [#116] -* CSS scanner uses @:id@ and @:tag@ now [#27] -* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right? -* Add .xaml file type [#121, thanks to Kozman Bálint] -* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] -* New token type @:id@ for CSS/Sass [#27] -* New token type @:done@ for Taskpaper [#39] -* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] -* New token type @:unknown@ for Debug scanner -* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] -* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * Fixed @:docstring@ token type style * Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148] -* @PluginHost@ now works with Strings instead of Symbols internally (to avoid using @#to_sym@) -* The @Debug@ scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input) -* The @Raydebug@ scanner highlights unknown token kinds as @:plain@ -* @Plugin@ does not warn about fallback when default is defined -* Fixed @HTML@ encoder when output is a StringIO (eg. when using @-HTML@) -* @HTML@ encoder will not warn about unclosed token groups at the end of the stream -* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now -* @Debug@ encoder will not warn about errors in the token stream -* New @DebugLint@ encoder that checks for empty tokens and correct nesting +* Fixed HTML encoder when output is a StringIO (eg. when using @-HTML@ as a command line parameter) +* TokenKinds should not be frozen [#130, thanks to Gavin Kistner] +* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] +* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] +* HTML encoder will not warn about unclosed token groups at the end of the stream + +Internals: + +* The Debug scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input) +* The Raydebug scanner highlights unknown token kinds as @:plain@ +* Debug encoder refactored; use DebugLint if you want strict checking now +* Debug encoder will not warn about errors in the token stream +* Plugin does not warn about fallback when default is defined +* PluginHost now works with Strings instead of Symbols internally (to avoid using @#to_sym@) +* Added @:keep_state@ functionality to more scanners [#116] h2. Changes in 1.0.9 From a48037b85a12228431b32103786456f36beb355f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 1 Sep 2013 01:01:35 +0200 Subject: [PATCH 102/236] final cleanup --- Changes.textile | 59 +++++++++++++++++++++----------------- README.markdown | 2 +- lib/coderay/scanners/go.rb | 1 - 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/Changes.textile b/Changes.textile index f8081dad..8e388e04 100644 --- a/Changes.textile +++ b/Changes.textile @@ -13,48 +13,58 @@ New scanners: More new stuff: -* Add .xaml file type [#121, thanks to Kozman Bálint] -* New token type @:id@ for CSS/Sass [#27] -* New token type @:done@ for Taskpaper [#39] -* New token type @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] -* New token type @:unknown@ for Debug scanner -* New DebugLint encoder that checks for empty tokens and correct nesting +* @.xaml@ file type [#121, thanks to Kozman Bálint] +* recognize @Guardfile@, @Vagrantfile@, and @Appraisals@ as Ruby files [#121, thanks to Kozman Bálint] +* new token kind @:id@ for CSS/Sass [#27] +* new token kind @:done@ for Taskpaper [#39] +* new token kind @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] +* new token kind @:unknown@ for Debug scanner +* new DebugLint encoder that checks for empty tokens and correct nesting Improvements: * CSS scanner uses @:id@ and @:tag@ now [#27] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly +* JSON scanner: simplify key/value heuristic, using look-ahead instead of a stack * HTML scanner displays style tags and attributes now [#145] -* Ruby scanner: Accept %i and %I symbol lists (Ruby 2.0) [thanks to Nathan Youngman] -* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] +* Ruby scanner: Accept @%i(…)@ and @%I(…)@ symbol lists (Ruby 2.0) [thanks to Nathan Youngman] +* Ruby scanner: Accept keywords as Ruby hash keys [#126] +* performance improvements to several scanners and encoders, especially Terminal and HTML +* added @:keep_state@ functionality to more scanners so they work nicely with diff now [#116] +* refactoring and cleanup to achieve better "Code Climate" ratings (but I don't really care) +* updated and cleaned up the documentation, +* documented list of TokenKinds +* Alpha style: tweaked colors for @.binary@, @.local-variable@, and @.predefined-type@ +* @rake generate@ supports Git now instead of Subversion Removed: * @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency -* Double-click toggle handler from HTML table output +* double-click toggle handler from HTML table output +* @rake_helpers@, @sample@ directories and several other ancient garbage Fixes: -* Fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143] -* Fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] -* Fixed @:docstring@ token type style -* Fixed several problems related to Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148] -* Fixed HTML encoder when output is a StringIO (eg. when using @-HTML@ as a command line parameter) +* fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143] +* fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] +* fixed @:docstring@ token type style +* fixed several infinite Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148] +* fixed HTML encoder when output is a StringIO (eg. when using @-HTML@ as a command line parameter) * TokenKinds should not be frozen [#130, thanks to Gavin Kistner] -* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] -* Override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] +* display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] +* override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] * HTML encoder will not warn about unclosed token groups at the end of the stream +* fixed problem with coderay/version.rb being loaded twice Internals: -* The Debug scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input) -* The Raydebug scanner highlights unknown token kinds as @:plain@ -* Debug encoder refactored; use DebugLint if you want strict checking now -* Debug encoder will not warn about errors in the token stream -* Plugin does not warn about fallback when default is defined -* PluginHost now works with Strings instead of Symbols internally (to avoid using @#to_sym@) -* Added @:keep_state@ functionality to more scanners [#116] +* The Debug scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input). +* The Raydebug scanner highlights unknown token kinds as @:plain@. +* The Debug encoder refactored; use DebugLint if you want strict checking now.. +* The Debug encoder will not warn about errors in the token stream. +* Plugin does not warn about fallback when default is defined. +* PluginHost now works with Strings instead of Symbols internally (to avoid using @#to_sym@). h2. Changes in 1.0.9 @@ -464,6 +474,3 @@ The helper classes were cleaned up; see above for details. * *CHANGED* @Plugin@ API was simplified and stripped of all unnecessary features. * *CHANGED* Moved @GZip@ and @FileType@ libraries into @CodeRay@; cleaned them up. - - - diff --git a/README.markdown b/README.markdown index f3336552..e23f6036 100644 --- a/README.markdown +++ b/README.markdown @@ -28,4 +28,4 @@ html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table) ## Documentation -See [http://coderay.rubychan.de/doc/](http://coderay.rubychan.de/doc/). +See [rubydoc](http://rubydoc.info/gems/coderay). diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb index 59473f6e..99fdd638 100644 --- a/lib/coderay/scanners/go.rb +++ b/lib/coderay/scanners/go.rb @@ -1,7 +1,6 @@ module CodeRay module Scanners - # Scanner for Go, copy from c class Go < Scanner register_for :go From e93aae88985667189bb5b24ad0d5f54cb5fdba70 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 20 Sep 2013 17:46:39 +0200 Subject: [PATCH 103/236] remove outdated, misleading and buggy "samples" --- rake_tasks/test.rake | 6 - test/samples/README | 1 - test/samples/cache.actual | 2 - test/samples/cache.expected | 2 - test/samples/cache.rb | 12 - test/samples/count.expected | 1 - test/samples/count.rb | 10 - test/samples/css.actual | 127 ---- test/samples/css.expected | 127 ---- test/samples/css.rb | 4 - test/samples/div.actual | 17 - test/samples/div.expected | 17 - test/samples/div.rb | 19 - test/samples/encoder.actual | 65 -- test/samples/encoder.expected | 65 -- test/samples/encoder.rb | 40 -- test/samples/global_vars.actual | 0 test/samples/global_vars.diff | 5 - test/samples/global_vars.expected | 3 - test/samples/global_vars.rb | 13 - test/samples/global_vars2.expected | 10 - test/samples/global_vars2.rb | 28 - test/samples/highlight.expected | 175 ------ test/samples/highlight.rb | 14 - test/samples/html.expected | 919 ----------------------------- test/samples/html.rb | 394 ------------- test/samples/html2.expected | 185 ------ test/samples/html2.rb | 11 - test/samples/html_list.expected | 160 ----- test/samples/html_list.rb | 12 - test/samples/load_encoder.expected | 8 - test/samples/load_encoder.rb | 25 - test/samples/load_scanner.expected | 8 - test/samples/load_scanner.rb | 25 - test/samples/more.expected | 2 - test/samples/more.rb | 205 ------- test/samples/scanner.expected | 16 - test/samples/scanner.rb | 36 -- test/samples/server.rb | 110 ---- test/samples/simple.expected | 1 - test/samples/simple.rb | 10 - test/samples/stream.rb | 25 - test/samples/stream2.expected | 2 - test/samples/stream2.rb | 8 - test/samples/suite.rb | 86 --- test/samples/tokens.expected | 1 - test/samples/tokens.rb | 3 - 47 files changed, 3015 deletions(-) delete mode 100644 test/samples/README delete mode 100644 test/samples/cache.actual delete mode 100644 test/samples/cache.expected delete mode 100644 test/samples/cache.rb delete mode 100644 test/samples/count.expected delete mode 100644 test/samples/count.rb delete mode 100644 test/samples/css.actual delete mode 100644 test/samples/css.expected delete mode 100644 test/samples/css.rb delete mode 100644 test/samples/div.actual delete mode 100644 test/samples/div.expected delete mode 100644 test/samples/div.rb delete mode 100644 test/samples/encoder.actual delete mode 100644 test/samples/encoder.expected delete mode 100644 test/samples/encoder.rb delete mode 100644 test/samples/global_vars.actual delete mode 100644 test/samples/global_vars.diff delete mode 100644 test/samples/global_vars.expected delete mode 100644 test/samples/global_vars.rb delete mode 100644 test/samples/global_vars2.expected delete mode 100644 test/samples/global_vars2.rb delete mode 100644 test/samples/highlight.expected delete mode 100644 test/samples/highlight.rb delete mode 100644 test/samples/html.expected delete mode 100644 test/samples/html.rb delete mode 100644 test/samples/html2.expected delete mode 100644 test/samples/html2.rb delete mode 100644 test/samples/html_list.expected delete mode 100644 test/samples/html_list.rb delete mode 100644 test/samples/load_encoder.expected delete mode 100644 test/samples/load_encoder.rb delete mode 100644 test/samples/load_scanner.expected delete mode 100644 test/samples/load_scanner.rb delete mode 100644 test/samples/more.expected delete mode 100644 test/samples/more.rb delete mode 100644 test/samples/scanner.expected delete mode 100644 test/samples/scanner.rb delete mode 100644 test/samples/server.rb delete mode 100644 test/samples/simple.expected delete mode 100644 test/samples/simple.rb delete mode 100644 test/samples/stream.rb delete mode 100644 test/samples/stream2.expected delete mode 100644 test/samples/stream2.rb delete mode 100644 test/samples/suite.rb delete mode 100644 test/samples/tokens.expected delete mode 100644 test/samples/tokens.rb diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 371214a2..b15b9993 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -1,9 +1,4 @@ namespace :test do - desc 'run all sample tests' - task :samples do - ruby './test/samples/suite.rb' - end - desc 'run functional tests' task :functional do ruby './test/functional/suite.rb' @@ -85,4 +80,3 @@ Please rename or remove it and run again to use the GitHub repository: end task :test => %w(test:functional test:units test:exe) -task :samples => 'test:samples' \ No newline at end of file diff --git a/test/samples/README b/test/samples/README deleted file mode 100644 index b0ab6e44..00000000 --- a/test/samples/README +++ /dev/null @@ -1 +0,0 @@ -These demos rely on Ruby 1.8.5, so the tests might fail on other versions. \ No newline at end of file diff --git a/test/samples/cache.actual b/test/samples/cache.actual deleted file mode 100644 index c131857f..00000000 --- a/test/samples/cache.actual +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/test/samples/cache.expected b/test/samples/cache.expected deleted file mode 100644 index c131857f..00000000 --- a/test/samples/cache.expected +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/test/samples/cache.rb b/test/samples/cache.rb deleted file mode 100644 index 0c0b8478..00000000 --- a/test/samples/cache.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'coderay' - -html_encoder = CodeRay.encoder :html - -scanner = Hash.new do |h, lang| - h[lang] = CodeRay.scanner lang -end - -for lang in [:ruby, :html] - tokens = scanner[lang].tokenize 'test ' - puts html_encoder.encode_tokens(tokens) -end diff --git a/test/samples/count.expected b/test/samples/count.expected deleted file mode 100644 index 7f493b6c..00000000 --- a/test/samples/count.expected +++ /dev/null @@ -1 +0,0 @@ -2 out of 4 tokens have the kind :integer. diff --git a/test/samples/count.rb b/test/samples/count.rb deleted file mode 100644 index bcb7c2dc..00000000 --- a/test/samples/count.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'coderay' - -stats = CodeRay.encoder(:statistic) -stats.encode("puts 17 + 4\n", :ruby) - -puts '%d out of %d tokens have the kind :integer.' % [ - stats.type_stats[:integer].count, - stats.real_token_count -] -#-> 2 out of 4 tokens have the kind :integer. diff --git a/test/samples/css.actual b/test/samples/css.actual deleted file mode 100644 index be73a7f9..00000000 --- a/test/samples/css.actual +++ /dev/null @@ -1,127 +0,0 @@ -.CodeRay { - background-color: hsl(0,0%,95%); - border: 1px solid silver; - color: black; -} -.CodeRay pre { - margin: 0px; -} - -span.CodeRay { white-space: pre; border: 0px; padding: 2px; } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } -table.CodeRay td { padding: 2px 4px; vertical-align: top; } - -.CodeRay .line-numbers { - background-color: hsl(180,65%,90%); - color: gray; - text-align: right; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; -} -.CodeRay .line-numbers a { - background-color: hsl(180,65%,90%) !important; - color: gray !important; - text-decoration: none !important; -} -.CodeRay .line-numbers pre { - word-break: normal; -} -.CodeRay .line-numbers a:target { color: blue !important; } -.CodeRay .line-numbers .highlighted { color: red !important; } -.CodeRay .line-numbers .highlighted a { color: red !important; } -.CodeRay span.line-numbers { padding: 0px 4px; } -.CodeRay .line { display: block; float: left; width: 100%; } -.CodeRay .code { width: 100%; } - -.CodeRay .debug { color: white !important; background: blue !important; } - -.CodeRay .annotation { color:#007 } -.CodeRay .attribute-name { color:#b48 } -.CodeRay .attribute-value { color:#700 } -.CodeRay .binary { color:#549 } -.CodeRay .binary .char { color:#325 } -.CodeRay .binary .delimiter { color:#325 } -.CodeRay .char { color:#D20 } -.CodeRay .char .content { color:#D20 } -.CodeRay .char .delimiter { color:#710 } -.CodeRay .class { color:#B06; font-weight:bold } -.CodeRay .class-variable { color:#369 } -.CodeRay .color { color:#0A0 } -.CodeRay .comment { color:#777 } -.CodeRay .comment .char { color:#444 } -.CodeRay .comment .delimiter { color:#444 } -.CodeRay .constant { color:#036; font-weight:bold } -.CodeRay .decorator { color:#B0B } -.CodeRay .definition { color:#099; font-weight:bold } -.CodeRay .delimiter { color:black } -.CodeRay .directive { color:#088; font-weight:bold } -.CodeRay .docstring { color:#D42; } -.CodeRay .doctype { color:#34b } -.CodeRay .done { text-decoration: line-through; color: gray } -.CodeRay .entity { color:#800; font-weight:bold } -.CodeRay .error { color:#F00; background-color:#FAA } -.CodeRay .escape { color:#666 } -.CodeRay .exception { color:#C00; font-weight:bold } -.CodeRay .float { color:#60E } -.CodeRay .function { color:#06B; font-weight:bold } -.CodeRay .function .delimiter { color:#024; font-weight:bold } -.CodeRay .global-variable { color:#d70 } -.CodeRay .hex { color:#02b } -.CodeRay .id { color:#33D; font-weight:bold } -.CodeRay .include { color:#B44; font-weight:bold } -.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.CodeRay .inline-delimiter { font-weight: bold; color: #666 } -.CodeRay .instance-variable { color:#33B } -.CodeRay .integer { color:#00D } -.CodeRay .imaginary { color:#f00 } -.CodeRay .important { color:#D00 } -.CodeRay .key { color: #606 } -.CodeRay .key .char { color: #60f } -.CodeRay .key .delimiter { color: #404 } -.CodeRay .keyword { color:#080; font-weight:bold } -.CodeRay .label { color:#970; font-weight:bold } -.CodeRay .local-variable { color:#963 } -.CodeRay .namespace { color:#707; font-weight:bold } -.CodeRay .octal { color:#40E } -.CodeRay .operator { } -.CodeRay .predefined { color:#369; font-weight:bold } -.CodeRay .predefined-constant { color:#069 } -.CodeRay .predefined-type { color:#0a5; font-weight:bold } -.CodeRay .preprocessor { color:#579 } -.CodeRay .pseudo-class { color:#00C; font-weight:bold } -.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } -.CodeRay .regexp .content { color:#808 } -.CodeRay .regexp .delimiter { color:#404 } -.CodeRay .regexp .modifier { color:#C2C } -.CodeRay .reserved { color:#080; font-weight:bold } -.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } -.CodeRay .shell .content { color:#2B2 } -.CodeRay .shell .delimiter { color:#161 } -.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } -.CodeRay .string .char { color: #b0b } -.CodeRay .string .content { color: #D20 } -.CodeRay .string .delimiter { color: #710 } -.CodeRay .string .modifier { color: #E40 } -.CodeRay .symbol { color:#A60 } -.CodeRay .symbol .content { color:#A60 } -.CodeRay .symbol .delimiter { color:#630 } -.CodeRay .tag { color:#070 } -.CodeRay .type { color:#339; font-weight:bold } -.CodeRay .value { color: #088 } -.CodeRay .variable { color:#037 } - -.CodeRay .insert { background: hsla(120,100%,50%,0.12) } -.CodeRay .delete { background: hsla(0,100%,50%,0.12) } -.CodeRay .change { color: #bbf; background: #007 } -.CodeRay .head { color: #f8f; background: #505 } -.CodeRay .head .filename { color: white; } - -.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } -.CodeRay .change .change { color: #88f } -.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.expected b/test/samples/css.expected deleted file mode 100644 index be73a7f9..00000000 --- a/test/samples/css.expected +++ /dev/null @@ -1,127 +0,0 @@ -.CodeRay { - background-color: hsl(0,0%,95%); - border: 1px solid silver; - color: black; -} -.CodeRay pre { - margin: 0px; -} - -span.CodeRay { white-space: pre; border: 0px; padding: 2px; } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } -table.CodeRay td { padding: 2px 4px; vertical-align: top; } - -.CodeRay .line-numbers { - background-color: hsl(180,65%,90%); - color: gray; - text-align: right; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; -} -.CodeRay .line-numbers a { - background-color: hsl(180,65%,90%) !important; - color: gray !important; - text-decoration: none !important; -} -.CodeRay .line-numbers pre { - word-break: normal; -} -.CodeRay .line-numbers a:target { color: blue !important; } -.CodeRay .line-numbers .highlighted { color: red !important; } -.CodeRay .line-numbers .highlighted a { color: red !important; } -.CodeRay span.line-numbers { padding: 0px 4px; } -.CodeRay .line { display: block; float: left; width: 100%; } -.CodeRay .code { width: 100%; } - -.CodeRay .debug { color: white !important; background: blue !important; } - -.CodeRay .annotation { color:#007 } -.CodeRay .attribute-name { color:#b48 } -.CodeRay .attribute-value { color:#700 } -.CodeRay .binary { color:#549 } -.CodeRay .binary .char { color:#325 } -.CodeRay .binary .delimiter { color:#325 } -.CodeRay .char { color:#D20 } -.CodeRay .char .content { color:#D20 } -.CodeRay .char .delimiter { color:#710 } -.CodeRay .class { color:#B06; font-weight:bold } -.CodeRay .class-variable { color:#369 } -.CodeRay .color { color:#0A0 } -.CodeRay .comment { color:#777 } -.CodeRay .comment .char { color:#444 } -.CodeRay .comment .delimiter { color:#444 } -.CodeRay .constant { color:#036; font-weight:bold } -.CodeRay .decorator { color:#B0B } -.CodeRay .definition { color:#099; font-weight:bold } -.CodeRay .delimiter { color:black } -.CodeRay .directive { color:#088; font-weight:bold } -.CodeRay .docstring { color:#D42; } -.CodeRay .doctype { color:#34b } -.CodeRay .done { text-decoration: line-through; color: gray } -.CodeRay .entity { color:#800; font-weight:bold } -.CodeRay .error { color:#F00; background-color:#FAA } -.CodeRay .escape { color:#666 } -.CodeRay .exception { color:#C00; font-weight:bold } -.CodeRay .float { color:#60E } -.CodeRay .function { color:#06B; font-weight:bold } -.CodeRay .function .delimiter { color:#024; font-weight:bold } -.CodeRay .global-variable { color:#d70 } -.CodeRay .hex { color:#02b } -.CodeRay .id { color:#33D; font-weight:bold } -.CodeRay .include { color:#B44; font-weight:bold } -.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.CodeRay .inline-delimiter { font-weight: bold; color: #666 } -.CodeRay .instance-variable { color:#33B } -.CodeRay .integer { color:#00D } -.CodeRay .imaginary { color:#f00 } -.CodeRay .important { color:#D00 } -.CodeRay .key { color: #606 } -.CodeRay .key .char { color: #60f } -.CodeRay .key .delimiter { color: #404 } -.CodeRay .keyword { color:#080; font-weight:bold } -.CodeRay .label { color:#970; font-weight:bold } -.CodeRay .local-variable { color:#963 } -.CodeRay .namespace { color:#707; font-weight:bold } -.CodeRay .octal { color:#40E } -.CodeRay .operator { } -.CodeRay .predefined { color:#369; font-weight:bold } -.CodeRay .predefined-constant { color:#069 } -.CodeRay .predefined-type { color:#0a5; font-weight:bold } -.CodeRay .preprocessor { color:#579 } -.CodeRay .pseudo-class { color:#00C; font-weight:bold } -.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } -.CodeRay .regexp .content { color:#808 } -.CodeRay .regexp .delimiter { color:#404 } -.CodeRay .regexp .modifier { color:#C2C } -.CodeRay .reserved { color:#080; font-weight:bold } -.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } -.CodeRay .shell .content { color:#2B2 } -.CodeRay .shell .delimiter { color:#161 } -.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } -.CodeRay .string .char { color: #b0b } -.CodeRay .string .content { color: #D20 } -.CodeRay .string .delimiter { color: #710 } -.CodeRay .string .modifier { color: #E40 } -.CodeRay .symbol { color:#A60 } -.CodeRay .symbol .content { color:#A60 } -.CodeRay .symbol .delimiter { color:#630 } -.CodeRay .tag { color:#070 } -.CodeRay .type { color:#339; font-weight:bold } -.CodeRay .value { color: #088 } -.CodeRay .variable { color:#037 } - -.CodeRay .insert { background: hsla(120,100%,50%,0.12) } -.CodeRay .delete { background: hsla(0,100%,50%,0.12) } -.CodeRay .change { color: #bbf; background: #007 } -.CodeRay .head { color: #f8f; background: #505 } -.CodeRay .head .filename { color: white; } - -.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } -.CodeRay .change .change { color: #88f } -.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.rb b/test/samples/css.rb deleted file mode 100644 index 52e4bcc7..00000000 --- a/test/samples/css.rb +++ /dev/null @@ -1,4 +0,0 @@ -require 'coderay' - -# print the default stylesheet for HTML codes -puts CodeRay::Encoders[:html]::CSS.new.stylesheet diff --git a/test/samples/div.actual b/test/samples/div.actual deleted file mode 100644 index d1e692ab..00000000 --- a/test/samples/div.actual +++ /dev/null @@ -1,17 +0,0 @@ -
- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/test/samples/div.expected b/test/samples/div.expected deleted file mode 100644 index d1e692ab..00000000 --- a/test/samples/div.expected +++ /dev/null @@ -1,17 +0,0 @@ -
- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/test/samples/div.rb b/test/samples/div.rb deleted file mode 100644 index 27b6f328..00000000 --- a/test/samples/div.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'coderay' - -puts CodeRay.scan(DATA.read, :ruby).div - -__END__ -for a in 0..255 - a = a.chr - begin - x = eval("?\\#{a}") - if x == a[0] - next - else - print "#{a}: #{x}" - end - rescue SyntaxError => boom - print "#{a}: error" - end - puts -end diff --git a/test/samples/encoder.actual b/test/samples/encoder.actual deleted file mode 100644 index 8bd83a92..00000000 --- a/test/samples/encoder.actual +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" -compressed: 79 byte < 1200 byte - -Undump: - -Code Statistics - -Tokens 800 - Non-Whitespace 400 -Bytes Total 1200 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 800 100.00 % 1.5 - space 400 50.00 % 1.0 - integer 200 25.00 % 1.5 - ident 100 12.50 % 4.0 - operator 100 12.50 % 1.0 - diff --git a/test/samples/encoder.expected b/test/samples/encoder.expected deleted file mode 100644 index 8bd83a92..00000000 --- a/test/samples/encoder.expected +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" -compressed: 79 byte < 1200 byte - -Undump: - -Code Statistics - -Tokens 800 - Non-Whitespace 400 -Bytes Total 1200 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 800 100.00 % 1.5 - space 400 50.00 % 1.0 - integer 200 25.00 % 1.5 - ident 100 12.50 % 4.0 - operator 100 12.50 % 1.0 - diff --git a/test/samples/encoder.rb b/test/samples/encoder.rb deleted file mode 100644 index bfcfbfa4..00000000 --- a/test/samples/encoder.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'coderay' - -SAMPLE = "puts 17 + 4\n" -puts 'Encoders Demo: ' + SAMPLE -scanner = CodeRay::Scanners[:ruby].new SAMPLE -encoder = CodeRay::Encoders[:statistic].new - -tokens = scanner.tokenize -stats = encoder.encode_tokens tokens - -puts -puts 'Statistic:' -puts stats - -# alternative 1 -tokens = CodeRay.scan SAMPLE, :ruby -encoder = CodeRay.encoder(:json) -textual = encoder.encode_tokens tokens -puts -puts 'Original text:' -puts textual - -# alternative 2 -yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby -puts -puts 'YAML:' -puts yaml - -# alternative 3 -require 'zlib' -BIGSAMPLE = SAMPLE * 100 -dump = Zlib::Deflate.deflate(CodeRay.scan(BIGSAMPLE, :ruby).debug) -puts -puts 'Dump:' -p dump -puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size] - -puts -puts 'Undump:' -puts CodeRay.scan(Zlib::Inflate.inflate(dump), :debug).statistic diff --git a/test/samples/global_vars.actual b/test/samples/global_vars.actual deleted file mode 100644 index e69de29b..00000000 diff --git a/test/samples/global_vars.diff b/test/samples/global_vars.diff deleted file mode 100644 index 2ed33821..00000000 --- a/test/samples/global_vars.diff +++ /dev/null @@ -1,5 +0,0 @@ -1,3d0 -< <--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -< ODER -< TEXT = <--$BLA-->.TEST(...) -\ No newline at end of file diff --git a/test/samples/global_vars.expected b/test/samples/global_vars.expected deleted file mode 100644 index 0dc13c8f..00000000 --- a/test/samples/global_vars.expected +++ /dev/null @@ -1,3 +0,0 @@ -<--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -ODER -TEXT = <--$BLA-->.TEST(...) \ No newline at end of file diff --git a/test/samples/global_vars.rb b/test/samples/global_vars.rb deleted file mode 100644 index 8066d67d..00000000 --- a/test/samples/global_vars.rb +++ /dev/null @@ -1,13 +0,0 @@ -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE - -require 'coderay' - -tokens = CodeRay.scan code, :ruby -tokens.each_text_token { |text, kind| text.upcase! } -tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text } - -print tokens diff --git a/test/samples/global_vars2.expected b/test/samples/global_vars2.expected deleted file mode 100644 index 964cf504..00000000 --- a/test/samples/global_vars2.expected +++ /dev/null @@ -1,10 +0,0 @@ - - - - - -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) - - diff --git a/test/samples/global_vars2.rb b/test/samples/global_vars2.rb deleted file mode 100644 index 76468906..00000000 --- a/test/samples/global_vars2.rb +++ /dev/null @@ -1,28 +0,0 @@ -require 'coderay' -require 'erb' -include ERB::Util - -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE -puts < - - - - -HTML - -CodeRay.scan_stream code, :ruby do |text, kind| - next if text.is_a? Symbol - text = h(text) - text = '%s' % text if kind == :global_variable - print text -end - -puts < - -HTML diff --git a/test/samples/highlight.expected b/test/samples/highlight.expected deleted file mode 100644 index 6a9b2784..00000000 --- a/test/samples/highlight.expected +++ /dev/null @@ -1,175 +0,0 @@ -
- 
puts "Hello, World!"
-
- - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-
require 'coderay'
-
-puts CodeRay.highlight('puts "Hello, World!"', :ruby)
-
-output = CodeRay.highlight_file($0, :line_numbers => :table)
-puts <<HTML
-<html>
-<head>
-#{output.stylesheet true}
-<body>
-#{output}
-</body>
-</html>
-HTML
-
- - - diff --git a/test/samples/highlight.rb b/test/samples/highlight.rb deleted file mode 100644 index 846efa45..00000000 --- a/test/samples/highlight.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'coderay' - -puts CodeRay.highlight('puts "Hello, World!"', :ruby) - -output = CodeRay.highlight_file($0, :line_numbers => :table) -puts < - -#{output.stylesheet true} - -#{output} - - -HTML diff --git a/test/samples/html.expected b/test/samples/html.expected deleted file mode 100644 index e98d5897..00000000 --- a/test/samples/html.expected +++ /dev/null @@ -1,919 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-59
-60
-61
-62
-63
-64
-65
-66
-67
-68
-69
-70
-71
-72
-73
-74
-75
-76
-77
-78
-79
-80
-81
-82
-83
-84
-85
-86
-87
-88
-89
-90
-91
-92
-93
-94
-95
-96
-97
-98
-99
-100
-101
-102
-103
-104
-105
-106
-107
-108
-109
-110
-111
-112
-113
-114
-115
-116
-117
-118
-119
-120
-121
-122
-123
-124
-125
-126
-127
-128
-129
-130
-131
-132
-133
-134
-135
-136
-137
-138
-139
-140
-141
-142
-143
-144
-145
-146
-147
-148
-149
-150
-151
-152
-153
-154
-155
-156
-157
-158
-159
-160
-161
-162
-163
-164
-165
-166
-167
-168
-169
-170
-171
-172
-173
-174
-175
-176
-177
-178
-179
-180
-181
-182
-183
-184
-185
-186
-187
-188
-189
-190
-191
-192
-193
-194
-195
-196
-197
-198
-199
-200
-201
-202
-203
-204
-205
-206
-207
-208
-209
-210
-211
-212
-213
-214
-215
-216
-217
-218
-219
-220
-221
-222
-223
-224
-225
-226
-227
-228
-229
-230
-231
-232
-233
-234
-235
-236
-237
-238
-239
-240
-241
-242
-243
-244
-245
-246
-247
-248
-249
-250
-251
-252
-253
-254
-255
-256
-257
-258
-259
-260
-261
-262
-263
-264
-265
-266
-267
-268
-269
-270
-271
-272
-273
-274
-275
-276
-277
-278
-279
-280
-281
-282
-283
-284
-285
-286
-287
-288
-289
-290
-291
-292
-293
-294
-295
-296
-297
-298
-299
-300
-301
-302
-303
-304
-305
-306
-307
-308
-309
-310
-311
-312
-313
-314
-315
-316
-317
-318
-319
-320
-321
-322
-323
-324
-325
-326
-327
-328
-329
-330
-331
-332
-333
-334
-335
-336
-337
-338
-339
-340
-341
-342
-343
-344
-345
-346
-347
-348
-349
-350
-351
-352
-353
-354
-355
-356
-357
-358
-359
-360
-361
-362
-363
-364
-365
-366
-367
-368
-369
-370
-371
-372
-373
-374
-375
-376
-377
-378
-379
-380
-381
-382
-383
-384
-385
-
require 'scanner'
-
-module CodeRay
-  
-  class RubyScanner < Scanner
-    
-    RESERVED_WORDS = [
-      'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
-      'defined?', 'ensure', 'module', 'redo', 'super', 'until',
-      'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
-      'when', 'END', 'case', 'else', 'for', 'retry',
-      'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
-      'undef', 'yield',
-    ]
-
-    DEF_KEYWORDS = ['def']
-    MODULE_KEYWORDS = ['class', 'module']
-    DEF_NEW_STATE = WordList.new(:initial).
-      add(DEF_KEYWORDS, :def_expected).
-      add(MODULE_KEYWORDS, :module_expected)
-
-    WORDS_ALLOWING_REGEXP = [
-      'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
-    ]
-    REGEXP_ALLOWED = WordList.new(false).
-      add(WORDS_ALLOWING_REGEXP, :set)
-    
-    PREDEFINED_CONSTANTS = [
-      'nil', 'true', 'false', 'self',
-      'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
-    ]
-
-    IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant)
-
-    METHOD_NAME = / #{IDENT} [?!]? /xo
-    METHOD_NAME_EX = /
-     #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
-     | \*\*?         # multiplication and power
-     | [-+~]@?       # plus, minus
-     | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
-     | \[\]=?        # array getter and setter
-     | <=?>? | >=?   # comparison, rocket operator
-     | << | >>       # append or shift left, shift right
-     | ===?          # simple equality and case equality
-    /ox
-    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
-
-    DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
-    SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
-    STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
-    SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
-    REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
-    
-    DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
-    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
-    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
-    BINARY = /0b[01]+(?:_[01]+)*/
-
-    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
-    FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
-    INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
-    
-    def reset
-      super
-      @regexp_allowed = false
-    end
-    
-    def next_token
-      return if @scanner.eos?
-
-      kind = :error
-      if @scanner.scan(/\s+/)  # in every state
-        kind = :space
-        @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
-
-      elsif @state == :def_expected
-        if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
-          kind = :method
-          @state = :initial
-        else
-          @scanner.scan(/./)
-          kind = :error
-        end
-        @state = :initial
-        
-      elsif @state == :module_expected
-        if @scanner.scan(/<</)
-          kind = :operator
-        else
-          if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
-            kind = :method
-          else
-            @scanner.scan(/./)
-            kind = :error
-          end
-          @state = :initial
-        end
-        
-      elsif # state == :initial
-        # IDENTIFIERS, KEYWORDS
-        if @scanner.scan(GLOBAL_VARIABLE)
-          kind = :global_variable
-        elsif @scanner.scan(/ @@ #{IDENT} /ox)
-          kind = :class_variable
-        elsif @scanner.scan(/ @ #{IDENT} /ox)
-          kind = :instance_variable
-        elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
-          kind = :comment
-        elsif @scanner.scan(METHOD_NAME)
-          if @last_token_dot
-            kind = :ident
-          else
-            matched = @scanner.matched
-            kind = IDENT_KIND[matched]
-            if kind == :ident and matched =~ /^[A-Z]/
-              kind = :constant
-            elsif kind == :reserved
-              @state = DEF_NEW_STATE[matched]
-              @regexp_allowed = REGEXP_ALLOWED[matched]
-            end
-          end
-          
-        elsif @scanner.scan(STRING)
-          kind = :string
-        elsif @scanner.scan(SHELL)
-          kind = :shell
-        ## HEREDOCS
-        elsif @scanner.scan(/\//) and @regexp_allowed
-           @scanner.unscan
-           @scanner.scan(REGEXP)
-          kind = :regexp
-        ## %strings
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        elsif @scanner.scan(/
-          \? (?:
-            [^\s\\]
-          | 
-            \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
-          )
-        /ox)
-          kind = :integer
-          
-        elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
-          kind = :operator
-          @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
-        elsif @scanner.scan(FLOAT)
-          kind = :float
-        elsif @scanner.scan(INTEGER)
-          kind = :integer
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        else
-          @scanner.scan(/./m)
-        end
-      end
-      
-      token = Token.new @scanner.matched, kind
-
-      if kind == :regexp
-        token.text << @scanner.scan(/[eimnosux]*/)
-      end
-      
-      @regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
-
-      token
-    end
-  end
-  
-  ScannerList.register RubyScanner, 'ruby'
-
-end
-
-module CodeRay
-  require 'scanner'
-
-  class Highlighter
-
-    def initialize lang
-      @scanner = Scanner[lang].new
-    end
-
-    def highlight code
-      @scanner.feed code
-      @scanner.all_tokens.map { |t| t.inspect }.join "\n"
-    end
-
-  end
-
-  class HTMLHighlighter < Highlighter
-    
-    ClassOfKind = {
-      :attribute_name => 'an',
-      :attribute_name_fat => 'af',
-      :attribute_value => 'av',
-      :attribute_value_fat => 'aw',
-      :bin => 'bi',
-       :char => 'ch',
-      :class => 'cl',
-      :class_variable => 'cv',
-      :color => 'cr',
-      :comment => 'c',
-      :constant => 'co',
-      :definition => 'df',
-      :directive => 'di',
-      :doc => 'do',
-      :doc_string => 'ds',
-      :exception => 'ex',
-      :error => 'er',
-      :float => 'fl',
-      :function => 'fu',
-      :global_variable => 'gv',
-      :hex => 'hx',
-      :include => 'ic',
-      :instance_variable => 'iv',
-      :integer => 'i',
-      :interpreted => 'in',
-      :label => 'la',
-      :local_variable => 'lv',
-      :oct => 'oc',
-      :operator_name => 'on',
-      :pre_constant => 'pc',
-      :pre_type => 'pt',
-      :predefined => 'pd',
-      :preprocessor => 'pp',
-      :regexp => 'rx',
-      :reserved => 'r',
-      :shell => 'sh',
-      :string => 's',
-      :symbol => 'sy',
-      :tag => 'ta',
-      :tag_fat => 'tf',
-      :tag_special => 'ts',
-      :type => 'ty',
-      :variable => 'v',
-      :xml_text => 'xt',
-
-      :ident => :NO_HIGHLIGHT,
-      :operator => :NO_HIGHLIGHT,
-      :space => :NO_HIGHLIGHT,
-    }
-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
-    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
-    
-    def initialize lang, options = {}
-      super lang
-      
-      @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
-      case level = options.fetch(:level, 'xhtml')
-        when 'html'
-          @HTML_BR = "<BR>\n"
-        when 'xhtml'
-          @HTML_BR = "<br />\n"
-      else
-        raise "Unknown HTML level: #{level}"
-      end
-    end
-
-    def highlight code
-      @scanner.feed code
-      
-      out = ''
-      while t = @scanner.next_token
-        warn t.inspect if t.text.nil?
-        out << to_html(t)
-      end
-      TEMPLATE =~ /<%CONTENT%>/
-      $` + out + $'
-    end
-    
-  private
-    def to_html token
-      css_class = ClassOfKind[token.kind]
-      if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
-        warn "no token class found for :#{token.kind}"
-      end
-        
-      text = text_to_html token.text
-      if css_class == :NO_HIGHLIGHT
-        text
-      else
-        "<span class=\"#{css_class}\">#{text}</span>"
-      end
-    end
-    
-    def text_to_html text
-      return '' if text.empty?
-      text = text.dup  # important
-      if text.index(/["><&]/)
-        text.gsub!('&', '&amp;')
-        text.gsub!('"', '&quot;')
-        text.gsub!('>', '&gt;')
-        text.gsub!('<', '&lt;')
-      end
-      if text.index(/\s/)
-        text.gsub!("\n", @HTML_BR)
-        text.gsub!("\t", @HTML_TAB)
-        text.gsub!(/^ /, '&nbsp;')
-        text.gsub!('  ', ' &nbsp;')
-      end
-      text
-    end
-    
-    TEMPLATE = <<-'TEMPLATE'
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
-<html dir="ltr">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-<meta http-equiv="Content-Style-Type" content="text/css">
-
-<title>RubyBB BBCode</title>
-<style type="text/css">
-.code {
-  width: 100%;
-  background-color: #FAFAFA;
-  border: 1px solid #D1D7DC;
-  font-family: 'Courier New', 'Terminal', monospace;
-  font-size: 10pt;
-  color: black;
-  vertical-align: top;
-  text-align: left;
-}
-.code .af { color:#00C; }
-.code .an { color:#007; }
-.code .av { color:#700; }
-.code .aw { color:#C00; }
-.code .bi { color:#509; font-weight:bold; }
-.code .c  { color:#888; }
-.code .ch { color:#C28; font-weight:bold; }
-.code .cl { color:#B06; font-weight:bold; }
-.code .co { color:#036; font-weight:bold; }
-.code .cr { color:#0A0; }
-.code .cv { color:#369; }
-.code .df { color:#099; font-weight:bold; }
-.code .di { color:#088; font-weight:bold; }
-.code .do { color:#970; }
-.code .ds { color:#D42; font-weight:bold; }
-.code .er { color:#F00; background-color:#FAA; }
-.code .ex { color:#F00; font-weight:bold; }
-.code .fl { color:#60E; font-weight:bold; }
-.code .fu { color:#06B; font-weight:bold; }
-.code .gv { color:#800; font-weight:bold; }
-.code .hx { color:#058; font-weight:bold; }
-.code .i  { color:#00D; font-weight:bold; }
-.code .ic { color:#B44; font-weight:bold; }
-.code .in { color:#B2B; font-weight:bold; }
-.code .iv { color:#33B; }
-.code .la { color:#970; font-weight:bold; }
-.code .lv { color:#963; }
-.code .oc { color:#40E; font-weight:bold; }
-.code .on { color:#000; font-weight:bold; }
-.code .pc { color:#038; font-weight:bold; }
-.code .pd { color:#369; font-weight:bold; }
-.code .pp { color:#579; }
-.code .pt { color:#339; font-weight:bold; }
-.code .r  { color:#080; font-weight:bold; }
-.code .rx { color:#927; font-weight:bold; }
-.code .s  { color:#D42; font-weight:bold; }
-.code .sh { color:#B2B; font-weight:bold; }
-.code .sy { color:#A60; }
-.code .ta { color:#070; }
-.code .tf { color:#070; font-weight:bold; }
-.code .ts { color:#D70; font-weight:bold; }
-.code .ty { color:#339; font-weight:bold; }
-.code .v  { color:#036; }
-.code .xt { color:#444; }
-</style>
-</head>
-<body>
-<div class="code">
-<%CONTENT%>
-</div>
-<div class="validators">
-<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
-<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
-</div>    
-</body>
-</html>
-    TEMPLATE
-
-  end
-
-end
-
- - - diff --git a/test/samples/html.rb b/test/samples/html.rb deleted file mode 100644 index c18284a1..00000000 --- a/test/samples/html.rb +++ /dev/null @@ -1,394 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan DATA.read, :ruby -html = tokens.page(:tab_width => 2, :line_numbers => :table, :title => 'CodeRay HTML Encoder Example') - -puts html - -__END__ -require 'scanner' - -module CodeRay - - class RubyScanner < Scanner - - RESERVED_WORDS = [ - 'and', 'def', 'end', 'in', 'or', 'unless', 'begin', - 'defined?', 'ensure', 'module', 'redo', 'super', 'until', - 'BEGIN', 'break', 'do', 'next', 'rescue', 'then', - 'when', 'END', 'case', 'else', 'for', 'retry', - 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return', - 'undef', 'yield', - ] - - DEF_KEYWORDS = ['def'] - MODULE_KEYWORDS = ['class', 'module'] - DEF_NEW_STATE = WordList.new(:initial). - add(DEF_KEYWORDS, :def_expected). - add(MODULE_KEYWORDS, :module_expected) - - WORDS_ALLOWING_REGEXP = [ - 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when' - ] - REGEXP_ALLOWED = WordList.new(false). - add(WORDS_ALLOWING_REGEXP, :set) - - PREDEFINED_CONSTANTS = [ - 'nil', 'true', 'false', 'self', - 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__', - ] - - IDENT_KIND = WordList.new(:ident). - add(RESERVED_WORDS, :reserved). - add(PREDEFINED_CONSTANTS, :pre_constant) - - METHOD_NAME = / #{IDENT} [?!]? /xo - METHOD_NAME_EX = / - #{METHOD_NAME} # common methods: split, foo=, empty?, gsub! - | \*\*? # multiplication and power - | [-+~]@? # plus, minus - | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system` - | \[\]=? # array getter and setter - | <=?>? | >=? # comparison, rocket operator - | << | >> # append or shift left, shift right - | ===? # simple equality and case equality - /ox - GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox - - DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox - SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox - STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox - SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox - REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox - - DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error - OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ - HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ - BINARY = /0b[01]+(?:_[01]+)*/ - - EXPONENT = / [eE] [+-]? #{DECIMAL} /ox - FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) / - INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/ - - def reset - super - @regexp_allowed = false - end - - def next_token - return if @scanner.eos? - - kind = :error - if @scanner.scan(/\s+/) # in every state - kind = :space - @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting - - elsif @state == :def_expected - if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox) - kind = :method - @state = :initial - else - @scanner.scan(/./) - kind = :error - end - @state = :initial - - elsif @state == :module_expected - if @scanner.scan(/<;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x) - kind = :operator - @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/ - elsif @scanner.scan(FLOAT) - kind = :float - elsif @scanner.scan(INTEGER) - kind = :integer - elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox) - kind = :global_variable - else - @scanner.scan(/./m) - end - end - - token = Token.new @scanner.matched, kind - - if kind == :regexp - token.text << @scanner.scan(/[eimnosux]*/) - end - - @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting - - token - end - end - - ScannerList.register RubyScanner, 'ruby' - -end - -module CodeRay - require 'scanner' - - class Highlighter - - def initialize lang - @scanner = Scanner[lang].new - end - - def highlight code - @scanner.feed code - @scanner.all_tokens.map { |t| t.inspect }.join "\n" - end - - end - - class HTMLHighlighter < Highlighter - - ClassOfKind = { - :attribute_name => 'an', - :attribute_name_fat => 'af', - :attribute_value => 'av', - :attribute_value_fat => 'aw', - :bin => 'bi', - :char => 'ch', - :class => 'cl', - :class_variable => 'cv', - :color => 'cr', - :comment => 'c', - :constant => 'co', - :definition => 'df', - :directive => 'di', - :doc => 'do', - :doc_string => 'ds', - :exception => 'ex', - :error => 'er', - :float => 'fl', - :function => 'fu', - :global_variable => 'gv', - :hex => 'hx', - :include => 'ic', - :instance_variable => 'iv', - :integer => 'i', - :interpreted => 'in', - :label => 'la', - :local_variable => 'lv', - :oct => 'oc', - :operator_name => 'on', - :pre_constant => 'pc', - :pre_type => 'pt', - :predefined => 'pd', - :preprocessor => 'pp', - :regexp => 'rx', - :reserved => 'r', - :shell => 'sh', - :string => 's', - :symbol => 'sy', - :tag => 'ta', - :tag_fat => 'tf', - :tag_special => 'ts', - :type => 'ty', - :variable => 'v', - :xml_text => 'xt', - - :ident => :NO_HIGHLIGHT, - :operator => :NO_HIGHLIGHT, - :space => :NO_HIGHLIGHT, - } - ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function] - ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' - - def initialize lang, options = {} - super lang - - @HTML_TAB = ' ' * options.fetch(:tabs2space, 8) - case level = options.fetch(:level, 'xhtml') - when 'html' - @HTML_BR = "
\n" - when 'xhtml' - @HTML_BR = "
\n" - else - raise "Unknown HTML level: #{level}" - end - end - - def highlight code - @scanner.feed code - - out = '' - while t = @scanner.next_token - warn t.inspect if t.text.nil? - out << to_html(t) - end - TEMPLATE =~ /<%CONTENT%>/ - $` + out + $' - end - - private - def to_html token - css_class = ClassOfKind[token.kind] - if defined? ::DEBUG and not ClassOfKind.has_key? token.kind - warn "no token class found for :#{token.kind}" - end - - text = text_to_html token.text - if css_class == :NO_HIGHLIGHT - text - else - "#{text}" - end - end - - def text_to_html text - return '' if text.empty? - text = text.dup # important - if text.index(/["><&]/) - text.gsub!('&', '&') - text.gsub!('"', '"') - text.gsub!('>', '>') - text.gsub!('<', '<') - end - if text.index(/\s/) - text.gsub!("\n", @HTML_BR) - text.gsub!("\t", @HTML_TAB) - text.gsub!(/^ /, ' ') - text.gsub!(' ', '  ') - end - text - end - - TEMPLATE = <<-'TEMPLATE' - - - - - - -RubyBB BBCode - - - -
-<%CONTENT%> -
-
-Valid HTML 4.01! -Valid CSS! -
- - - TEMPLATE - - end - -end diff --git a/test/samples/html2.expected b/test/samples/html2.expected deleted file mode 100644 index c8ae56a7..00000000 --- a/test/samples/html2.expected +++ /dev/null @@ -1,185 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-
require 'coderay'
-
-# scan this file
-tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-
-# output it with two styles of line numbers
-out = tokens.div(:line_numbers => :table)
-out << '<hr />'
-out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-
-puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- 
 8 require 'coderay'
- 9 
-10 # scan this file
-11 tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-12 
-13 # output it with two styles of line numbers
-14 out = tokens.div(:line_numbers => :table)
-15 out << '<hr />'
-16 out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-17 
-18 puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- - - diff --git a/test/samples/html2.rb b/test/samples/html2.rb deleted file mode 100644 index 618d168d..00000000 --- a/test/samples/html2.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'coderay' - -# scan this file -tokens = CodeRay.scan(File.read($0) * 1, :ruby) - -# output it with two styles of line numbers -out = tokens.div(:line_numbers => :table) -out << '
' -out << tokens.div(:line_numbers => :inline, :line_number_start => 8) - -puts out.page(:title => 'CodeRay HTML Encoder Example') diff --git a/test/samples/html_list.expected b/test/samples/html_list.expected deleted file mode 100644 index a4092c8d..00000000 --- a/test/samples/html_list.expected +++ /dev/null @@ -1,160 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - -
- 
-1 $: << '..'
- 0 require 'coderay'
- 1 
- 2 tokens = CodeRay.scan File.read(__FILE__), :ruby
- 3 html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)
- 4 
- 5 puts html.page(:title => 'CodeRay HTML Encoder Example')
- 6 
- 7 commment = <<_
- 8 This code must be > 10 lines
- 9 because I want to test the correct adjustment of the line numbers.
-10 _
-
-
- - - diff --git a/test/samples/html_list.rb b/test/samples/html_list.rb deleted file mode 100644 index fdfa5123..00000000 --- a/test/samples/html_list.rb +++ /dev/null @@ -1,12 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan File.read(__FILE__), :ruby -html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1) - -puts html.page(:title => 'CodeRay HTML Encoder Example') - -commment = <<_ -This code must be > 10 lines -because I want to test the correct adjustment of the line numbers. -_ diff --git a/test/samples/load_encoder.expected b/test/samples/load_encoder.expected deleted file mode 100644 index 1cff356d..00000000 --- a/test/samples/load_encoder.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::YAML is not defined; you must load it first. -Now it is loaded: CodeRay::Encoders::YAML -See? -Require is also possible: CodeRay::Encoders::Tokens -See? -Now load some mapped encoders: stats and plain. -Require all Encoders: -[[:count, CodeRay::Encoders::Count], [:debug, CodeRay::Encoders::Debug], [:div, CodeRay::Encoders::Div], [:html, CodeRay::Encoders::HTML], [:null, CodeRay::Encoders::Null], [:page, CodeRay::Encoders::Page], [:plain, :text], [:span, CodeRay::Encoders::Span], [:statistic, CodeRay::Encoders::Statistic], [:stats, CodeRay::Encoders::Statistic], [:text, CodeRay::Encoders::Text], [:tokens, CodeRay::Encoders::Tokens], [:xml, CodeRay::Encoders::XML], [:yaml, CodeRay::Encoders::YAML]] diff --git a/test/samples/load_encoder.rb b/test/samples/load_encoder.rb deleted file mode 100644 index 9594bfa1..00000000 --- a/test/samples/load_encoder.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Encoders::YAML -rescue - puts 'CodeRay::Encoders::YAML is not defined; you must load it first.' -end - -yaml_encoder = CodeRay::Encoders[:yaml] -print 'Now it is loaded: ' -p yaml_encoder -puts 'See?' - -tokens_encoder = CodeRay.require_plugin 'CodeRay::Encoders/tokens' -print 'Require is also possible: ' -p tokens_encoder -puts 'See?' - -puts 'Now load some mapped encoders: stats and plain.' -CodeRay.require_plugin 'CodeRay::Encoders/stats' -CodeRay.require_plugin 'CodeRay::Encoders/plain' - -puts 'Require all Encoders:' -CodeRay::Encoders.load_all -p CodeRay::Encoders.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/load_scanner.expected b/test/samples/load_scanner.expected deleted file mode 100644 index a2d200d7..00000000 --- a/test/samples/load_scanner.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::Ruby is not defined; you must load it first. -Now it is loaded: CodeRay::Scanners::Ruby -See? -Require is also possible: CodeRay::Scanners::C -See? -Now load some mapped scanners: cpp and plain. -Require all Scanners: -[[nil, :plain], [:c, CodeRay::Scanners::C], [:cpp, :c], [:delphi, CodeRay::Scanners::Delphi], [:html, CodeRay::Scanners::HTML], [:irb, :ruby], [:nitro, :nitro_xhtml], [:nitro_xhtml, CodeRay::Scanners::NitroXHTML], [:pascal, :delphi], [:plain, CodeRay::Scanners::Plaintext], [:plaintext, CodeRay::Scanners::Plaintext], [:rhtml, CodeRay::Scanners::RHTML], [:ruby, CodeRay::Scanners::Ruby], [:xhtml, :nitro_xhtml], [:xml, :html]] diff --git a/test/samples/load_scanner.rb b/test/samples/load_scanner.rb deleted file mode 100644 index 23be8a29..00000000 --- a/test/samples/load_scanner.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Scanners::Ruby -rescue - puts 'CodeRay::Encoders::Ruby is not defined; you must load it first.' -end - -ruby_scanner = CodeRay::Scanners[:ruby] -print 'Now it is loaded: ' -p ruby_scanner -puts 'See?' - -c_scanner = CodeRay.require_plugin 'CodeRay::Scanners/c' -print 'Require is also possible: ' -p c_scanner -puts 'See?' - -puts 'Now load some mapped scanners: cpp and plain.' -CodeRay.require_plugin 'CodeRay::Scanners/cpp' -CodeRay.require_plugin 'CodeRay::Scanners/plain' - -puts 'Require all Scanners:' -CodeRay::Scanners.load_all -p CodeRay::Scanners.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/more.expected b/test/samples/more.expected deleted file mode 100644 index 196904d8..00000000 --- a/test/samples/more.expected +++ /dev/null @@ -1,2 +0,0 @@ -Input: 4983B, Output: 23484B -Take a look with your browser. diff --git a/test/samples/more.rb b/test/samples/more.rb deleted file mode 100644 index 0db7ba47..00000000 --- a/test/samples/more.rb +++ /dev/null @@ -1,205 +0,0 @@ -require 'coderay' - -c, ruby = DATA.read.split(/^---$/) -DATA.rewind -me = DATA.read[/.*^__END__$/m] -$input = c + ruby + me - -require 'benchmark' -time = Benchmark.realtime do - - # here CodeRay comes to play - hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div) - c = hl.highlight c, :c - ruby = hl.highlight ruby, :ruby - me = hl.highlight me, :ruby - - body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code| - "

#{title}

\n#{code}" - end.join - body = hl.class::Output.new(body, hl.css, :div).page! - - # CodeRay also provides a simple page generator - $output = body #hl.class.wrap_in_page body -end - -File.open('test.html', 'w') do |f| - f.write $output -end -puts 'Input: %dB, Output: %dB' % [$input.size, $output.size] -#puts 'Created "test.html" in %0.3f seconds (%d KB/s).' % [time, $input.size / 1024.0 / time] -puts 'Take a look with your browser.' - -__END__ -/********************************************************************** - - version.c - - - $Author: nobu $ - $Date: 2004/03/25 12:01:40 $ - created at: Thu Sep 30 20:08:01 JST 1993 - - Copyright (C) 1993-2003 Yukihiro Matsumoto - -**********************************************************************/ - -#include "ruby.h" -#include "version.h" -#include - -const char ruby_version[] = RUBY_VERSION; -const char ruby_release_date[] = RUBY_RELEASE_DATE; -const char ruby_platform[] = RUBY_PLATFORM; - -void -Init_version() -{ - VALUE v = rb_obj_freeze(rb_str_new2(ruby_version)); - VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date)); - VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform)); - - rb_define_global_const("RUBY_VERSION", v); - rb_define_global_const("RUBY_RELEASE_DATE", d); - rb_define_global_const("RUBY_PLATFORM", p); -} - -void -ruby_show_version() -{ - printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM); -} - -void -ruby_show_copyright() -{ - printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR); - exit(0); -} ---- -# -# = ostruct.rb: OpenStruct implementation -# -# Author:: Yukihiro Matsumoto -# Documentation:: Gavin Sinclair -# -# OpenStruct allows the creation of data objects with arbitrary attributes. -# See OpenStruct for an example. -# - -# -# OpenStruct allows you to create data objects and set arbitrary attributes. -# For example: -# -# require 'ostruct' -# -# record = OpenStruct.new -# record.name = "John Smith" -# record.age = 70 -# record.pension = 300 -# -# puts record.name # -> "John Smith" -# puts record.address # -> nil -# -# It is like a hash with a different way to access the data. In fact, it is -# implemented with a hash, and you can initialize it with one. -# -# hash = { "country" => "Australia", :population => 20_000_000 } -# data = OpenStruct.new(hash) -# -# p data # -> -# -class OpenStruct - # - # Create a new OpenStruct object. The optional +hash+, if given, will - # generate attributes and values. For example. - # - # require 'ostruct' - # hash = { "country" => "Australia", :population => 20_000_000 } - # data = OpenStruct.new(hash) - # - # p data # -> - # - # By default, the resulting OpenStruct object will have no attributes. - # - def initialize(hash=nil) - @table = {} - if hash - for k,v in hash - @table[k.to_sym] = v - new_ostruct_member(k) - end - end - end - - # Duplicate an OpenStruct object members. - def initialize_copy(orig) - super - @table = @table.dup - end - - def marshal_dump - @table - end - def marshal_load(x) - @table = x - @table.each_key{|key| new_ostruct_member(key)} - end - - def new_ostruct_member(name) - unless self.respond_to?(name) - self.instance_eval %{ - def #{name}; @table[:#{name}]; end - def #{name}=(x); @table[:#{name}] = x; end - } - end - end - - def method_missing(mid, *args) # :nodoc: - mname = mid.id2name - len = args.length - if mname =~ /=$/ - if len != 1 - raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1) - end - if self.frozen? - raise TypeError, "can't modify frozen #{self.class}", caller(1) - end - mname.chop! - @table[mname.intern] = args[0] - self.new_ostruct_member(mname) - elsif len == 0 - @table[mid] - else - raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1) - end - end - - # - # Remove the named field from the object. - # - def delete_field(name) - @table.delete name.to_sym - end - - # - # Returns a string containing a detailed summary of the keys and values. - # - def inspect - str = "<#{self.class}" - for k,v in @table - str << " #{k}=#{v.inspect}" - end - str << ">" - end - - attr_reader :table # :nodoc: - protected :table - - # - # Compare this object and +other+ for equality. - # - def ==(other) - return false unless(other.kind_of?(OpenStruct)) - return @table == other.table - end -end diff --git a/test/samples/scanner.expected b/test/samples/scanner.expected deleted file mode 100644 index 5015168f..00000000 --- a/test/samples/scanner.expected +++ /dev/null @@ -1,16 +0,0 @@ -C Code: if (*p == '{') nest++; - -> print only operators: -(*==)++; ------------------------------- - -Ruby Code: ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee) - -> has a string? -false - -> number of regexps? -2 - -> has a string? -"ruby_code" (ident), "(" (operator), ":can" (symbol), "," (operator), " " (space), "BE" (constant), "," (operator), " " (space), "%r[" (delimiter), "q" (content), "[" (nesting_delimiter), "ui" (content), "]" (nesting_delimiter), "te " (content), "#{" (inline_delimiter), " " (space), "/" (delimiter), "comple" (content), "/" (delimiter), "x" (modifier), " " (space), "}" (inline_delimiter), "," (content), "]" (delimiter), " " (space), "=" (operator), ">" (operator), " " (space), "$-s" (global_variable), "," (operator), " " (space), "&" (operator), "?\xee" (integer), ")" (operator) diff --git a/test/samples/scanner.rb b/test/samples/scanner.rb deleted file mode 100644 index 6a0245ea..00000000 --- a/test/samples/scanner.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'coderay' - -c_code = "if (*p == '{') nest++;" -puts 'C Code: ' + c_code -puts - -c_scanner = CodeRay::Scanners[:c].new c_code - -puts '> print only operators:' -for text, kind in c_scanner - print text if kind == :operator -end -puts -puts '-' * 30 -puts - -ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)! -puts 'Ruby Code: ' + ruby_code -puts - -ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code - -puts '> has a string?' -puts ruby_scanner. - any? { |text, kind| kind == :string } -puts - -puts '> number of regexps?' -puts ruby_scanner. - select { |token| token == [:open, :regexp] }.size -puts - -puts '> has a string?' -puts ruby_scanner. - reject { |text, kind| not text.is_a? String }. - map { |text, kind| %("#{text}" (#{kind})) }.join(', ') diff --git a/test/samples/server.rb b/test/samples/server.rb deleted file mode 100644 index ccdff324..00000000 --- a/test/samples/server.rb +++ /dev/null @@ -1,110 +0,0 @@ -# CodeRay dynamic highlighter - -unless ARGV.grep(/-[hv]|--(help|version)/).empty? - puts <<-USAGE -CodeRay Server 0.5 -$Id: demo_server.rb 113 2006-03-15 23:24:37Z murphy $ - -Usage: - 1) Start this and your browser. - 2) Go to http://localhost:2468/? - and you should get the highlighted version. - -Parameters: - -d Debug mode; reload CodeRay engine for every file. - (prepare for MANY "already initialized" and "method redefined" - messages - ingore it.) - - ... More to come. - USAGE - exit -end - -require 'webrick' -require 'pathname' - -class << File - alias dir? directory? -end - -require 'erb' -include ERB::Util -def url_decode s - s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' } -end - -class String - def to_link name = File.basename(self) - "#{name}" - end -end - -require 'coderay' -class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet - - STYLE = 'style="font-family: sans-serif; color: navy;"' - BANNER = '

Highlighted by CodeRay

' - - def do_GET req, res - q = req.query_string || '' - args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) } - path = args.fetch 'path', '.' - - backlinks = '

current path: %s
' % html_escape(path) + - (Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' + - '.'.to_link('current') + '

' - - res.body = - if File.dir? path - path = Pathname.new(path).cleanpath.to_s - dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p } - - page = "" - page << backlinks - - page << '
' - page << "
Directories
\n" + dirs.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
Files
\n" + files.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
\n" - page << "#{BANNER}" - - elsif File.exist? path - if $DEBUG - $".delete_if { |f| f =~ /coderay/ } - require 'coderay' - end - div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div, :hint => :info - div.replace <<-DIV -
- #{backlinks} -#{div} -
- #{BANNER} - DIV - div.page - end - - res['Content-Type'] = 'text/html' - end -end - -# This port is taken by "qip_msgd" - I don't know that. Do you? -module CodeRay - PORT = 0xC0DE / 20 -end - -server = WEBrick::HTTPServer.new :Port => CodeRay::PORT - -server.mount '/', CodeRayServlet - -server.mount_proc '/version' do |req, res| - res.body = 'CodeRay::Version = ' + CodeRay::Version - res['Content-Type'] = "text/plain" -end - -trap("INT") { server.shutdown } -server.start diff --git a/test/samples/simple.expected b/test/samples/simple.expected deleted file mode 100644 index b3d78753..00000000 --- a/test/samples/simple.expected +++ /dev/null @@ -1 +0,0 @@ -puts 'Hello, world!' diff --git a/test/samples/simple.rb b/test/samples/simple.rb deleted file mode 100644 index a3129b01..00000000 --- a/test/samples/simple.rb +++ /dev/null @@ -1,10 +0,0 @@ - -# Load CodeRay -# If this doesn't work, try ruby -rubygems. -require 'coderay' - -# Generate HTML page for Ruby code. -page = CodeRay.scan("puts 'Hello, world!'", :ruby).span - -# Print it -puts page diff --git a/test/samples/stream.rb b/test/samples/stream.rb deleted file mode 100644 index 7ed8a22b..00000000 --- a/test/samples/stream.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -code = File.read($0) * 500 -puts "Size of code: %d KB" % [code.size / 1024] - -puts "Use your system's memory tracker to see how much RAM this takes." -print 'Press some key to continue...'; gets - -require 'benchmark' -e = CodeRay.encoder(:div) -for do_stream in [true, false] - puts "Scanning and encoding in %s mode, please wait..." % - [do_stream ? 'streaming' : 'normal'] - output = '' - time = Benchmark.realtime do - if do_stream - output = e.encode_stream(code, :ruby) - else - output = e.encode_tokens(t = CodeRay.scan(code, :ruby)) - end - end - puts 'Finished after %4.2f seconds.' % time - puts "Size of output: %d KB" % [output.size / 1024] - print 'Press some key to continue...'; gets -end diff --git a/test/samples/stream2.expected b/test/samples/stream2.expected deleted file mode 100644 index 83aee987..00000000 --- a/test/samples/stream2.expected +++ /dev/null @@ -1,2 +0,0 @@ -kind: regexp, text size: 5. -kind: space, text size: 1. diff --git a/test/samples/stream2.rb b/test/samples/stream2.rb deleted file mode 100644 index d43cc9ad..00000000 --- a/test/samples/stream2.rb +++ /dev/null @@ -1,8 +0,0 @@ -require 'coderay' - -token_stream = CodeRay::TokenStream.new do |kind, text| - puts 'kind: %s, text size: %d.' % [kind, text.size] -end - -token_stream << [:regexp, '/\d+/'] << [:space, "\n"] -#-> kind: rexpexp, text size: 5. diff --git a/test/samples/suite.rb b/test/samples/suite.rb deleted file mode 100644 index cfe53c0f..00000000 --- a/test/samples/suite.rb +++ /dev/null @@ -1,86 +0,0 @@ -mydir = File.dirname(__FILE__) -$:.unshift mydir + '/../../lib/' - -$VERBOSE = true - -require 'test/unit' -include Test::Unit - -class CodeRaySuite < TestCase - - def self.dir &block - @dir ||= File.dirname(__FILE__) - if block - Dir.chdir @dir, &block - end - @dir - end - - def dir &block - self.class.dir(&block) - end - - def test_ALL - dir do - for input in Dir["*.rb"] - %w(server.rb stream.rb suite.rb) - next if input[/^load_/] - puts "[ testing #{input}... ]" - name = File.basename(input, ".rb") - output = name + '.expected' - code = File.open(input, 'rb') { |f| break f.read } - - result = `ruby -wI../../lib #{input}` - - diff = output.sub '.expected', '.diff' - File.delete diff if File.exist? diff - computed = output.sub '.expected', '.actual' - if File.exist? output - expected = File.read output - ok = expected == result - unless ok - File.open(computed, 'w') { |f| f.write result } - `diff #{output} #{computed} > #{diff}` - puts "Test failed; output written to #{diff}." - end - assert(ok, "Output error: #{computed} != #{output}") - else - File.open(output, 'w') do |f| f.write result end - puts "New test: #{output}" - end - - end - end - end - -end - -require 'test/unit/testsuite' -$suite = TestSuite.new 'CodeRay Demos Test' -$suite << CodeRaySuite.suite - -def load_suite name - begin - require name + '/suite.rb' - rescue LoadError - $stderr.puts <<-ERR - -!! Folder #{File.split(__FILE__).first + '/' + name} not found - - ERR - false - end -end - -if subsuite = ARGV.find { |a| break $1 if a[/^([^-].*)/] } - load_suite(subsuite) or exit -else - Dir[mydir + '/*/'].each { |suite| load_suite suite } -end - -if ARGV.include? '-f' - require 'test/unit/ui/fox/testrunner' - UI::Fox::TestRunner.run $suite -else - require 'test/unit/ui/console/testrunner' - UI::Console::TestRunner.run $suite -end diff --git a/test/samples/tokens.expected b/test/samples/tokens.expected deleted file mode 100644 index 747904e5..00000000 --- a/test/samples/tokens.expected +++ /dev/null @@ -1 +0,0 @@ -[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]] diff --git a/test/samples/tokens.rb b/test/samples/tokens.rb deleted file mode 100644 index 91b8abbf..00000000 --- a/test/samples/tokens.rb +++ /dev/null @@ -1,3 +0,0 @@ -require 'coderay' - -p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby) From b09e97b08c3c073e79159ff09f6a7e0779fcfd2e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 22 Oct 2013 01:11:31 +0200 Subject: [PATCH 104/236] use autoload again --- lib/coderay.rb | 18 +-- lib/coderay/encoders.rb | 18 +++ lib/coderay/{ => encoders}/encoder.rb | 11 -- lib/coderay/encoders/html.rb | 2 +- lib/coderay/helpers/plugin.rb | 219 ------------------------- lib/coderay/helpers/plugin_host.rb | 223 ++++++++++++++++++++++++++ lib/coderay/scanners.rb | 23 +++ lib/coderay/scanners/java.rb | 2 +- lib/coderay/{ => scanners}/scanner.rb | 18 --- lib/coderay/styles.rb | 11 ++ lib/coderay/{ => styles}/style.rb | 7 +- lib/coderay/tokens.rb | 2 +- 12 files changed, 288 insertions(+), 266 deletions(-) create mode 100644 lib/coderay/encoders.rb rename lib/coderay/{ => encoders}/encoder.rb (93%) create mode 100644 lib/coderay/helpers/plugin_host.rb create mode 100644 lib/coderay/scanners.rb rename lib/coderay/{ => scanners}/scanner.rb (94%) create mode 100644 lib/coderay/styles.rb rename lib/coderay/{ => styles}/style.rb (64%) diff --git a/lib/coderay.rb b/lib/coderay.rb index f759ed63..c3de20b5 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -134,7 +134,7 @@ def self.coderay_path *path File.join CODERAY_PATH, *path end - require 'coderay/version' + autoload :VERSION, 'coderay/version' # helpers autoload :FileType, coderay_path('helpers', 'file_type') @@ -145,13 +145,13 @@ def self.coderay_path *path autoload :TokenKinds, coderay_path('token_kinds') # Plugin system - autoload :PluginHost, coderay_path('helpers', 'plugin') + autoload :PluginHost, coderay_path('helpers', 'plugin_host') autoload :Plugin, coderay_path('helpers', 'plugin') # Plugins - autoload :Scanners, coderay_path('scanner') - autoload :Encoders, coderay_path('encoder') - autoload :Styles, coderay_path('style') + autoload :Scanners, coderay_path('scanners') + autoload :Encoders, coderay_path('encoders') + autoload :Styles, coderay_path('styles') # convenience access and reusable Encoder/Scanner pair autoload :Duo, coderay_path('duo') @@ -166,7 +166,7 @@ class << self # # See also demo/demo_simple. def scan code, lang, options = {}, &block - TokensProxy.new code, lang, options, block + CodeRay::TokensProxy.new code, lang, options, block end # Scans +filename+ (a path to a code file) with the Scanner for +lang+. @@ -181,7 +181,7 @@ def scan code, lang, options = {}, &block # require 'coderay' # page = CodeRay.scan_file('some_c_code.c').html def scan_file filename, lang = :auto, options = {}, &block - lang = FileType.fetch filename, :text, true if lang == :auto + lang = CodeRay::FileType.fetch filename, :text, true if lang == :auto code = File.read filename scan code, lang, options, &block end @@ -258,7 +258,7 @@ def highlight_file filename, options = { :css => :class }, format = :div # ] # #-> 2 out of 4 tokens have the kind :integer. def encoder format, options = {} - Encoders[format].new options + CodeRay::Encoders[format].new options end # Finds the Scanner class for +lang+ and creates an instance, passing @@ -266,7 +266,7 @@ def encoder format, options = {} # # See Scanner.new. def scanner lang, options = {}, &block - Scanners[lang].new '', options, &block + CodeRay::Scanners[lang].new '', options, &block end # Extract the options for the scanner from the +options+ hash. diff --git a/lib/coderay/encoders.rb b/lib/coderay/encoders.rb new file mode 100644 index 00000000..6599186e --- /dev/null +++ b/lib/coderay/encoders.rb @@ -0,0 +1,18 @@ +module CodeRay + + # This module holds the Encoder class and its subclasses. + # For example, the HTML encoder is named CodeRay::Encoders::HTML + # can be found in coderay/encoders/html. + # + # Encoders also provides methods and constants for the register + # mechanism and the [] method that returns the Encoder class + # belonging to the given format. + module Encoders + + extend PluginHost + plugin_path File.dirname(__FILE__), 'encoders' + + autoload :Encoder, CodeRay.coderay_path('encoders', 'encoder') + + end +end diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoders/encoder.rb similarity index 93% rename from lib/coderay/encoder.rb rename to lib/coderay/encoders/encoder.rb index d2d6c7e6..fa5695d6 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoders/encoder.rb @@ -1,17 +1,6 @@ module CodeRay - - # This module holds the Encoder class and its subclasses. - # For example, the HTML encoder is named CodeRay::Encoders::HTML - # can be found in coderay/encoders/html. - # - # Encoders also provides methods and constants for the register - # mechanism and the [] method that returns the Encoder class - # belonging to the given format. module Encoders - extend PluginHost - plugin_path File.dirname(__FILE__), 'encoders' - # = Encoder # # The Encoder base class. Together with Scanner and diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index d2ebb5af..093df08b 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -289,7 +289,7 @@ def make_span_for_kinds method, hint Hash.new do |h, kinds| begin css_class = css_class_for_kinds(kinds) - title = HTML.token_path_to_hint hint, kinds if hint + title = Html.token_path_to_hint hint, kinds if hint if css_class || title if method == :style diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index 9a724fff..45679436 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -1,224 +1,5 @@ module CodeRay - # = PluginHost - # - # A simple subclass/subfolder plugin system. - # - # Example: - # class Generators - # extend PluginHost - # plugin_path 'app/generators' - # end - # - # class Generator - # extend Plugin - # PLUGIN_HOST = Generators - # end - # - # class FancyGenerator < Generator - # register_for :fancy - # end - # - # Generators[:fancy] #-> FancyGenerator - # # or - # CodeRay.require_plugin 'Generators/fancy' - # # or - # Generators::Fancy - module PluginHost - - # Raised if Encoders::[] fails because: - # * a file could not be found - # * the requested Plugin is not registered - PluginNotFound = Class.new LoadError - HostNotFound = Class.new LoadError - - PLUGIN_HOSTS = [] - PLUGIN_HOSTS_BY_ID = {} # dummy hash - - # Loads all plugins using list and load. - def load_all - for plugin in list - load plugin - end - end - - # Returns the Plugin for +id+. - # - # Example: - # yaml_plugin = MyPluginHost[:yaml] - def [] id, *args, &blk - plugin = validate_id(id) - begin - plugin = plugin_hash.[](plugin, *args, &blk) - end while plugin.is_a? String - plugin - end - - alias load [] - - # Tries to +load+ the missing plugin by translating +const+ to the - # underscore form (eg. LinesOfCode becomes lines_of_code). - def const_missing const - id = const.to_s. - gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). - gsub(/([a-z\d])([A-Z])/,'\1_\2'). - downcase - load id - end - - class << self - - # Adds the module/class to the PLUGIN_HOSTS list. - def extended mod - PLUGIN_HOSTS << mod - end - - end - - # The path where the plugins can be found. - def plugin_path *args - unless args.empty? - @plugin_path = File.expand_path File.join(*args) - end - @plugin_path ||= '' - end - - # Map a plugin_id to another. - # - # Usage: Put this in a file plugin_path/_map.rb. - # - # class MyColorHost < PluginHost - # map :navy => :dark_blue, - # :maroon => :brown, - # :luna => :moon - # end - def map hash - for from, to in hash - from = validate_id from - to = validate_id to - plugin_hash[from] = to unless plugin_hash.has_key? from - end - end - - # Define the default plugin to use when no plugin is found - # for a given id, or return the default plugin. - # - # See also map. - # - # class MyColorHost < PluginHost - # map :navy => :dark_blue - # default :gray - # end - # - # MyColorHost.default # loads and returns the Gray plugin - def default id = nil - if id - id = validate_id id - raise "The default plugin can't be named \"default\"." if id == :default - plugin_hash[:default] = id - else - load :default - end - end - - # Every plugin must register itself for +id+ by calling register_for, - # which calls this method. - # - # See Plugin#register_for. - def register plugin, id - plugin_hash[validate_id(id)] = plugin - end - - # A Hash of plugion_id => Plugin pairs. - def plugin_hash - @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } - end - - # Returns an array of all .rb files in the plugin path. - # - # The extension .rb is not included. - def list - Dir[path_to('*')].select do |file| - File.basename(file)[/^(?!_)\w+\.rb$/] - end.map do |file| - File.basename(file, '.rb').to_sym - end - end - - # Returns an array of all Plugins. - # - # Note: This loads all plugins using load_all. - def all_plugins - load_all - plugin_hash.values.grep(Class) - end - - # Loads the map file (see map). - # - # This is done automatically when plugin_path is called. - def load_plugin_map - mapfile = path_to '_map' - if File.exist? mapfile - require mapfile - true - else - false - end - end - - protected - - # Return a plugin hash that automatically loads plugins. - def make_plugin_hash - Hash.new do |h, plugin_id| - id = validate_id(plugin_id) - path = path_to id - begin - require path - rescue LoadError => boom - if h.has_key?(:default) - h[:default] - else - raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] - end - else - # Plugin should have registered by now - if h.has_key? id - h[id] - else - raise PluginNotFound, "No #{self.name} plugin for #{id.inspect} found in #{path}." - end - end - end - end - - # Returns the expected path to the plugin file for the given id. - def path_to plugin_id - File.join plugin_path, "#{plugin_id}.rb" - end - - # Converts +id+ to a valid plugin ID String, or returns +nil+. - # - # Raises +ArgumentError+ for all other objects, or if the - # given String includes non-alphanumeric characters (\W). - def validate_id id - case id - when Symbol - id.to_s - when String - if id[/\w+/] == id - id.downcase - else - raise ArgumentError, "Invalid id given: #{id}" - end - else - raise ArgumentError, "Symbol or String expected, but #{id.class} given." - end - end - - end - - # = Plugin # # Plugins have to include this module. diff --git a/lib/coderay/helpers/plugin_host.rb b/lib/coderay/helpers/plugin_host.rb new file mode 100644 index 00000000..b0b3aef7 --- /dev/null +++ b/lib/coderay/helpers/plugin_host.rb @@ -0,0 +1,223 @@ +module CodeRay + + # = PluginHost + # + # A simple subclass/subfolder plugin system. + # + # Example: + # class Generators + # extend PluginHost + # plugin_path 'app/generators' + # end + # + # class Generator + # extend Plugin + # PLUGIN_HOST = Generators + # end + # + # class FancyGenerator < Generator + # register_for :fancy + # end + # + # Generators[:fancy] #-> FancyGenerator + # # or + # CodeRay.require_plugin 'Generators/fancy' + # # or + # Generators::Fancy + module PluginHost + + # Raised if Encoders::[] fails because: + # * a file could not be found + # * the requested Plugin is not registered + PluginNotFound = Class.new LoadError + HostNotFound = Class.new LoadError + + PLUGIN_HOSTS = [] + PLUGIN_HOSTS_BY_ID = {} # dummy hash + + # Loads all plugins using list and load. + def load_all + for plugin in list + load plugin + end + end + + # Returns the Plugin for +id+. + # + # Example: + # yaml_plugin = MyPluginHost[:yaml] + def [] id, *args, &blk + # const = id.to_s.titleize + # const_get const + plugin = validate_id(id) + begin + plugin = plugin_hash.[](plugin, *args, &blk) + end while plugin.is_a? String + plugin + end + + alias load [] + + # Tries to +load+ the missing plugin by translating +const+ to the + # underscore form (eg. LinesOfCode becomes lines_of_code). + def const_missing const + id = const.to_s. + gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). + gsub(/([a-z\d])([A-Z])/,'\1_\2'). + downcase + load id + end + + class << self + + # Adds the module/class to the PLUGIN_HOSTS list. + def extended mod + PLUGIN_HOSTS << mod + end + + end + + # The path where the plugins can be found. + def plugin_path *args + unless args.empty? + @plugin_path = File.expand_path File.join(*args) + end + @plugin_path ||= '' + end + + # Map a plugin_id to another. + # + # Usage: Put this in a file plugin_path/_map.rb. + # + # class MyColorHost < PluginHost + # map :navy => :dark_blue, + # :maroon => :brown, + # :luna => :moon + # end + def map hash + for from, to in hash + from = validate_id from + to = validate_id to + plugin_hash[from] = to unless plugin_hash.has_key? from + end + end + + # Define the default plugin to use when no plugin is found + # for a given id, or return the default plugin. + # + # See also map. + # + # class MyColorHost < PluginHost + # map :navy => :dark_blue + # default :gray + # end + # + # MyColorHost.default # loads and returns the Gray plugin + def default id = nil + if id + id = validate_id id + raise "The default plugin can't be named \"default\"." if id == :default + plugin_hash[:default] = id + else + load :default + end + end + + # Every plugin must register itself for +id+ by calling register_for, + # which calls this method. + # + # See Plugin#register_for. + def register plugin, id + plugin_hash[validate_id(id)] = plugin + end + + # A Hash of plugion_id => Plugin pairs. + def plugin_hash + @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } + end + + # Returns an array of all .rb files in the plugin path. + # + # The extension .rb is not included. + def list + Dir[path_to('*')].select do |file| + File.basename(file)[/^(?!_)\w+\.rb$/] + end.map do |file| + File.basename(file, '.rb').to_sym + end + end + + # Returns an array of all Plugins. + # + # Note: This loads all plugins using load_all. + def all_plugins + load_all + plugin_hash.values.grep(Class) + end + + # Loads the map file (see map). + # + # This is done automatically when plugin_path is called. + def load_plugin_map + mapfile = path_to '_map' + if File.exist? mapfile + require mapfile + true + else + false + end + end + + protected + + # Return a plugin hash that automatically loads plugins. + def make_plugin_hash + Hash.new do |h, plugin_id| + id = validate_id(plugin_id) + path = path_to id + begin + require path + rescue LoadError => boom + if h.has_key?(:default) + h[:default] + else + raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] + end + else + # Plugin should have registered by now + if h.has_key? id + h[id] + else + raise PluginNotFound, "No #{self.name} plugin for #{id.inspect} found in #{path}." + end + end + end + end + + # Returns the expected path to the plugin file for the given id. + def path_to plugin_id + File.join plugin_path, "#{plugin_id}.rb" + end + + # Converts +id+ to a valid plugin ID String, or returns +nil+. + # + # Raises +ArgumentError+ for all other objects, or if the + # given String includes non-alphanumeric characters (\W). + def validate_id id + case id + when Symbol + id.to_s + when String + if id[/\w+/] == id + id.downcase + else + raise ArgumentError, "Invalid id given: #{id}" + end + else + raise ArgumentError, "Symbol or String expected, but #{id.class} given." + end + end + + end + +end diff --git a/lib/coderay/scanners.rb b/lib/coderay/scanners.rb new file mode 100644 index 00000000..f824f500 --- /dev/null +++ b/lib/coderay/scanners.rb @@ -0,0 +1,23 @@ +require 'strscan' + +module CodeRay + + autoload :WordList, coderay_path('helpers', 'word_list') + + # = Scanners + # + # This module holds the Scanner class and its subclasses. + # For example, the Ruby scanner is named CodeRay::Scanners::Ruby + # can be found in coderay/scanners/ruby. + # + # Scanner also provides methods and constants for the register + # mechanism and the [] method that returns the Scanner class + # belonging to the given lang. + # + # See PluginHost. + module Scanners + extend PluginHost + plugin_path File.dirname(__FILE__), 'scanners' + end + +end diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index b282864a..962154eb 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -36,7 +36,7 @@ class Java < Scanner add(BuiltinTypes::List, :predefined_type). add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception). add(DIRECTIVES, :directive) # :nodoc: - + ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: STRING_CONTENT_PATTERN = { diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanners/scanner.rb similarity index 94% rename from lib/coderay/scanner.rb rename to lib/coderay/scanners/scanner.rb index b3f7e175..efa710d9 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanners/scanner.rb @@ -1,25 +1,7 @@ # encoding: utf-8 -require 'strscan' module CodeRay - - autoload :WordList, coderay_path('helpers', 'word_list') - - # = Scanners - # - # This module holds the Scanner class and its subclasses. - # For example, the Ruby scanner is named CodeRay::Scanners::Ruby - # can be found in coderay/scanners/ruby. - # - # Scanner also provides methods and constants for the register - # mechanism and the [] method that returns the Scanner class - # belonging to the given lang. - # - # See PluginHost. module Scanners - extend PluginHost - plugin_path File.dirname(__FILE__), 'scanners' - # = Scanner # diff --git a/lib/coderay/styles.rb b/lib/coderay/styles.rb new file mode 100644 index 00000000..a7c43e4b --- /dev/null +++ b/lib/coderay/styles.rb @@ -0,0 +1,11 @@ +module CodeRay + + # This module holds the Style class and its subclasses. + # + # See Plugin. + module Styles + extend PluginHost + plugin_path File.dirname(__FILE__), 'styles' + end + +end diff --git a/lib/coderay/style.rb b/lib/coderay/styles/style.rb similarity index 64% rename from lib/coderay/style.rb rename to lib/coderay/styles/style.rb index df4704f4..a3353861 100644 --- a/lib/coderay/style.rb +++ b/lib/coderay/styles/style.rb @@ -1,11 +1,6 @@ module CodeRay - - # This module holds the Style class and its subclasses. - # - # See Plugin. + module Styles - extend PluginHost - plugin_path File.dirname(__FILE__), 'styles' # Base class for styles. # diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index e7bffce2..aeb3b792 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -67,7 +67,7 @@ def to_s def method_missing meth, options = {} encode meth, options rescue PluginHost::PluginNotFound - super + raise end # Split the tokens into parts of the given +sizes+. From d21348f85036273fd8f370a6a850e1c0440c6b02 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 22 Oct 2013 03:00:46 +0200 Subject: [PATCH 105/236] tweaks --- lib/coderay/encoders/html.rb | 2 +- lib/coderay/scanners.rb | 4 ++++ lib/coderay/styles.rb | 4 ++++ lib/coderay/tokens.rb | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 093df08b..d2ebb5af 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -289,7 +289,7 @@ def make_span_for_kinds method, hint Hash.new do |h, kinds| begin css_class = css_class_for_kinds(kinds) - title = Html.token_path_to_hint hint, kinds if hint + title = HTML.token_path_to_hint hint, kinds if hint if css_class || title if method == :style diff --git a/lib/coderay/scanners.rb b/lib/coderay/scanners.rb index f824f500..8d8156ff 100644 --- a/lib/coderay/scanners.rb +++ b/lib/coderay/scanners.rb @@ -16,8 +16,12 @@ module CodeRay # # See PluginHost. module Scanners + extend PluginHost plugin_path File.dirname(__FILE__), 'scanners' + + autoload :Encoder, CodeRay.coderay_path('scanners', 'scanner') + end end diff --git a/lib/coderay/styles.rb b/lib/coderay/styles.rb index a7c43e4b..d8fa8aa7 100644 --- a/lib/coderay/styles.rb +++ b/lib/coderay/styles.rb @@ -4,8 +4,12 @@ module CodeRay # # See Plugin. module Styles + extend PluginHost plugin_path File.dirname(__FILE__), 'styles' + + autoload :Style, CodeRay.coderay_path('styles', 'style') + end end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index aeb3b792..e7bffce2 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -67,7 +67,7 @@ def to_s def method_missing meth, options = {} encode meth, options rescue PluginHost::PluginNotFound - raise + super end # Split the tokens into parts of the given +sizes+. From d91c9c6bd5cad4f57e2229638a23a8ecb8ce12af Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 22 Oct 2013 03:15:44 +0200 Subject: [PATCH 106/236] fix --- lib/coderay/scanners.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners.rb b/lib/coderay/scanners.rb index 8d8156ff..3c7e594d 100644 --- a/lib/coderay/scanners.rb +++ b/lib/coderay/scanners.rb @@ -20,7 +20,7 @@ module Scanners extend PluginHost plugin_path File.dirname(__FILE__), 'scanners' - autoload :Encoder, CodeRay.coderay_path('scanners', 'scanner') + autoload :Scanner, CodeRay.coderay_path('scanners', 'scanner') end From 76ef0f0b928f49b70710a541a11c99956b86e0c3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Tue, 22 Oct 2013 03:16:38 +0200 Subject: [PATCH 107/236] cleanup --- lib/coderay/helpers/plugin_host.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/coderay/helpers/plugin_host.rb b/lib/coderay/helpers/plugin_host.rb index b0b3aef7..e9bc17c1 100644 --- a/lib/coderay/helpers/plugin_host.rb +++ b/lib/coderay/helpers/plugin_host.rb @@ -47,8 +47,6 @@ def load_all # Example: # yaml_plugin = MyPluginHost[:yaml] def [] id, *args, &blk - # const = id.to_s.titleize - # const_get const plugin = validate_id(id) begin plugin = plugin_hash.[](plugin, *args, &blk) From d3197be3f207f8fcf52954d8815a0ea1948d25a4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Feb 2014 00:33:54 +0100 Subject: [PATCH 108/236] fix for #163 (SQL scanner), declare 1.1.1 --- Changes.textile | 4 ++++ lib/coderay/scanners/sql.rb | 40 +++++++++++++++---------------------- lib/coderay/version.rb | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/Changes.textile b/Changes.textile index 8e388e04..137460af 100644 --- a/Changes.textile +++ b/Changes.textile @@ -2,6 +2,10 @@ h1=. CodeRay Version History p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ +h2. Changes in 1.1.1 + +* SQL scanner: fix open strings [#163, thanks to Adam] + h2. Changes in 1.1 New scanners: diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index 93aeaf39..c25f6d25 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -57,6 +57,12 @@ class SQL < Scanner STRING_PREFIXES = /[xnb]|_\w+/i + STRING_CONTENT_PATTERN = { + '"' => / (?: [^\\"] | "" )+ /x, + "'" => / (?: [^\\'] | '' )+ /x, + '`' => / (?: [^\\`] | `` )+ /x, + } + def scan_tokens encoder, options state = :initial @@ -115,40 +121,26 @@ def scan_tokens encoder, options end elsif state == :string - if match = scan(/[^\\"'`]+/) - string_content << match - next + if match = scan(STRING_CONTENT_PATTERN[string_type]) + encoder.text_token match, :content elsif match = scan(/["'`]/) if string_type == match if peek(1) == string_type # doubling means escape - string_content << string_type << getch - next - end - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' + encoder.text_token match + getch, :content + else + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + string_type = nil end - encoder.text_token match, :delimiter - encoder.end_group :string - state = :initial - string_type = nil else - string_content << match + encoder.text_token match, :content end elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :char elsif match = scan(/ \\ . /mox) - string_content << match - next + encoder.text_token match, :content elsif match = scan(/ \\ | $ /x) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :error unless match.empty? encoder.end_group :string state = :initial diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f0850..7ea3f70c 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.1' end From f1d1e5bd49bc862be3872928e9b7652051bf2cef Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 22 Feb 2014 00:57:54 +0100 Subject: [PATCH 109/236] update term-ansicolor dependency --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 15a71aea..9d465011 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ group :development do gem "bundler", ">= 1.0.0" gem "rake" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor", '~> 1.2.2' + gem "term-ansicolor", '~> 1.3.0' gem "shoulda-context", "~> 1.1.2" gem "json" if RUBY_VERSION < '1.9' gem "rdoc" From da39961195a297293bfe274e4f60c607ad21eada Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 17 May 2014 21:16:38 +0200 Subject: [PATCH 110/236] HTML envoder keeps \t with tab_width: false Fixes #170 --- lib/coderay/encoders/html.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index d2ebb5af..c7c0c2dd 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -180,7 +180,7 @@ def setup options @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t") @opened = [] @last_opened = nil From 38e3338f2824ddc519097a7ab9d03790025f802c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 18 May 2014 00:08:48 +0200 Subject: [PATCH 111/236] relax gem version requirements --- Gemfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 9d465011..0977943f 100644 --- a/Gemfile +++ b/Gemfile @@ -6,11 +6,11 @@ gemspec # Add dependencies to develop your gem here. # Include everything needed to run rake, tests, features, etc. group :development do - gem "bundler", ">= 1.0.0" + gem "bundler" gem "rake" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor", '~> 1.3.0' - gem "shoulda-context", "~> 1.1.2" + gem "term-ansicolor" + gem "shoulda-context" gem "json" if RUBY_VERSION < '1.9' gem "rdoc" end From e1aa98e7386609fd4c84bdcd2c3ea4b26663c8b7 Mon Sep 17 00:00:00 2001 From: BenBasson Date: Wed, 11 Jun 2014 22:47:51 +0100 Subject: [PATCH 112/236] Allow $ in SQL object names. --- lib/coderay/scanners/sql.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index c25f6d25..7d57f773 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -96,7 +96,7 @@ def scan_tokens encoder, options state = :string encoder.text_token match, :delimiter - elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x) + elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x) encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match]) name_expected = false From 20b921c6c94dfdf793b8031a2dafc0f7a3df5fb2 Mon Sep 17 00:00:00 2001 From: Ryunosuke SATO Date: Wed, 12 Nov 2014 23:18:36 +0900 Subject: [PATCH 113/236] Test against Ruby 2.1 and 2.2 on Travis CI --- .travis.yml | 2 ++ Gemfile | 1 + README.markdown | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6d926f32..8e18c0ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ rvm: - ree - 1.9.3 - 2.0.0 + - 2.1 + - 2.2 - ruby-head - jruby-18mode - jruby-19mode diff --git a/Gemfile b/Gemfile index 0977943f..0fae04b8 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ group :development do gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" gem "term-ansicolor" gem "shoulda-context" + gem "test-unit" gem "json" if RUBY_VERSION < '1.9' gem "rdoc" end diff --git a/README.markdown b/README.markdown index e23f6036..15b34470 100644 --- a/README.markdown +++ b/README.markdown @@ -16,7 +16,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co ### Dependencies -CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby. +CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby. ## Example Usage From e5624a07e95cc7a3c704a4d08cddea582adc7f31 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 21 Mar 2015 03:44:49 +0100 Subject: [PATCH 114/236] prevent running out of regexp stack --- lib/coderay/scanners/diff.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index fd1aed67..74a6c27a 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -100,7 +100,7 @@ def scan_tokens encoder, options next elsif match = scan(/-/) deleted_lines_count += 1 - if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) + if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) deleted_lines = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) } inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/) ; scan(/.*/) } From cb18c6af5f53cba503fb9704ce656596ae3db075 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 22 Apr 2015 00:57:21 +0200 Subject: [PATCH 115/236] don't lie in --help output; thanks @Quintus --- bin/coderay | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/coderay b/bin/coderay index 889ae726..130a50ba 100755 --- a/bin/coderay +++ b/bin/coderay @@ -35,7 +35,7 @@ defaults: common: coderay file.rb # highlight file to terminal - coderay file.rb > file.html # highlight file to HTML page + coderay file.rb -page > file.html # highlight file to HTML page coderay file.rb -div > file.html # highlight file to HTML snippet configure output: From 080f8a8225cb911d037d1f6e58e581dec9558c58 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 11:40:13 +0100 Subject: [PATCH 116/236] add support for Ruby 2.1 number literal suffixes --- lib/coderay/scanners/ruby.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 80165cae..0492a558 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -191,7 +191,10 @@ def scan_tokens encoder, options encoder.text_token match, :error method_call_expected = false else - encoder.text_token match, self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + kind = self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + match << 'r' if match !~ /e/i && scan(/r/) + match << 'i' if scan(/i/) + encoder.text_token match, kind end value_expected = false From 39cbd37815f65f21e0433f4da4cf5fbeda2e1e3f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 12:06:26 +0100 Subject: [PATCH 117/236] add support for Ruby 2.2 quoted hash keys KNOWN ISSUE: string interpolation will not work! --- lib/coderay/scanners/ruby.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 0492a558..165d66b5 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -164,15 +164,18 @@ def scan_tokens encoder, options end elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx) - encoder.begin_group :string if match.size == 1 + encoder.begin_group :string encoder.text_token match, :delimiter state = self.class::StringState.new :string, match == '"', match # important for streaming else + kind = value_expected == true && scan(/:/) ? :key : :string + encoder.begin_group kind encoder.text_token match[0,1], :delimiter encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter - encoder.end_group :string + encoder.end_group kind + encoder.text_token ':', :operator if kind == :key value_expected = false end From d9d1eedcb235b371683eed22a6e4217caef73ffa Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 12:08:21 +0100 Subject: [PATCH 118/236] add support for Ruby 2.3 safe navigation operator --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 165d66b5..24ab71f6 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -201,7 +201,7 @@ def scan_tokens encoder, options end value_expected = false - elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x) + elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x) value_expected = true encoder.text_token match, :operator From 376884d457ac7953914cc84b94fe6404cd904fe0 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 12:10:18 +0100 Subject: [PATCH 119/236] add support for Ruby 2.3 squiggly heredoc --- lib/coderay/scanners/ruby/patterns.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index 0b36e13b..3dd6ad50 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -114,7 +114,7 @@ module Ruby::Patterns # :nodoc: all # NOTE: This is not completely correct, but # nobody needs heredoc delimiters ending with \n. HEREDOC_OPEN = / - << (-)? # $1 = float + << ([-~])? # $1 = float (?: ( [A-Za-z_0-9]+ ) # $2 = delim | From 415498eaf9417cf30656c4a745eef0409b214afc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 13:11:31 +0100 Subject: [PATCH 120/236] allow indentation of squiggly heredoc delimiter --- lib/coderay/scanners/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 24ab71f6..f7feb462 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -214,7 +214,7 @@ def scan_tokens encoder, options encoder.end_group kind heredocs ||= [] # create heredocs if empty heredocs << self.class::StringState.new(kind, quote != "'", delim, - self[1] == '-' ? :indented : :linestart) + self[1] ? :indented : :linestart) value_expected = false elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o) From c33f3f5c43064f7b468a59e086dc4a9a4f949ff7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 13:17:23 +0100 Subject: [PATCH 121/236] check for keys with escape sequences, too --- lib/coderay/scanners/ruby.rb | 5 +++-- lib/coderay/scanners/ruby/string_state.rb | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index f7feb462..5b8de42f 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -165,9 +165,10 @@ def scan_tokens encoder, options elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx) if match.size == 1 - encoder.begin_group :string + kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string + encoder.begin_group kind encoder.text_token match, :delimiter - state = self.class::StringState.new :string, match == '"', match # important for streaming + state = self.class::StringState.new kind, match == '"', match # important for streaming else kind = value_expected == true && scan(/:/) ? :key : :string encoder.begin_group kind diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 28ddd6c6..93e72086 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -37,6 +37,14 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, end end + def self.simple_key_pattern delim + if delim == "'" + / (?> (?: [^\\']+ | \\. )* ) ' : /mx + else + / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#(?!\{) )* ) " : /mx + end + end + def initialize kind, interpreted, delim, heredoc = false if heredoc pattern = heredoc_pattern delim, interpreted, heredoc == :indented From 036fb3291274ed87f106bdbeb65bbd10b4c561f9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 13:39:08 +0100 Subject: [PATCH 122/236] skip over interpolation if not nested --- lib/coderay/scanners/ruby/string_state.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 93e72086..95f1e832 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -41,7 +41,7 @@ def self.simple_key_pattern delim if delim == "'" / (?> (?: [^\\']+ | \\. )* ) ' : /mx else - / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#(?!\{) )* ) " : /mx + / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx end end From 998d1fc874d28759f5b9e4376a0e82809c8fc828 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 13:43:09 +0100 Subject: [PATCH 123/236] changelog --- Changes.textile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changes.textile b/Changes.textile index 137460af..3c389ffc 100644 --- a/Changes.textile +++ b/Changes.textile @@ -5,6 +5,10 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1.1 * SQL scanner: fix open strings [#163, thanks to Adam] +* Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1) +* Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2) +* Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3) +* Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3) h2. Changes in 1.1 From a14639c31bbe33c23853a66d6feb817da4248e1a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 13:44:18 +0100 Subject: [PATCH 124/236] don't ruin indentation --- lib/coderay/scanners/ruby/patterns.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index 3dd6ad50..e5a156d8 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -114,7 +114,7 @@ module Ruby::Patterns # :nodoc: all # NOTE: This is not completely correct, but # nobody needs heredoc delimiters ending with \n. HEREDOC_OPEN = / - << ([-~])? # $1 = float + << ([-~])? # $1 = float (?: ( [A-Za-z_0-9]+ ) # $2 = delim | From 57de8d31099fcf4004255cebacf6482b98d51340 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 14:32:04 +0100 Subject: [PATCH 125/236] try Travis new infrastructure --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8e18c0ae..9244ff40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,3 +21,4 @@ matrix: - rvm: rbx-18mode - rvm: rbx-19mode script: "rake test" # test:scanners" +sudo: false From 153f9fb053e7e59af7ac34744265a608c1c90ff7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 14:35:45 +0100 Subject: [PATCH 126/236] add Ruby 2.3 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9244ff40..f8156982 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,10 @@ rvm: - 1.8.7 - ree - 1.9.3 - - 2.0.0 + - 2.0 - 2.1 - 2.2 + - 2.3.0 - ruby-head - jruby-18mode - jruby-19mode From ae5d868a13ac722e49f0c83080ee2e05ab8d9aa8 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 14:40:46 +0100 Subject: [PATCH 127/236] fix issue with tins on older Ruby versions --- Gemfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile b/Gemfile index 0fae04b8..6d3a176f 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,7 @@ group :development do gem "rake" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" gem "term-ansicolor" + gem 'tins', '~> 1.6.0' gem "shoulda-context" gem "test-unit" gem "json" if RUBY_VERSION < '1.9' From 3e4bb6a660f8f341eca9a87a310d00170ec40872 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 15:31:41 +0100 Subject: [PATCH 128/236] add changelog for #164 --- Changes.textile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 3c389ffc..bf319efb 100644 --- a/Changes.textile +++ b/Changes.textile @@ -4,7 +4,8 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1.1 -* SQL scanner: fix open strings [#163, thanks to Adam] +* SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson] +* SQL scanner: Fix open strings [#163, thanks to Adam] * Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1) * Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2) * Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3) From 7f1f2287650c3f3da75ffe6d9e79793dfcc7a67d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 15:39:51 +0100 Subject: [PATCH 129/236] document new option to keep tabs --- lib/coderay/encoders/html.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index c7c0c2dd..942b9c89 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -25,7 +25,8 @@ module Encoders # == Options # # === :tab_width - # Convert \t characters to +n+ spaces (a number.) + # Convert \t characters to +n+ spaces (a number or false.) + # false will keep tab characters untouched. # # Default: 8 # From 0a1f500d524ff0fb5eeafef051ccbb641954a87a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 15:41:25 +0100 Subject: [PATCH 130/236] add older changes to changelog --- Changes.textile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes.textile b/Changes.textile index bf319efb..1276a339 100644 --- a/Changes.textile +++ b/Changes.textile @@ -10,6 +10,8 @@ h2. Changes in 1.1.1 * Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2) * Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3) * Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3) +* Diff scanner: Prevent running out of regexp stack. +* HTML encoder: You can keep tabs intact now by setting @tab_width: false@. h2. Changes in 1.1 From 7b0040fa41d2c55cde1ea0c91aa615bf57158e80 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 17:02:37 +0100 Subject: [PATCH 131/236] changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index 1276a339..50da5c78 100644 --- a/Changes.textile +++ b/Changes.textile @@ -12,6 +12,7 @@ h2. Changes in 1.1.1 * Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3) * Diff scanner: Prevent running out of regexp stack. * HTML encoder: You can keep tabs intact now by setting @tab_width: false@. +* Alpha style: Tweaked colors for @:function@ group with @:content@. h2. Changes in 1.1 From 50ddfcc14d435b8f667249b9e90603f113b79282 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 13 Feb 2016 17:17:27 +0100 Subject: [PATCH 132/236] changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index 50da5c78..10f1d6eb 100644 --- a/Changes.textile +++ b/Changes.textile @@ -13,6 +13,7 @@ h2. Changes in 1.1.1 * Diff scanner: Prevent running out of regexp stack. * HTML encoder: You can keep tabs intact now by setting @tab_width: false@. * Alpha style: Tweaked colors for @:function@ group with @:content@. +* File structure: One module per file, autoload CodeRay::Version, paths follow namespace hierarchy. h2. Changes in 1.1 From 0abcb3d94be31a758efbd6511694ef20164e9274 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 20 Feb 2016 16:54:36 +0100 Subject: [PATCH 133/236] Use SVG badges --- README.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.markdown b/README.markdown index 15b34470..c3f71061 100644 --- a/README.markdown +++ b/README.markdown @@ -1,8 +1,8 @@ # CodeRay -[![Build Status](https://travis-ci.org/rubychan/coderay.png)](https://travis-ci.org/rubychan/coderay) -[![Gem Version](https://badge.fury.io/rb/coderay.png)](http://badge.fury.io/rb/coderay) -[![Dependency Status](https://gemnasium.com/rubychan/coderay.png)](https://gemnasium.com/rubychan/coderay) +[![Build Status](https://travis-ci.org/rubychan/coderay.svg?branch=master)](https://travis-ci.org/rubychan/coderay) +[![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) +[![Dependency Status](https://gemnasium.com/rubychan/coderay.svg)](https://gemnasium.com/rubychan/coderay) ## About From f06e0e319e13e0e4f1f90a7bdf634bc1ca66b182 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Thu, 10 Mar 2016 14:06:48 +0100 Subject: [PATCH 134/236] Add C++11 keywords * lib/coderay/scanners/cpp.rb: --- lib/coderay/scanners/cpp.rb | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index e61f56f4..b0ffc068 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -2,14 +2,14 @@ module CodeRay module Scanners # Scanner for C++. - # + # # Aliases: +cplusplus+, c++ class CPlusPlus < Scanner register_for :cpp file_extension 'cpp' title 'C++' - + #-- http://www.cppreference.com/wiki/keywords/start KEYWORDS = [ 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break', @@ -17,14 +17,15 @@ class CPlusPlus < Scanner 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else', 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new', 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return', - 'sizeof', 'static_cast', 'struct', 'switch', 'template', - 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', + 'sizeof', 'static_assert', 'static_cast', 'struct', 'switch', + 'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', 'while', 'xor', 'xor_eq', ] # :nodoc: - + PREDEFINED_TYPES = [ - 'bool', 'char', 'double', 'float', 'int', 'long', - 'short', 'signed', 'unsigned', 'wchar_t', 'string', + 'bool', 'char', 'char16_t', 'char32_t', 'double', 'float', + 'int', 'long', 'nullptr' 'short', 'signed', 'unsigned', + 'wchar_t', 'string', ] # :nodoc: PREDEFINED_CONSTANTS = [ 'false', 'true', @@ -34,11 +35,12 @@ class CPlusPlus < Scanner 'this', ] # :nodoc: DIRECTIVES = [ - 'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator', - 'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void', - 'volatile', + 'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit', + 'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator', + 'override', 'private', 'protected', 'public', 'register', 'static', + 'thread_local', 'using', 'virtual', 'void', 'volatile', ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). @@ -48,9 +50,9 @@ class CPlusPlus < Scanner ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - + protected - + def scan_tokens encoder, options state = :initial @@ -107,7 +109,7 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -180,7 +182,7 @@ def scan_tokens encoder, options state = :initial end - + when :class_name_expected if match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) encoder.text_token match, :class @@ -194,7 +196,7 @@ def scan_tokens encoder, options state = :initial end - + else raise_inspect 'Unknown state', encoder From 26127899acf888a1888195c8b50c4603167e2808 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Fri, 11 Mar 2016 08:04:21 +0100 Subject: [PATCH 135/236] Fixed typo * FOLDERS: --- FOLDERS | 96 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/FOLDERS b/FOLDERS index f29255ae..1709d08a 100644 --- a/FOLDERS +++ b/FOLDERS @@ -1,48 +1,48 @@ -= CodeRay - folder structure - -== bench - Benchmarking system - -All benchmarking stuff goes here. - -Test inputs are stored in files named example.. -Test outputs go to bench/test.. - -Run bench/bench.rb to get a usage description. - -Run rake bench to perform an example benchmark. - - -== bin - Scripts - -Executional files for CodeRay. - -coderay:: The CodeRay executable. - -== demo - Demos and functional tests - -Demonstrational scripts to show of CodeRay's features. - -Run them as functional tests with rake test:demos. - - -== etc - Lots of stuff - -Some addidtional files for CodeRay, mainly graphics and Vim scripts. - - -== lib - CodeRay library code - -This is the base directory for the CodeRay library. - - -== rake_helpers - Rake helper libraries - -Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. - - -== test - Tests - -In the subfolder scanners/ are the scanners tests. -Each language has its own subfolder and sub-suite. - -Run with rake test. += CodeRay - folder structure + +== bench - Benchmarking system + +All benchmarking stuff goes here. + +Test inputs are stored in files named example.. +Test outputs go to bench/test.. + +Run bench/bench.rb to get a usage description. + +Run rake bench to perform an example benchmark. + + +== bin - Scripts + +Executional files for CodeRay. + +coderay:: The CodeRay executable. + +== demo - Demos and functional tests + +Demonstrational scripts to show of CodeRay's features. + +Run them as functional tests with rake test:demos. + + +== etc - Lots of stuff + +Some additional files for CodeRay, mainly graphics and Vim scripts. + + +== lib - CodeRay library code + +This is the base directory for the CodeRay library. + + +== rake_helpers - Rake helper libraries + +Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. + + +== test - Tests + +In the subfolder scanners/ are the scanners tests. +Each language has its own subfolder and sub-suite. + +Run with rake test. From 92a8e56ccd8d3aee05db9c7a821b24382528c674 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Sun, 13 Mar 2016 11:47:46 +0100 Subject: [PATCH 136/236] Small changes to grouping of new C++11 keywords * lib/coderay/scanners/cpp.rb: --- lib/coderay/scanners/cpp.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index b0ffc068..50a25b7a 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -15,7 +15,7 @@ class CPlusPlus < Scanner 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break', 'case', 'catch', 'class', 'compl', 'const_cast', 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else', - 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new', + 'enum', 'export', 'final', 'for', 'goto', 'if', 'namespace', 'new', 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return', 'sizeof', 'static_assert', 'static_cast', 'struct', 'switch', 'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', @@ -24,19 +24,19 @@ class CPlusPlus < Scanner PREDEFINED_TYPES = [ 'bool', 'char', 'char16_t', 'char32_t', 'double', 'float', - 'int', 'long', 'nullptr' 'short', 'signed', 'unsigned', + 'int', 'long', 'short', 'signed', 'unsigned', 'wchar_t', 'string', ] # :nodoc: PREDEFINED_CONSTANTS = [ 'false', 'true', - 'EOF', 'NULL', + 'EOF', 'NULL', 'nullptr' ] # :nodoc: PREDEFINED_VARIABLES = [ 'this', ] # :nodoc: DIRECTIVES = [ 'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit', - 'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator', + 'extern', 'friend', 'inline', 'mutable', 'noexcept', 'operator', 'override', 'private', 'protected', 'public', 'register', 'static', 'thread_local', 'using', 'virtual', 'void', 'volatile', ] # :nodoc: From cee12238d0ca5ea0d9d1ae9ef0e4dbf80fdfc0dc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Apr 2016 09:26:02 +0200 Subject: [PATCH 137/236] fix tests for Ruby 1.8 --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 6d3a176f..592b79fb 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,7 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem "bundler" - gem "rake" + gem "rake", "~> 10.5" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" gem "term-ansicolor" gem 'tins', '~> 1.6.0' From e94cf8ea0e342f41837c4685a45c4d2af488fe6e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Apr 2016 11:07:02 +0200 Subject: [PATCH 138/236] fix Windows line breaks in FOLDERS file --- FOLDERS | 96 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/FOLDERS b/FOLDERS index f29255ae..81784299 100644 --- a/FOLDERS +++ b/FOLDERS @@ -1,48 +1,48 @@ -= CodeRay - folder structure - -== bench - Benchmarking system - -All benchmarking stuff goes here. - -Test inputs are stored in files named example.. -Test outputs go to bench/test.. - -Run bench/bench.rb to get a usage description. - -Run rake bench to perform an example benchmark. - - -== bin - Scripts - -Executional files for CodeRay. - -coderay:: The CodeRay executable. - -== demo - Demos and functional tests - -Demonstrational scripts to show of CodeRay's features. - -Run them as functional tests with rake test:demos. - - -== etc - Lots of stuff - -Some addidtional files for CodeRay, mainly graphics and Vim scripts. - - -== lib - CodeRay library code - -This is the base directory for the CodeRay library. - - -== rake_helpers - Rake helper libraries - -Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. - - -== test - Tests - -In the subfolder scanners/ are the scanners tests. -Each language has its own subfolder and sub-suite. - -Run with rake test. += CodeRay - folder structure + +== bench - Benchmarking system + +All benchmarking stuff goes here. + +Test inputs are stored in files named example.. +Test outputs go to bench/test.. + +Run bench/bench.rb to get a usage description. + +Run rake bench to perform an example benchmark. + + +== bin - Scripts + +Executional files for CodeRay. + +coderay:: The CodeRay executable. + +== demo - Demos and functional tests + +Demonstrational scripts to show of CodeRay's features. + +Run them as functional tests with rake test:demos. + + +== etc - Lots of stuff + +Some addidtional files for CodeRay, mainly graphics and Vim scripts. + + +== lib - CodeRay library code + +This is the base directory for the CodeRay library. + + +== rake_helpers - Rake helper libraries + +Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. + + +== test - Tests + +In the subfolder scanners/ are the scanners tests. +Each language has its own subfolder and sub-suite. + +Run with rake test. From 54c94b78f18939ef8517dd2818fbb699c0fa1391 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Fri, 27 May 2016 20:40:53 +0200 Subject: [PATCH 139/236] Add env setting to allow C extensions which are not allowed by default anymore by travis-ci, see https://docs.travis-ci.com/user/languages/ruby * .travis.yml: --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index f8156982..b638af12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +env: + global: + - "JRUBY_OPTS=-Xcext.enabled=true" rvm: - 1.8.7 - ree From 2bce2ac91ad7994f9ead130387c8b24905853064 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Fri, 27 May 2016 21:06:00 +0200 Subject: [PATCH 140/236] Allow failures with jruby 18 and 19 mode, RedCloth has a known problem, see https://jgarber.lighthouseapp.com/projects/13054/tickets/230-use-rbconfig-instead-of-obsolete-and-deprecated-config * .travis.yml: --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b638af12..5aa9e5f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,6 @@ rvm: - 2.2 - 2.3.0 - ruby-head - - jruby-18mode - - jruby-19mode - jruby-head - rbx-18mode - rbx-19mode @@ -24,5 +22,7 @@ matrix: - rvm: jruby-head - rvm: rbx-18mode - rvm: rbx-19mode + - rvm: jruby-18mode + - rvm: jruby-19mode script: "rake test" # test:scanners" sudo: false From cd3bffb1086420c02774ba905a403f8efe313f46 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Fri, 27 May 2016 21:07:35 +0200 Subject: [PATCH 141/236] Shouldn't have removed the jruby lines * .travis.yml: --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5aa9e5f9..498158f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ rvm: - 2.3.0 - ruby-head - jruby-head + - jruby-18mode + - jruby-19mode - rbx-18mode - rbx-19mode branches: From de2e2acd91988e6b54f50f6c44ad85f28b68c615 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 29 May 2016 23:28:43 +0200 Subject: [PATCH 142/236] tweaking list of Rubies for CI --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8156982..f3520043 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,16 +10,16 @@ rvm: - jruby-18mode - jruby-19mode - jruby-head - - rbx-18mode - - rbx-19mode + - rbx-2 branches: only: - master matrix: allow_failures: - rvm: ruby-head + - rvm: jruby-18mode + - rvm: jruby-19mode - rvm: jruby-head - - rvm: rbx-18mode - - rvm: rbx-19mode + - rvm: rbx-2 script: "rake test" # test:scanners" sudo: false From f664af2d8e9451cd03bf1d815139514bff8956d1 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Mon, 30 May 2016 08:41:30 +0200 Subject: [PATCH 143/236] Moved final from keywords to directive * lib/coderay/scanners/cpp.rb: --- lib/coderay/scanners/cpp.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index 50a25b7a..40aeb426 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -15,7 +15,7 @@ class CPlusPlus < Scanner 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break', 'case', 'catch', 'class', 'compl', 'const_cast', 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else', - 'enum', 'export', 'final', 'for', 'goto', 'if', 'namespace', 'new', + 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new', 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return', 'sizeof', 'static_assert', 'static_cast', 'struct', 'switch', 'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', @@ -36,7 +36,7 @@ class CPlusPlus < Scanner ] # :nodoc: DIRECTIVES = [ 'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit', - 'extern', 'friend', 'inline', 'mutable', 'noexcept', 'operator', + 'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator', 'override', 'private', 'protected', 'public', 'register', 'static', 'thread_local', 'using', 'virtual', 'void', 'volatile', ] # :nodoc: From 1ad3f9651b0cdc75b7efd819b112ffa1efa16538 Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Mon, 30 May 2016 08:42:20 +0200 Subject: [PATCH 144/236] Removed merge issue * .travis.yml: --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2c7b0f4e..ea1f1c10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ rvm: - jruby-19mode - jruby-head - rbx-2 ->>>>>>> de2e2acd91988e6b54f50f6c44ad85f28b68c615 branches: only: - master From 7e330abe861bcd2b23b5ea09456335a45555530b Mon Sep 17 00:00:00 2001 From: Johnny Willemsen Date: Mon, 30 May 2016 08:50:49 +0200 Subject: [PATCH 145/236] Removed double line * .travis.yml: --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea1f1c10..e04b1645 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ rvm: - jruby-head - jruby-18mode - jruby-19mode - - jruby-head - rbx-2 branches: only: From fdb27f78983cc4d4e52d887875b1a4cd256c2757 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 2 Jun 2016 18:58:26 +0200 Subject: [PATCH 146/236] revert some changes not related to #195 --- .travis.yml | 2 +- FOLDERS | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e04b1645..a8080ce4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,9 @@ rvm: - 2.2 - 2.3.0 - ruby-head - - jruby-head - jruby-18mode - jruby-19mode + - jruby-head - rbx-2 branches: only: diff --git a/FOLDERS b/FOLDERS index 9eae35ec..1709d08a 100644 --- a/FOLDERS +++ b/FOLDERS @@ -29,6 +29,7 @@ Run them as functional tests with rake test:demos. Some additional files for CodeRay, mainly graphics and Vim scripts. + == lib - CodeRay library code This is the base directory for the CodeRay library. From 4c7bedc3bf2392444f62e90b7b8dc91d1a2effbb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Thu, 2 Jun 2016 19:01:56 +0200 Subject: [PATCH 147/236] add changelog --- Changes.textile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changes.textile b/Changes.textile index 10f1d6eb..d77cff29 100644 --- a/Changes.textile +++ b/Changes.textile @@ -2,6 +2,10 @@ h1=. CodeRay Version History p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ +h2. Changes in 1.1.2 + +* C++ scanner: Added C++11 keywords. [#195, thanks to Johnny Willemsen] + h2. Changes in 1.1.1 * SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson] From 935f003b2c15d6effb637abfc2ba40c20ad4ba98 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 28 Dec 2016 12:07:17 +0100 Subject: [PATCH 148/236] tweak benchmark numbers, improve accuracy --- bench/bench.rb | 10 +++++++--- rake_tasks/benchmark.rake | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bench/bench.rb b/bench/bench.rb index 92f9d07f..34ea9f9e 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -15,7 +15,7 @@ format = ARGV.fetch(1, 'html').downcase encoder = CodeRay.encoder(format) -size = ARGV.fetch(2, 1000).to_i * 1000 +size = ARGV.fetch(2, 2000).to_i * 1000 unless size.zero? data += data until data.size >= size data = data[0, size] @@ -23,14 +23,18 @@ size = data.size puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] -n = ARGV.fetch(3, 5).to_s[/\d+/].to_i +n = ARGV.fetch(3, 10).to_s[/\d+/].to_i require 'profile' if ARGV.include? '-p' +times = [] n.times do |i| time = Benchmark.realtime { encoder.encode(data, lang) } puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] + times << time end -STDIN.gets if ARGV.include? '-w' +times_sum = times.inject(0) { |time, sum| sum + time } +puts 'Average time: %5.2f s, %4.0f kB/s' % [times_sum / times.size, (size * n) / times_sum / 1000.0] +puts 'Best time: %5.2f s, %4.0f kB/s' % [times.min, size / times.min / 1000.0] __END__ Usage: diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake index 2e38b577..8edeffb0 100644 --- a/rake_tasks/benchmark.rake +++ b/rake_tasks/benchmark.rake @@ -1,6 +1,6 @@ desc 'Do a benchmark' task :benchmark do - ruby 'bench/bench.rb ruby html 3000' + ruby 'bench/bench.rb ruby html' end task :bench => :benchmark From 443911f134b2485f2d0b101b78342e0c0e1548fe Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Tue, 10 Jan 2017 18:38:57 +0100 Subject: [PATCH 149/236] Add Ruby 2.4.0 test to Travis CI. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a8080ce4..4c005471 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ rvm: - 2.1 - 2.2 - 2.3.0 + - 2.4.0 - ruby-head - jruby-18mode - jruby-19mode From b22fc36299faf1348f2168773b10fb1eeacfcb26 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Tue, 10 Jan 2017 19:01:40 +0100 Subject: [PATCH 150/236] Change license file name correctly. --- MIT-LICENSE.txt => MIT-LICENSE | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename MIT-LICENSE.txt => MIT-LICENSE (100%) diff --git a/MIT-LICENSE.txt b/MIT-LICENSE similarity index 100% rename from MIT-LICENSE.txt rename to MIT-LICENSE From 8afe8aa1ea8b6c7bcf46144977bcb45a72c997f6 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Wed, 11 Jan 2017 14:45:19 +0100 Subject: [PATCH 151/236] Remove executable bit for the script files with shebang. --- lib/coderay/token_kinds.rb | 0 test/functional/basic.rb | 0 test/functional/examples.rb | 0 test/functional/suite.rb | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/coderay/token_kinds.rb mode change 100755 => 100644 test/functional/basic.rb mode change 100755 => 100644 test/functional/examples.rb mode change 100755 => 100644 test/functional/suite.rb diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb old mode 100755 new mode 100644 diff --git a/test/functional/basic.rb b/test/functional/basic.rb old mode 100755 new mode 100644 diff --git a/test/functional/examples.rb b/test/functional/examples.rb old mode 100755 new mode 100644 diff --git a/test/functional/suite.rb b/test/functional/suite.rb old mode 100755 new mode 100644 From f38438f31319cf87fc87db71768b5902671a99f5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 15 Jan 2017 18:30:04 +1300 Subject: [PATCH 152/236] fixing tests --- .travis.yml | 4 ++-- Gemfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c005471..3a803e3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ rvm: - 2.0 - 2.1 - 2.2 - - 2.3.0 - - 2.4.0 + - 2.3 + - 2.4 - ruby-head - jruby-18mode - jruby-19mode diff --git a/Gemfile b/Gemfile index 592b79fb..d849d689 100644 --- a/Gemfile +++ b/Gemfile @@ -9,10 +9,10 @@ group :development do gem "bundler" gem "rake", "~> 10.5" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor" + gem "term-ansicolor", "~> 1.3.2" gem 'tins', '~> 1.6.0' gem "shoulda-context" gem "test-unit" - gem "json" if RUBY_VERSION < '1.9' + gem "json", "~> 1.8" if RUBY_VERSION < '1.9' gem "rdoc" end From 294183efae11e1a002cd455480e4dde8301f6b1e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 15 Jan 2017 18:35:33 +1300 Subject: [PATCH 153/236] fixing tests... --- .travis.yml | 4 ++-- Gemfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a803e3d..dc228fcb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ rvm: - 2.0 - 2.1 - 2.2 - - 2.3 - - 2.4 + - 2.3.3 + - 2.4.0 - ruby-head - jruby-18mode - jruby-19mode diff --git a/Gemfile b/Gemfile index d849d689..c4463a9f 100644 --- a/Gemfile +++ b/Gemfile @@ -14,5 +14,5 @@ group :development do gem "shoulda-context" gem "test-unit" gem "json", "~> 1.8" if RUBY_VERSION < '1.9' - gem "rdoc" + gem "rdoc", "~> 4.2.2" end From 6f9b2b8eb356327fa73e3e1d6c4ce18420f9db2a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 15 Jan 2017 18:38:46 +1300 Subject: [PATCH 154/236] downgrade shoulda-context --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index c4463a9f..87105557 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ group :development do gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" gem "term-ansicolor", "~> 1.3.2" gem 'tins', '~> 1.6.0' - gem "shoulda-context" + gem "shoulda-context", "= 1.2.1" gem "test-unit" gem "json", "~> 1.8" if RUBY_VERSION < '1.9' gem "rdoc", "~> 4.2.2" From 6734fffd1c7fd951ecae0bb7a2b228a0b24c833a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 15 Jan 2017 18:42:07 +1300 Subject: [PATCH 155/236] only test with latest JRuby and Rubinius --- .travis.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc228fcb..75a57bb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,19 +11,15 @@ rvm: - 2.3.3 - 2.4.0 - ruby-head - - jruby-18mode - - jruby-19mode - - jruby-head - - rbx-2 + - jruby + - rbx branches: only: - master matrix: allow_failures: - rvm: ruby-head - - rvm: jruby-18mode - - rvm: jruby-19mode - - rvm: jruby-head - - rvm: rbx-2 + - rvm: jruby + - rvm: rbx script: "rake test" # test:scanners" sudo: false From 6b3df316a81e94ca7336f75d82bb1da6c10de9f7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 15 Jan 2017 18:46:11 +1300 Subject: [PATCH 156/236] does this version work with JRuby? --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 87105557..f631cbe3 100644 --- a/Gemfile +++ b/Gemfile @@ -8,7 +8,7 @@ gemspec group :development do gem "bundler" gem "rake", "~> 10.5" - gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" + gem "RedCloth", RUBY_PLATFORM == 'java' ? "= 4.2.9" : ">= 4.0.3" gem "term-ansicolor", "~> 1.3.2" gem 'tins', '~> 1.6.0' gem "shoulda-context", "= 1.2.1" From 06f6ec4702c47fb5b7d654c477461a532062d742 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Sun, 15 Jan 2017 14:24:49 +0100 Subject: [PATCH 157/236] Remote test files and Rakefile from gem distribution. This fixes #205. --- coderay.gemspec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/coderay.gemspec b/coderay.gemspec index 328b94c1..50c195b5 100644 --- a/coderay.gemspec +++ b/coderay.gemspec @@ -24,8 +24,7 @@ Gem::Specification.new do |s| readme_file = 'README_INDEX.rdoc' - s.files = `git ls-files -- lib/* test/functional/* Rakefile #{readme_file} MIT-LICENSE`.split("\n") - s.test_files = `git ls-files -- test/functional/*`.split("\n") + s.files = `git ls-files -- lib/* #{readme_file} MIT-LICENSE`.split("\n") s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } s.require_paths = ['lib'] From 457ffbf3ee02b1b6f57eacb33df44c80f4bf9d71 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 9 Apr 2017 18:40:31 +0200 Subject: [PATCH 158/236] allow "-" in Haml tags --- Changes.textile | 1 + lib/coderay/scanners/haml.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index d77cff29..6b448549 100644 --- a/Changes.textile +++ b/Changes.textile @@ -5,6 +5,7 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1.2 * C++ scanner: Added C++11 keywords. [#195, thanks to Johnny Willemsen] +* Haml scanner: Allow @-@ in tags. h2. Changes in 1.1.1 diff --git a/lib/coderay/scanners/haml.rb b/lib/coderay/scanners/haml.rb index 5433790a..d516ba9e 100644 --- a/lib/coderay/scanners/haml.rb +++ b/lib/coderay/scanners/haml.rb @@ -75,7 +75,7 @@ def scan_tokens encoder, options tag = false - if match = scan(/%[\w:]+\/?/) + if match = scan(/%[-\w:]+\/?/) encoder.text_token match, :tag # if match = scan(/( +)(.+)/) # encoder.text_token self[1], :space From 761234f380b3e42c283e82d784826bae280f9075 Mon Sep 17 00:00:00 2001 From: Pat Allan Date: Wed, 28 Jun 2017 10:36:18 +1000 Subject: [PATCH 159/236] Get tests running with frozen string literals. --- lib/coderay/encoders/encoder.rb | 2 +- lib/coderay/encoders/html.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/coderay/encoders/encoder.rb b/lib/coderay/encoders/encoder.rb index fa5695d6..2baeedb6 100644 --- a/lib/coderay/encoders/encoder.rb +++ b/lib/coderay/encoders/encoder.rb @@ -146,7 +146,7 @@ def setup options end def get_output options - options[:out] || '' + options[:out] || ''.dup end # Append data.to_s to the output. Returns the argument. diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 942b9c89..1b33e921 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -176,7 +176,7 @@ def setup options if options[:wrap] || options[:line_numbers] @real_out = @out - @out = '' + @out = ''.dup end @break_lines = (options[:break_lines] == true) @@ -314,7 +314,7 @@ def check_group_nesting name, kind end def break_lines text, style - reopen = '' + reopen = ''.dup @opened.each_with_index do |kind, index| reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '') end From eccb20a661eaed79cbd987a524579da92edcbf9c Mon Sep 17 00:00:00 2001 From: t-gergely Date: Tue, 4 Jul 2017 17:01:52 +0200 Subject: [PATCH 160/236] allow for non-ASCII identifiers --- lib/coderay/scanners/java.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index 962154eb..b40d4a69 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -44,7 +44,7 @@ class Java < Scanner '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } # :nodoc: - IDENT = /[a-zA-Z_][A-Za-z_0-9]*/ # :nodoc: + IDENT = /[\p{L}_][\p{L}_0-9]*/ # :nodoc: protected From e94b1a91ce85306622f03996792d3665a675d69d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 7 Jul 2017 08:39:43 +0200 Subject: [PATCH 161/236] ensure that all string literals can be frozen (thanks, @pat) --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 75a57bb5..a486c233 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,8 @@ rvm: branches: only: - master +before_script: +- if (ruby -e "exit RUBY_VERSION.to_f >= 2.4"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: - rvm: ruby-head From ef5e2611d6a937fab885eb18a0c6ac5906e65ee8 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 7 Jul 2017 08:41:07 +0200 Subject: [PATCH 162/236] update Ruby minor versions --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a486c233..96233c6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ rvm: - 2.0 - 2.1 - 2.2 - - 2.3.3 - - 2.4.0 + - 2.3 + - 2.4 - ruby-head - jruby - rbx From eb3f281428d59760271c757287066d39430449ae Mon Sep 17 00:00:00 2001 From: t-gergely Date: Fri, 7 Jul 2017 10:45:30 +0200 Subject: [PATCH 163/236] compatibility with Ruby < 2 As requested by korny. Thanks. --- lib/coderay/scanners/java.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index b40d4a69..5fde4339 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -44,7 +44,7 @@ class Java < Scanner '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } # :nodoc: - IDENT = /[\p{L}_][\p{L}_0-9]*/ # :nodoc: + IDENT = /[[[:alpha:]]_][[[:alnum:]]_]*/ # :nodoc: protected From 0bea645c30a10b0e0c2949d629ec0d1932861c90 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:05:27 +0200 Subject: [PATCH 164/236] don't test on rubinius anymore --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 96233c6a..2c90b189 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ rvm: - 2.4 - ruby-head - jruby - - rbx branches: only: - master From 931ee0a74ba13049e452fb6c6d594750768cb908 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:32:22 +0200 Subject: [PATCH 165/236] don't try to run RedCloth tests with frozen string literals (they fail) --- rake_tasks/test.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index b15b9993..1a23a5bc 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -2,7 +2,7 @@ namespace :test do desc 'run functional tests' task :functional do ruby './test/functional/suite.rb' - ruby './test/functional/for_redcloth.rb' + ruby './test/functional/for_redcloth.rb' unless (''.chop! rescue true) end desc 'run unit tests' From c3473412188de4349a5cae4d4a4dc73ceac9e36f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:32:34 +0200 Subject: [PATCH 166/236] update changelog --- Changes.textile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Changes.textile b/Changes.textile index 6b448549..f9846109 100644 --- a/Changes.textile +++ b/Changes.textile @@ -4,7 +4,8 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release h2. Changes in 1.1.2 -* C++ scanner: Added C++11 keywords. [#195, thanks to Johnny Willemsen] +* Ruby future: Add support for frozen string literals. [#211, thanks to Pat Allan] +* C++ scanner: Add C++11 keywords. [#195, thanks to Johnny Willemsen] * Haml scanner: Allow @-@ in tags. h2. Changes in 1.1.1 From 860c2e132d2eed2b10285276fb6d1ca509f56cdc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:34:33 +0200 Subject: [PATCH 167/236] flag is available since Ruby 2.3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2c90b189..b99c95e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ branches: only: - master before_script: -- if (ruby -e "exit RUBY_VERSION.to_f >= 2.4"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT +- if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: - rvm: ruby-head From d4d27b2f9b090291f0e85b82f94b6e3bf266256d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:47:15 +0200 Subject: [PATCH 168/236] update changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index f9846109..37de4ace 100644 --- a/Changes.textile +++ b/Changes.textile @@ -7,6 +7,7 @@ h2. Changes in 1.1.2 * Ruby future: Add support for frozen string literals. [#211, thanks to Pat Allan] * C++ scanner: Add C++11 keywords. [#195, thanks to Johnny Willemsen] * Haml scanner: Allow @-@ in tags. +* Java scanner: Allow Unicode characters in identifiers. [#212, thanks to t-gergely] h2. Changes in 1.1.1 From f16a659e503171024924d64feb1d478dd2527e39 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:47:23 +0200 Subject: [PATCH 169/236] update development gems --- Gemfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index f631cbe3..89cd72b3 100644 --- a/Gemfile +++ b/Gemfile @@ -7,10 +7,10 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem "bundler" - gem "rake", "~> 10.5" + gem "rake", ">= 10.5" gem "RedCloth", RUBY_PLATFORM == 'java' ? "= 4.2.9" : ">= 4.0.3" gem "term-ansicolor", "~> 1.3.2" - gem 'tins', '~> 1.6.0' + gem 'tins', '>= 1.6.0' gem "shoulda-context", "= 1.2.1" gem "test-unit" gem "json", "~> 1.8" if RUBY_VERSION < '1.9' From 436c1a48074d786a542bf59ea2ea2da69cb121b9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 10:49:32 +0200 Subject: [PATCH 170/236] update more gems --- Gemfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 89cd72b3..f07e274f 100644 --- a/Gemfile +++ b/Gemfile @@ -9,10 +9,10 @@ group :development do gem "bundler" gem "rake", ">= 10.5" gem "RedCloth", RUBY_PLATFORM == 'java' ? "= 4.2.9" : ">= 4.0.3" - gem "term-ansicolor", "~> 1.3.2" + gem "term-ansicolor", ">= 1.3.2" gem 'tins', '>= 1.6.0' - gem "shoulda-context", "= 1.2.1" + gem "shoulda-context", ">= 1.2.1" gem "test-unit" - gem "json", "~> 1.8" if RUBY_VERSION < '1.9' - gem "rdoc", "~> 4.2.2" + gem "json", ">= 1.8" if RUBY_VERSION < '1.9' + gem "rdoc", ">= 4.2.2" end From 6f95c964d94a35b306e8086018917d16d7adc9ca Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 11:01:37 +0200 Subject: [PATCH 171/236] restore support for Ruby 1.8.7 --- Gemfile | 18 +++++++++--------- lib/coderay/scanners/java.rb | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index f07e274f..951e3d1b 100644 --- a/Gemfile +++ b/Gemfile @@ -6,13 +6,13 @@ gemspec # Add dependencies to develop your gem here. # Include everything needed to run rake, tests, features, etc. group :development do - gem "bundler" - gem "rake", ">= 10.5" - gem "RedCloth", RUBY_PLATFORM == 'java' ? "= 4.2.9" : ">= 4.0.3" - gem "term-ansicolor", ">= 1.3.2" - gem 'tins', '>= 1.6.0' - gem "shoulda-context", ">= 1.2.1" - gem "test-unit" - gem "json", ">= 1.8" if RUBY_VERSION < '1.9' - gem "rdoc", ">= 4.2.2" + gem 'bundler' + gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' + gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' + gem 'term-ansicolor', '>= 1.3.2' + gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' + gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' + gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' + gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' + gem 'rdoc', RUBY_VERSION < '1.9' ? '~> 4.2.2' : '>= 4.2.2' end diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index 5fde4339..3ac7efe7 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -44,7 +44,7 @@ class Java < Scanner '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } # :nodoc: - IDENT = /[[[:alpha:]]_][[[:alnum:]]_]*/ # :nodoc: + IDENT = RUBY_VERSION < '1.9' ? /[a-zA-Z_][A-Za-z_0-9]*/ : /[[[:alpha:]]_][[[:alnum:]]_]*/ # :nodoc: protected From 51ee233e72d5165f480ccad94487d995b3b2280f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 11:07:04 +0200 Subject: [PATCH 172/236] don't change benchmark rules --- bench/bench.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/bench.rb b/bench/bench.rb index 34ea9f9e..a47721e3 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -15,7 +15,7 @@ format = ARGV.fetch(1, 'html').downcase encoder = CodeRay.encoder(format) -size = ARGV.fetch(2, 2000).to_i * 1000 +size = ARGV.fetch(2, 3000).to_i * 1000 unless size.zero? data += data until data.size >= size data = data[0, size] From 44a4f08fb3a5ee07a85557ff0d7f38fa54105d2b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 11:12:52 +0200 Subject: [PATCH 173/236] fixing gems for Ruby 1.9 --- Gemfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 951e3d1b..530c0e80 100644 --- a/Gemfile +++ b/Gemfile @@ -7,12 +7,12 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem 'bundler' - gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' - gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' - gem 'term-ansicolor', '>= 1.3.2' - gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' - gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' - gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' + gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' + gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' + gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' + gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' + gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' + gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' - gem 'rdoc', RUBY_VERSION < '1.9' ? '~> 4.2.2' : '>= 4.2.2' + gem 'rdoc', RUBY_VERSION < '1.9' ? '~> 4.2.2' : '>= 4.2.2' end From 1632c66c81b9fde52df23bbbe0bbeea077a9c5dc Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 11:18:11 +0200 Subject: [PATCH 174/236] avoid regexp syntax warnings on Ruby 1.8 --- lib/coderay/scanners/java.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index 3ac7efe7..982a796f 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -44,7 +44,7 @@ class Java < Scanner '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } # :nodoc: - IDENT = RUBY_VERSION < '1.9' ? /[a-zA-Z_][A-Za-z_0-9]*/ : /[[[:alpha:]]_][[[:alnum:]]_]*/ # :nodoc: + IDENT = RUBY_VERSION < '1.9' ? /[a-zA-Z_][A-Za-z_0-9]*/ : Regexp.new('[[[:alpha:]]_][[[:alnum:]]_]*') # :nodoc: protected From e15cf96405177153e1418496a7a8e85beaa679fb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 3 Sep 2017 11:18:21 +0200 Subject: [PATCH 175/236] bump version to 1.1.2 --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 7ea3f70c..f5e7a39d 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.1' + VERSION = '1.1.2' end From 161c17d2c537a32f38f0dcca75218af69f96102b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 3 Nov 2017 00:39:44 +0100 Subject: [PATCH 176/236] port a few tweaks from dsl branch --- lib/coderay/scanners/java_script.rb | 1 - lib/coderay/scanners/lua.rb | 2 +- rake_tasks/test.rake | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 9eb0a0a1..5e278137 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -100,7 +100,6 @@ def scan_tokens encoder, options # TODO: scan over nested tags xml_scanner.tokenize match, :tokens => encoder value_expected = false - next elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x) value_expected = true diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb index fb1e45a7..81d7dae4 100644 --- a/lib/coderay/scanners/lua.rb +++ b/lib/coderay/scanners/lua.rb @@ -76,7 +76,7 @@ def scan_tokens(encoder, options) encoder.text_token(match, :comment) elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] - num_equals = match.count("=") # Number must match for comment end + num_equals = match.count("=") # Number must match for string end encoder.begin_group(:string) encoder.text_token(match, :delimiter) state = :long_string diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 1a23a5bc..ce32a02a 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -37,7 +37,7 @@ Please rename or remove it and run again to use the GitHub repository: else puts 'Downloading scanner test suite...' sh 'git clone https://github.com/rubychan/coderay-scanner-tests.git test/scanners/' - end + end unless ENV['SKIP_UPDATE_SCANNER_SUITE'] end namespace :scanner do From d4117f6a90068f3afa1afcc48f7ad9f9d3d3a533 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Fri, 3 Nov 2017 00:41:37 +0100 Subject: [PATCH 177/236] backport .gitignore from dsl branch --- .gitignore | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index deed1a27..4d962c0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,12 @@ -.DS_Store -.*~ +.* +bench/example.* coverage -pkg -spec/reports doc Gemfile.lock -.rvmrc -.ruby-gemset -.ruby-version +old-stuff +pkg +spec/examples.txt +spec/reports test/executable/source.rb.html test/executable/source.rb.json test/scanners -old-stuff From ec891978d3756c186104d8d243283f8d3104b85a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 5 Nov 2017 18:10:57 +0100 Subject: [PATCH 178/236] trying to fix tests for Ruby 2.4 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b99c95e0..1e020903 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ rvm: - 2.1 - 2.2 - 2.3 - - 2.4 + - 2.4.2 - ruby-head - jruby branches: @@ -21,6 +21,5 @@ matrix: allow_failures: - rvm: ruby-head - rvm: jruby - - rvm: rbx script: "rake test" # test:scanners" sudo: false From e603d988d7723841bc416160c45acefd9f2464eb Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 16 Dec 2017 00:36:19 +0100 Subject: [PATCH 179/236] test with ruby 2.5, too --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1e020903..a299b72e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,8 @@ rvm: - 2.1 - 2.2 - 2.3 - - 2.4.2 + - 2.4 + - 2.5 - ruby-head - jruby branches: From d38502167541a1cd1b505a0e468e0098e3ae7538 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 16 Dec 2017 00:40:39 +0100 Subject: [PATCH 180/236] tweak list of rubies to test --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a299b72e..49829cd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ rvm: - 2.1 - 2.2 - 2.3 - - 2.4 + - 2.4.2 - 2.5 - ruby-head - jruby @@ -20,6 +20,7 @@ before_script: - if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: + - rvm: 2.5 - rvm: ruby-head - rvm: jruby script: "rake test" # test:scanners" From 913c1665970ffa4e1da79470fa732aa924569ec0 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Mon, 27 Aug 2018 15:02:30 +0200 Subject: [PATCH 181/236] Remove the statement that is not always reached. --- lib/coderay/encoders/html/output.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/coderay/encoders/html/output.rb b/lib/coderay/encoders/html/output.rb index de6f6ea1..ee87fea5 100644 --- a/lib/coderay/encoders/html/output.rb +++ b/lib/coderay/encoders/html/output.rb @@ -76,8 +76,6 @@ def wrap! element, *args apply_title! title end self - when nil - return self else raise "Unknown value %p for :wrap" % element end From acf422a444813a84a952b39a569bc0f26c77c5a5 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 30 Sep 2018 10:13:54 +0200 Subject: [PATCH 182/236] support for special type 'var' --- lib/coderay/scanners/java.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index 982a796f..a490ec60 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -20,7 +20,7 @@ class Java < Scanner MAGIC_VARIABLES = %w[ this super ] # :nodoc: TYPES = %w[ boolean byte char class double enum float int interface long - short void + short void var ] << '[]' # :nodoc: because int[] should be highlighted as a type DIRECTIVES = %w[ abstract extends final implements native private protected public From dc767fca8ae78cf5760d3bf1d7e7150fde6c5951 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Thu, 14 Feb 2019 15:17:56 +0100 Subject: [PATCH 183/236] Add Ruby 2.6 fixing issues * Remove existing Tokens#filter (Array#filter) for Ruby 2.6 compatibility. * Install proper version's rdoc considering installed Ruby version. --- .travis.yml | 2 +- Gemfile | 2 +- lib/coderay/tokens.rb | 3 +++ test/unit/filter.rb | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49829cd4..c1fa23a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ rvm: - 2.3 - 2.4.2 - 2.5 + - 2.6 - ruby-head - jruby branches: @@ -20,7 +21,6 @@ before_script: - if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: - - rvm: 2.5 - rvm: ruby-head - rvm: jruby script: "rake test" # test:scanners" diff --git a/Gemfile b/Gemfile index 530c0e80..c19ac08f 100644 --- a/Gemfile +++ b/Gemfile @@ -14,5 +14,5 @@ group :development do gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' - gem 'rdoc', RUBY_VERSION < '1.9' ? '~> 4.2.2' : '>= 4.2.2' + gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index e7bffce2..b5f78e71 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -39,6 +39,9 @@ module CodeRay # You can serialize it to a JSON string and store it in a database, pass it # around to encode it more than once, send it to other algorithms... class Tokens < Array + # Remove Array#filter that is a new alias for Array#select on Ruby 2.6, + # for method_missing called with filter method. + undef_method :filter if instance_methods.include?(:filter) # The Scanner instance that created the tokens. attr_accessor :scanner diff --git a/test/unit/filter.rb b/test/unit/filter.rb index 25dff77c..6e939f32 100644 --- a/test/unit/filter.rb +++ b/test/unit/filter.rb @@ -18,6 +18,7 @@ def test_filtering_text_tokens tokens.text_token i.to_s, :index end assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) + assert_equal CodeRay::Tokens, tokens.filter.class assert_equal tokens, tokens.filter end @@ -32,6 +33,7 @@ def test_filtering_block_tokens tokens.end_line :index end assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) + assert_equal CodeRay::Tokens, tokens.filter.class assert_equal tokens, tokens.filter end From 80a33fcfcf3a46afb1541c464742edf4bf1da4e8 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 23 Feb 2019 15:30:07 +0100 Subject: [PATCH 184/236] add numeric to SQL types --- lib/coderay/scanners/sql.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index 7d57f773..c8725a8f 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -29,7 +29,7 @@ class SQL < Scanner char varchar varchar2 enum binary text tinytext mediumtext longtext blob tinyblob mediumblob longblob timestamp date time datetime year double decimal float int - integer tinyint mediumint bigint smallint unsigned bit + integer tinyint mediumint bigint smallint unsigned bit numeric bool boolean hex bin oct ) From cd7f90f4f7360c231b24e06193ea3138de5a7b84 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 23 Feb 2019 15:30:35 +0100 Subject: [PATCH 185/236] remove defunct Gemnasium badge --- README.markdown | 1 - 1 file changed, 1 deletion(-) diff --git a/README.markdown b/README.markdown index c3f71061..1402fe10 100644 --- a/README.markdown +++ b/README.markdown @@ -2,7 +2,6 @@ [![Build Status](https://travis-ci.org/rubychan/coderay.svg?branch=master)](https://travis-ci.org/rubychan/coderay) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) -[![Dependency Status](https://gemnasium.com/rubychan/coderay.svg)](https://gemnasium.com/rubychan/coderay) ## About From 9907f88568691916e4a869bc44126de8040a274d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 01:28:31 +0100 Subject: [PATCH 186/236] update changelog --- Changes.textile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Changes.textile b/Changes.textile index 37de4ace..99b79c8d 100644 --- a/Changes.textile +++ b/Changes.textile @@ -2,6 +2,12 @@ h1=. CodeRay Version History p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ +h2. Changes in 1.1.3 + +* Tokens: Ensure Ruby 2.6 compatibility. [#233, thanks to Jun Aruga] +* SQL scanner: Add @numeric@ data type. [#223, thanks to m16a1] +* Java scanner: Add @var@ as type. [#229, thanks to Davide Angelocola] + h2. Changes in 1.1.2 * Ruby future: Add support for frozen string literals. [#211, thanks to Pat Allan] From d8b4818ec4b1f06a25206e2f1e61354940af9b4a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 01:36:05 +0100 Subject: [PATCH 187/236] apparently, 1.8.7 fails on Travis? --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c1fa23a4..81917894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ rvm: - 2.1 - 2.2 - 2.3 - - 2.4.2 + - 2.4 - 2.5 - 2.6 - ruby-head @@ -21,6 +21,7 @@ before_script: - if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: + - rvm: 1.8.7 - rvm: ruby-head - rvm: jruby script: "rake test" # test:scanners" From f79710241c5bd19324418efcf24ecbf6d853a23c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 01:39:28 +0100 Subject: [PATCH 188/236] add CodeClimate config --- .codeclimate.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 00000000..f6a420d4 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,6 @@ +engines: + rubocop: + enabled: true + checks: + Rubocop/Layout/TrailingWhitespace: + enabled: false From 1b140ba2183f6eabe086547834d243f71fe03134 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 02:25:20 +0100 Subject: [PATCH 189/236] remove .codeclimate.yml --- .codeclimate.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index f6a420d4..00000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,6 +0,0 @@ -engines: - rubocop: - enabled: true - checks: - Rubocop/Layout/TrailingWhitespace: - enabled: false From 1e66d13121efecb948a1684889cbb399e4c1ff3e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 03:27:06 +0100 Subject: [PATCH 190/236] fix heredoc indentation --- test/functional/basic.rb | 8 ++++---- test/functional/for_redcloth.rb | 8 ++++---- test/unit/comment_filter.rb | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/functional/basic.rb b/test/functional/basic.rb index 752d4ba0..059d56c3 100644 --- a/test/functional/basic.rb +++ b/test/functional/basic.rb @@ -97,7 +97,7 @@ def test_comment_filter code more code - EXPECTED + EXPECTED #!/usr/bin/env ruby =begin A multi-line comment. @@ -105,7 +105,7 @@ def test_comment_filter code # A single-line comment. more code # and another comment, in-line. - INPUT + INPUT end def test_lines_of_code @@ -117,7 +117,7 @@ def test_lines_of_code code # A single-line comment. more code # and another comment, in-line. - INPUT + INPUT rHTML = <<-RHTML @@ -138,7 +138,7 @@ def test_lines_of_code - RHTML + RHTML assert_equal 0, CodeRay.scan(rHTML, :html).lines_of_code assert_equal 0, CodeRay.scan(rHTML, :php).lines_of_code assert_equal 0, CodeRay.scan(rHTML, :yaml).lines_of_code diff --git a/test/functional/for_redcloth.rb b/test/functional/for_redcloth.rb index 9fd244ed..d2b53f80 100644 --- a/test/functional/for_redcloth.rb +++ b/test/functional/for_redcloth.rb @@ -22,7 +22,7 @@ def test_for_redcloth 
puts "Hello, World!"
- BLOCKCODE + BLOCKCODE RedCloth.new('bc[ruby]. puts "Hello, World!"').to_html end @@ -32,7 +32,7 @@ def test_for_redcloth_no_lang RedCloth.new('@puts "Hello, World!"@').to_html assert_equal <<-BLOCKCODE.chomp, 
puts \"Hello, World!\"
- BLOCKCODE + BLOCKCODE RedCloth.new('bc. puts "Hello, World!"').to_html end @@ -40,7 +40,7 @@ def test_for_redcloth_style require 'coderay/for_redcloth' assert_equal <<-BLOCKCODE.chomp, 
puts \"Hello, World!\"
- BLOCKCODE + BLOCKCODE RedCloth.new('bc{color: red}. puts "Hello, World!"').to_html end @@ -52,7 +52,7 @@ def test_for_redcloth_escapes 
&
- BLOCKCODE + BLOCKCODE RedCloth.new('bc[ruby]. &').to_html end diff --git a/test/unit/comment_filter.rb b/test/unit/comment_filter.rb index e255d07f..c8147e93 100644 --- a/test/unit/comment_filter.rb +++ b/test/unit/comment_filter.rb @@ -47,7 +47,7 @@ def mymethod(self): def myfunction(): -PYTHON_FILTERED + PYTHON_FILTERED end end \ No newline at end of file From cb79f78f2d2e9c46f2cecd96071bcffb7b8b2f4a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 03:27:13 +0100 Subject: [PATCH 191/236] reorder gems --- Gemfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index c19ac08f..10dc31c2 100644 --- a/Gemfile +++ b/Gemfile @@ -7,12 +7,12 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem 'bundler' + gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' + gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' - gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' - gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' + gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' - gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' - gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' + gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' end From 8e70c5de684d247f04589215f3709da514cb2e4d Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 03:27:31 +0100 Subject: [PATCH 192/236] start using RuboCop --- .rubocop.yml | 34 ++ .rubocop_todo.yml | 1224 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1258 insertions(+) create mode 100644 .rubocop.yml create mode 100644 .rubocop_todo.yml diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..cfc5479a --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,34 @@ +inherit_from: .rubocop_todo.yml + +require: + - rubocop-performance + +AllCops: + TargetRubyVersion: 2.3 + Exclude: + - 'test/scanners/**/*' + - 'bench/example.ruby' + - 'old-stuff/**/*' + - 'test/lib/**/*' + +Gemspec/RequiredRubyVersion: + Enabled: false + +Gemspec/DuplicatedAssignment: + Enabled: false + +Layout/AccessModifierIndentation: + Enabled: false + +Layout/AlignArguments: + Enabled: false + +Layout/AlignArray: + Enabled: false + +Layout/AlignHash: + Enabled: false + +Layout/SpaceInsideBlockBraces: + EnforcedStyle: space + EnforcedStyleForEmptyBraces: space diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml new file mode 100644 index 00000000..57b00e4d --- /dev/null +++ b/.rubocop_todo.yml @@ -0,0 +1,1224 @@ +# This configuration was generated by +# `rubocop --auto-gen-config` +# on 2019-11-24 03:18:41 +0100 using RuboCop version 0.76.0. +# The point is for the user to remove these configuration records +# one by one as the offenses are removed from the code base. +# Note that changes in the inspected code, or installation of new +# versions of RuboCop, may require this file to be generated again. + +# Offense count: 26 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentOneStep, IndentationWidth. +# SupportedStyles: case, end +Layout/CaseIndentation: + Exclude: + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +Layout/CommentIndentation: + Exclude: + - 'bin/coderay' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/lua.rb' + +# Offense count: 82 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: leading, trailing +Layout/DotPosition: + Enabled: false + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: AllowBorderComment, AllowMarginComment. +Layout/EmptyComment: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 30 +# Cop supports --auto-correct. +Layout/EmptyLineAfterGuardClause: + Exclude: + - 'lib/coderay/encoders/debug_lint.rb' + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/encoders/lint.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'lib/coderay/scanners/yaml.rb' + - 'lib/coderay/tokens.rb' + - 'rake_tasks/generator.rake' + - 'rake_tasks/test.rake' + +# Offense count: 6 +# Cop supports --auto-correct. +Layout/EmptyLineAfterMagicComment: + Exclude: + - 'lib/coderay.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + - 'test/functional/basic.rb' + +# Offense count: 6 +# Cop supports --auto-correct. +# Configuration parameters: AllowAdjacentOneLineDefs, NumberOfEmptyLines. +Layout/EmptyLineBetweenDefs: + Exclude: + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/tokens.rb' + +# Offense count: 9 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: around, only_before +Layout/EmptyLinesAroundAccessModifier: + Exclude: + - 'lib/coderay/encoders/filter.rb' + - 'lib/coderay/encoders/json.rb' + - 'lib/coderay/encoders/text.rb' + - 'lib/coderay/encoders/token_kind_filter.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/encoders/yaml.rb' + +# Offense count: 18 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only +Layout/EmptyLinesAroundClassBody: + Exclude: + - 'lib/coderay/duo.rb' + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/delphi.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/xml.rb' + - 'rake_tasks/code_statistics.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Layout/EmptyLinesAroundMethodBody: + Exclude: + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/java.rb' + +# Offense count: 16 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines +Layout/EmptyLinesAroundModuleBody: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/delphi.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/xml.rb' + - 'lib/coderay/styles.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleAlignWith, AutoCorrect, Severity. +# SupportedStylesAlignWith: keyword, variable, start_of_line +Layout/EndAlignment: + Exclude: + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 2 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: native, lf, crlf +Layout/EndOfLine: + Exclude: + - 'rake_tasks/documentation.rake' + - 'rake_tasks/statistic.rake' + +# Offense count: 140 +# Cop supports --auto-correct. +# Configuration parameters: AllowForAlignment, AllowBeforeTrailingComments, ForceEqualSignAlignment. +Layout/ExtraSpacing: + Enabled: false + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: IndentationWidth. +# SupportedStyles: special_inside_parentheses, consistent, align_brackets +Layout/IndentFirstArrayElement: + EnforcedStyle: consistent + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: IndentationWidth. +# SupportedStyles: special_inside_parentheses, consistent, align_braces +Layout/IndentFirstHashElement: + EnforcedStyle: consistent + +# Offense count: 48 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: squiggly, active_support, powerpack, unindent +Layout/IndentHeredoc: + Enabled: false + +# Offense count: 53 +# Cop supports --auto-correct. +# Configuration parameters: Width, IgnoredPatterns. +Layout/IndentationWidth: + Enabled: false + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: AllowDoxygenCommentStyle. +Layout/LeadingCommentSpace: + Exclude: + - 'lib/coderay/scanners/html.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: symmetrical, new_line, same_line +Layout/MultilineArrayBraceLayout: + Exclude: + - 'lib/coderay/scanners/lua.rb' + +# Offense count: 78 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented, indented_relative_to_receiver +Layout/MultilineMethodCallIndentation: + Enabled: false + +# Offense count: 9 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented +Layout/MultilineOperationIndentation: + Exclude: + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/encoders/token_kind_filter.rb' + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'test/functional/basic.rb' + +# Offense count: 17 +# Cop supports --auto-correct. +Layout/SpaceAfterComma: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + - 'lib/coderay/token_kinds.rb' + +# Offense count: 37 +# Cop supports --auto-correct. +# Configuration parameters: AllowForAlignment. +Layout/SpaceAroundOperators: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'rake_tasks/code_statistics.rb' + - 'test/unit/json_encoder.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Layout/SpaceBeforeComment: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/token_kinds.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Layout/SpaceBeforeSemicolon: + Exclude: + - 'lib/coderay/scanners/diff.rb' + +# Offense count: 17 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces. +# SupportedStyles: space, no_space, compact +# SupportedStylesForEmptyBraces: space, no_space +Layout/SpaceInsideHashLiteralBraces: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'lib/coderay/styles/style.rb' + - 'test/unit/json_encoder.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: space, no_space +Layout/SpaceInsideParens: + Exclude: + - 'rake_tasks/code_statistics.rb' + +# Offense count: 32 +# Cop supports --auto-correct. +Layout/SpaceInsidePercentLiteralDelimiters: + Exclude: + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/sql.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Layout/SpaceInsideRangeLiteral: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/numbering.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBrackets. +# SupportedStyles: space, no_space +# SupportedStylesForEmptyBrackets: space, no_space +Layout/SpaceInsideReferenceBrackets: + Exclude: + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 6 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: space, no_space +Layout/SpaceInsideStringInterpolation: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 12 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: final_newline, final_blank_line +Layout/TrailingBlankLines: + Exclude: + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'test/executable/source.rb' + - 'test/functional/for_redcloth.rb' + - 'test/unit/comment_filter.rb' + - 'test/unit/count.rb' + - 'test/unit/json_encoder.rb' + - 'test/unit/lines_of_code.rb' + - 'test/unit/null.rb' + - 'test/unit/statistic.rb' + - 'test/unit/text.rb' + - 'test/unit/tokens.rb' + +# Offense count: 1680 +# Cop supports --auto-correct. +# Configuration parameters: AllowInHeredoc. +Layout/TrailingWhitespace: + Enabled: false + +# Offense count: 485 +# Configuration parameters: AllowSafeAssignment. +Lint/AssignmentInCondition: + Enabled: false + +# Offense count: 2 +# Configuration parameters: AllowComments. +Lint/HandleExceptions: + Exclude: + - 'bin/coderay' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 2 +Lint/IneffectiveAccessModifier: + Exclude: + - 'lib/coderay/encoders/html.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: runtime_error, standard_error +Lint/InheritException: + Exclude: + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 3 +Lint/LiteralAsCondition: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/scanners/haml.rb' + - 'test/executable/suite.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Lint/LiteralInInterpolation: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 2 +Lint/Loop: + Exclude: + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/tokens.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Lint/RedundantSplatExpansion: + Exclude: + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Lint/SendWithMixinArgument: + Exclude: + - 'lib/coderay/for_redcloth.rb' + +# Offense count: 1 +# Configuration parameters: IgnoreImplicitReferences. +Lint/ShadowedArgument: + Exclude: + - 'lib/coderay/for_redcloth.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments. +Lint/UnusedBlockArgument: + Exclude: + - 'lib/coderay/encoders/statistic.rb' + - 'lib/coderay/scanners/diff.rb' + - 'rake_tasks/code_statistics.rb' + +# Offense count: 38 +# Cop supports --auto-correct. +# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods. +Lint/UnusedMethodArgument: + Enabled: false + +# Offense count: 2 +# Configuration parameters: ContextCreatingMethods, MethodCreatingMethods. +Lint/UselessAccessModifier: + Exclude: + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/php.rb' + +# Offense count: 8 +Lint/UselessAssignment: + Exclude: + - 'lib/coderay/scanners/sql.rb' + - 'lib/coderay/scanners/yaml.rb' + - 'rake_tasks/code_statistics.rb' + - 'test/executable/suite.rb' + +# Offense count: 7 +# Configuration parameters: CheckForMethodsWithNoSideEffects. +Lint/Void: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 49 +Metrics/AbcSize: + Max: 361 + +# Offense count: 5 +# Configuration parameters: CountComments, ExcludedMethods. +# ExcludedMethods: refine +Metrics/BlockLength: + Max: 71 + +# Offense count: 183 +# Configuration parameters: CountBlocks. +Metrics/BlockNesting: + Max: 8 + +# Offense count: 27 +# Configuration parameters: CountComments. +Metrics/ClassLength: + Max: 380 + +# Offense count: 34 +Metrics/CyclomaticComplexity: + Max: 148 + +# Offense count: 63 +# Configuration parameters: CountComments, ExcludedMethods. +Metrics/MethodLength: + Max: 366 + +# Offense count: 5 +# Configuration parameters: CountComments. +Metrics/ModuleLength: + Max: 410 + +# Offense count: 34 +Metrics/PerceivedComplexity: + Max: 161 + +# Offense count: 2 +Naming/AccessorMethodName: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 24 +Naming/ConstantName: + Exclude: + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/java/builtin_types.rb' + +# Offense count: 1 +# Configuration parameters: ExpectMatchingDefinition, Regex, IgnoreExecutableScripts, AllowedAcronyms. +# AllowedAcronyms: CLI, DSL, ACL, API, ASCII, CPU, CSS, DNS, EOF, GUID, HTML, HTTP, HTTPS, ID, IP, JSON, LHS, QPS, RAM, RHS, RPC, SLA, SMTP, SQL, SSH, TCP, TLS, TTL, UDP, UI, UID, UUID, URI, URL, UTF8, VM, XML, XMPP, XSRF, XSS +Naming/FileName: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 1 +# Configuration parameters: EnforcedStyleForLeadingUnderscores. +# SupportedStylesForLeadingUnderscores: disallowed, required, optional +Naming/MemoizedInstanceVariableName: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: PreferredName. +Naming/RescuedExceptionsVariableName: + Exclude: + - 'bin/coderay' + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 5 +# Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames. +# AllowedNames: io, id, to, by, on, in, at, ip, db, os +Naming/UncommunicativeMethodParamName: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 8 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: snake_case, camelCase +Naming/VariableName: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html.rb' + - 'test/functional/basic.rb' + +# Offense count: 2 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: snake_case, normalcase, non_integer +Naming/VariableNumber: + Exclude: + - 'test/unit/tokens.rb' + +# Offense count: 1 +Performance/Caller: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Performance/Casecmp: + Exclude: + - 'rake_tasks/generator.rake' + +# Offense count: 1 +# Cop supports --auto-correct. +Performance/StringReplacement: + Exclude: + - 'lib/coderay/encoders/html.rb' + +# Offense count: 3 +Performance/UnfreezeString: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect. +Security/JSONLoad: + Exclude: + - 'test/unit/json_encoder.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Security/YAMLLoad: + Exclude: + - 'test/unit/duo.rb' + +# Offense count: 1 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: inline, group +Style/AccessModifierDeclarations: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + +# Offense count: 8 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: always, conditionals +Style/AndOr: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/erb.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 9 +# Configuration parameters: AllowedChars. +Style/AsciiComments: + Exclude: + - 'lib/coderay/scanners/lua.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/Attr: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, IgnoredMethods, AllowBracesOnProceduralOneLiners. +# SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces +# ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object +# FunctionalMethods: let, let!, subject, watch +# IgnoredMethods: lambda, proc, it +Style/BlockDelimiters: + Exclude: + - 'lib/coderay/scanners/python.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: braces, no_braces, context_dependent +Style/BracesAroundHashParameters: + Exclude: + - 'lib/coderay/scanners/ruby/patterns.rb' + +# Offense count: 3 +Style/CaseEquality: + Exclude: + - 'bin/coderay' + - 'rake_tasks/generator.rake' + - 'test/executable/suite.rb' + +# Offense count: 35 +# Cop supports --auto-correct. +Style/CharacterLiteral: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/go.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'lib/coderay/scanners/sql.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, EnforcedStyle. +# SupportedStyles: nested, compact +Style/ClassAndModuleChildren: + Exclude: + - 'lib/coderay/helpers/word_list.rb' + - 'lib/coderay/scanners/java/builtin_types.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/ClassMethods: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + +# Offense count: 2 +Style/ClassVars: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + +# Offense count: 2 +Style/CommentedKeyword: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 16 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, SingleLineConditionsOnly, IncludeTernaryExpressions. +# SupportedStyles: assign_to_condition, assign_inside_condition +Style/ConditionalAssignment: + Exclude: + - 'bin/coderay' + - 'coderay.gemspec' + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/scanners/debug.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/raydebug.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'rake_tasks/code_statistics.rb' + - 'test/executable/suite.rb' + +# Offense count: 21 +Style/Documentation: + Enabled: false + +# Offense count: 2 +Style/DoubleNegation: + Exclude: + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +Style/EachWithObject: + Exclude: + - 'bin/coderay' + - 'lib/coderay/helpers/plugin.rb' + - 'rake_tasks/code_statistics.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +Style/EmptyCaseCondition: + Exclude: + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Style/EmptyLiteral: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: compact, expanded +Style/EmptyMethod: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 8 +# Cop supports --auto-correct. +Style/Encoding: + Exclude: + - 'lib/coderay.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'test/functional/basic.rb' + +# Offense count: 12 +# Cop supports --auto-correct. +Style/ExpandPathArguments: + Exclude: + - 'bench/bench.rb' + - 'coderay.gemspec' + - 'lib/coderay.rb' + - 'test/executable/suite.rb' + - 'test/functional/basic.rb' + - 'test/functional/examples.rb' + - 'test/functional/for_redcloth.rb' + - 'test/functional/suite.rb' + - 'test/unit/file_type.rb' + - 'test/unit/lines_of_code.rb' + - 'test/unit/plugin.rb' + +# Offense count: 22 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: each, for +Style/For: + Exclude: + - 'Rakefile' + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/tokens.rb' + - 'rake_tasks/generator.rake' + - 'rake_tasks/test.rake' + - 'test/functional/basic.rb' + - 'test/functional/suite.rb' + - 'test/unit/html.rb' + - 'test/unit/json_encoder.rb' + - 'test/unit/suite.rb' + - 'test/unit/token_kind_filter.rb' + +# Offense count: 62 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: format, sprintf, percent +Style/FormatString: + Enabled: false + +# Offense count: 87 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: annotated, template, unannotated +Style/FormatStringToken: + Enabled: false + +# Offense count: 112 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: always, never +Style/FrozenStringLiteralComment: + Enabled: false + +# Offense count: 9 +# Configuration parameters: AllowedVariables. +Style/GlobalVars: + Exclude: + - 'bin/coderay' + - 'lib/coderay.rb' + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'test/functional/suite.rb' + - 'test/unit/suite.rb' + +# Offense count: 16 +# Configuration parameters: MinBodyLength. +Style/GuardClause: + Exclude: + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/encoders/terminal.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/haml.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'test/executable/suite.rb' + - 'test/unit/file_type.rb' + +# Offense count: 306 +# Cop supports --auto-correct. +# Configuration parameters: UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols. +# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys +Style/HashSyntax: + EnforcedStyle: hash_rockets + +# Offense count: 4 +Style/IdenticalConditionalBranches: + Exclude: + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 2 +# Configuration parameters: AllowIfModifier. +Style/IfInsideElse: + Exclude: + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/sass.rb' + +# Offense count: 42 +# Cop supports --auto-correct. +Style/IfUnlessModifier: + Enabled: false + +# Offense count: 3 +Style/IfUnlessModifierOfIfUnless: + Exclude: + - 'lib/coderay/encoders/text.rb' + - 'lib/coderay/scanners/erb.rb' + - 'rake_tasks/test.rake' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/InfiniteLoop: + Exclude: + - 'lib/coderay/scanners/haml.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Style/LineEndConcatenation: + Exclude: + - 'lib/coderay/for_redcloth.rb' + - 'test/functional/basic.rb' + +# Offense count: 221 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: require_parentheses, require_no_parentheses, require_no_parentheses_except_multiline +Style/MethodDefParentheses: + Enabled: false + +# Offense count: 1 +Style/MethodMissingSuper: + Exclude: + - 'lib/coderay/tokens_proxy.rb' + +# Offense count: 2 +Style/MissingRespondToMissing: + Exclude: + - 'lib/coderay/tokens.rb' + - 'lib/coderay/tokens_proxy.rb' + +# Offense count: 1 +Style/MultilineBlockChain: + Exclude: + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +Style/MultilineIfModifier: + Exclude: + - 'lib/coderay/encoders/text.rb' + - 'lib/coderay/scanners/erb.rb' + - 'rake_tasks/documentation.rake' + - 'rake_tasks/test.rake' + - 'test/functional/for_redcloth.rb' + +# Offense count: 10 +Style/MultilineTernaryOperator: + Exclude: + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 7 +Style/MultipleComparison: + Exclude: + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 247 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: literals, strict +Style/MutableConstant: + Enabled: false + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: both, prefix, postfix +Style/NegatedIf: + Exclude: + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + +# Offense count: 6 +Style/NestedTernaryOperator: + Exclude: + - 'Gemfile' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/sql.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: predicate, comparison +Style/NilComparison: + Exclude: + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/scanners/python.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Style/Not: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/erb.rb' + +# Offense count: 12 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods. +# SupportedStyles: predicate, comparison +Style/NumericPredicate: + Exclude: + - 'spec/**/*' + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/haml.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/tokens.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/OneLineConditional: + Exclude: + - 'rake_tasks/code_statistics.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/OrAssignment: + Exclude: + - 'lib/coderay/scanners/groovy.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Style/ParallelAssignment: + Exclude: + - 'lib/coderay/encoders/statistic.rb' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 30 +# Cop supports --auto-correct. +# Configuration parameters: PreferredDelimiters. +Style/PercentLiteralDelimiters: + Enabled: false + +# Offense count: 6 +# Cop supports --auto-correct. +Style/PerlBackrefs: + Exclude: + - 'bin/coderay' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/for_redcloth.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: short, verbose +Style/PreferredHashMethods: + Exclude: + - 'lib/coderay/encoders/debug_lint.rb' + - 'lib/coderay/encoders/lint.rb' + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: AllowMultipleReturnValues. +Style/RedundantReturn: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Style/RedundantSelf: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/RedundantSort: + Exclude: + - 'test/unit/plugin.rb' + +# Offense count: 58 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, AllowInnerSlashes. +# SupportedStyles: slashes, percent_r, mixed +Style/RegexpLiteral: + Enabled: false + +# Offense count: 4 +# Cop supports --auto-correct. +Style/RescueModifier: + Exclude: + - 'rake_tasks/code_statistics.rb' + - 'rake_tasks/test.rake' + - 'test/functional/for_redcloth.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: implicit, explicit +Style/RescueStandardError: + Exclude: + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: ConvertCodeThatCanStartToReturnNil, Whitelist. +# Whitelist: present?, blank?, presence, try, try! +Style/SafeNavigation: + Exclude: + - 'bin/coderay' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: AllowAsExpressionSeparator. +Style/Semicolon: + Exclude: + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: AllowIfMethodIsEmpty. +Style/SingleLineMethods: + Exclude: + - 'lib/coderay/tokens.rb' + +# Offense count: 24 +# Cop supports --auto-correct. +# Configuration parameters: . +# SupportedStyles: use_perl_names, use_english_names +Style/SpecialGlobalVars: + EnforcedStyle: use_perl_names + +# Offense count: 1 +# Cop supports --auto-correct. +Style/StderrPuts: + Exclude: + - 'lib/coderay/encoders/json.rb' + +# Offense count: 131 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline. +# SupportedStyles: single_quotes, double_quotes +Style/StringLiterals: + Enabled: false + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: single_quotes, double_quotes +Style/StringLiteralsInInterpolation: + Exclude: + - 'rake_tasks/code_statistics.rb' + +# Offense count: 1 +Style/StructInheritance: + Exclude: + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 37 +# Cop supports --auto-correct. +# Configuration parameters: MinSize. +# SupportedStyles: percent, brackets +Style/SymbolArray: + EnforcedStyle: brackets + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: IgnoredMethods. +# IgnoredMethods: respond_to, define_method +Style/SymbolProc: + Exclude: + - 'bin/coderay' + - 'lib/coderay/scanners/scanner.rb' + - 'test/unit/plugin.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, AllowSafeAssignment. +# SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex +Style/TernaryParentheses: + Exclude: + - 'lib/coderay/scanners/diff.rb' + +# Offense count: 21 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInArrayLiteral: + Exclude: + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/delphi.rb' + - 'lib/coderay/scanners/go.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/json.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'test/unit/json_encoder.rb' + +# Offense count: 26 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInHashLiteral: + Exclude: + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/terminal.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/sql.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Style/VariableInterpolation: + Exclude: + - 'bin/coderay' + - 'ideosyncratic-ruby.rb' + +# Offense count: 10 +# Cop supports --auto-correct. +# Configuration parameters: WordRegex. +# SupportedStyles: percent, brackets +Style/WordArray: + EnforcedStyle: percent + MinSize: 69 + +# Offense count: 1 +# Cop supports --auto-correct. +Style/ZeroLengthPredicate: + Exclude: + - 'lib/coderay/encoders/html.rb' + +# Offense count: 813 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. +# URISchemes: http, https +Metrics/LineLength: + Max: 266 From 279c9239afa1d6537db5965b31b9e883a0877876 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 03:29:11 +0100 Subject: [PATCH 193/236] not available on CodeClimate --- .rubocop.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index cfc5479a..978ab2bd 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,8 +1,5 @@ inherit_from: .rubocop_todo.yml -require: - - rubocop-performance - AllCops: TargetRubyVersion: 2.3 Exclude: From a632d9056853984aac6c930523a27fde42ae28a5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 14:28:25 +0100 Subject: [PATCH 194/236] Update README.markdown --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index 1402fe10..410d1bff 100644 --- a/README.markdown +++ b/README.markdown @@ -1,7 +1,7 @@ # CodeRay [![Build Status](https://travis-ci.org/rubychan/coderay.svg?branch=master)](https://travis-ci.org/rubychan/coderay) -[![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) +[![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) ## About From dceb150aff9dca50e1817636f03aa7fd7d1bb9a5 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 14:36:39 +0100 Subject: [PATCH 195/236] try setting up code climate test coverage --- .travis.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 81917894..19932b4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ env: global: - "JRUBY_OPTS=-Xcext.enabled=true" + - "CC_TEST_REPORTER_ID=faa393209ff0a104cf37511a9a03510bcee37951971b1ca4ffc2af217851d47e" +language: ruby rvm: - 1.8.7 - ree @@ -14,15 +16,20 @@ rvm: - 2.6 - ruby-head - jruby -branches: - only: - - master -before_script: -- if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: - rvm: 1.8.7 - rvm: ruby-head - rvm: jruby +branches: + only: + - master +before_script: + - if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT + - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + - chmod +x ./cc-test-reporter + - ./cc-test-reporter before-build script: "rake test" # test:scanners" +after_script: + - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT sudo: false From 3b34dc32db8e9371ad6a4bc0f810656aac8c3385 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 14:50:20 +0100 Subject: [PATCH 196/236] enfore SpaceAroundOperators --- .rubocop_todo.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 57b00e4d..317e6a47 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -237,18 +237,7 @@ Layout/SpaceAfterComma: # Cop supports --auto-correct. # Configuration parameters: AllowForAlignment. Layout/SpaceAroundOperators: - Exclude: - - 'ideosyncratic-ruby.rb' - - 'lib/coderay/scanners/c.rb' - - 'lib/coderay/scanners/cpp.rb' - - 'lib/coderay/scanners/diff.rb' - - 'lib/coderay/scanners/groovy.rb' - - 'lib/coderay/scanners/java.rb' - - 'lib/coderay/scanners/java_script.rb' - - 'lib/coderay/scanners/python.rb' - - 'lib/coderay/scanners/ruby/patterns.rb' - - 'rake_tasks/code_statistics.rb' - - 'test/unit/json_encoder.rb' + AllowForAlignment: true # Offense count: 2 # Cop supports --auto-correct. From ad756954fda50c328f000bf88da30a2b09c99043 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 14:55:02 +0100 Subject: [PATCH 197/236] tweaks to RuboCop config --- .rubocop.yml | 1 + .rubocop_todo.yml | 23 ----------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 978ab2bd..e248a433 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -2,6 +2,7 @@ inherit_from: .rubocop_todo.yml AllCops: TargetRubyVersion: 2.3 + DisplayStyleGuide: true Exclude: - 'test/scanners/**/*' - 'bench/example.ruby' diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 317e6a47..17f16e57 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -530,29 +530,6 @@ Naming/VariableNumber: Exclude: - 'test/unit/tokens.rb' -# Offense count: 1 -Performance/Caller: - Exclude: - - 'lib/coderay/scanners/scanner.rb' - -# Offense count: 1 -# Cop supports --auto-correct. -Performance/Casecmp: - Exclude: - - 'rake_tasks/generator.rake' - -# Offense count: 1 -# Cop supports --auto-correct. -Performance/StringReplacement: - Exclude: - - 'lib/coderay/encoders/html.rb' - -# Offense count: 3 -Performance/UnfreezeString: - Exclude: - - 'lib/coderay/encoders/encoder.rb' - - 'lib/coderay/encoders/html.rb' - # Offense count: 1 # Cop supports --auto-correct. # Configuration parameters: AutoCorrect. From 7eee081137cd911678e63c62413fc8edba337ea1 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:08:54 +0100 Subject: [PATCH 198/236] enforce RuboCop version --- .codeclimate.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 00000000..ae1b8e51 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,4 @@ +plugins: + rubocop: + enabled: true + channel: rubocop-0-76 From 88ca92c19d51307dd365210b5bc824afdbcc1833 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:16:26 +0100 Subject: [PATCH 199/236] tunr off maintainability checks --- .codeclimate.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.codeclimate.yml b/.codeclimate.yml index ae1b8e51..c01311f6 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,3 +1,25 @@ +version: "2" +checks: + argument-count: + enabled: false + complex-logic: + enabled: false + file-lines: + enabled: false + identical-code: + enabled: false + method-complexity: + enabled: false + method-count: + enabled: false + method-lines: + enabled: false + nested-control-flow: + enabled: false + return-statements: + enabled: false + similar-code: + enabled: false plugins: rubocop: enabled: true From 591c67b65dc4daada24ed1809605e9cbcfb3336b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:20:45 +0100 Subject: [PATCH 200/236] fix spaces in JSONEncoderTest --- test/unit/json_encoder.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unit/json_encoder.rb b/test/unit/json_encoder.rb index 4e44a646..a3a8152b 100644 --- a/test/unit/json_encoder.rb +++ b/test/unit/json_encoder.rb @@ -10,13 +10,13 @@ def test_json_output $:.delete File.dirname(__FILE__) json = CodeRay.scan('puts "Hello world!"', :ruby).json assert_equal [ - {"type"=>"text", "text"=>"puts", "kind"=>"ident"}, - {"type"=>"text", "text"=>" ", "kind"=>"space"}, - {"type"=>"block", "action"=>"open", "kind"=>"string"}, - {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"}, - {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"}, - {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"}, - {"type"=>"block", "action"=>"close", "kind"=>"string"}, + { "type" => "text", "text" => "puts", "kind" => "ident" }, + { "type" => "text", "text" => " ", "kind" => "space" }, + { "type" => "block", "action" => "open", "kind" => "string" }, + { "type" => "text", "text" => "\"", "kind" => "delimiter" }, + { "type" => "text", "text" => "Hello world!", "kind" => "content" }, + { "type" => "text", "text" => "\"", "kind" => "delimiter" }, + { "type" => "block", "action" => "close", "kind" => "string" }, ], JSON.load(json) ensure for path in old_load_paths - $: From b5b3430d4635682b767c44469e28a70fe234187e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:22:21 +0100 Subject: [PATCH 201/236] fix spaces around operators (RuboCop) --- lib/coderay/scanners/c.rb | 2 +- lib/coderay/scanners/cpp.rb | 2 +- lib/coderay/scanners/diff.rb | 4 ++-- lib/coderay/scanners/groovy.rb | 4 ++-- lib/coderay/scanners/java.rb | 2 +- lib/coderay/scanners/java_script.rb | 4 ++-- lib/coderay/scanners/python.rb | 2 +- lib/coderay/scanners/ruby/patterns.rb | 2 +- rake_tasks/code_statistics.rb | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb index 84b6e8ec..fb2f30db 100644 --- a/lib/coderay/scanners/c.rb +++ b/lib/coderay/scanners/c.rb @@ -37,7 +37,7 @@ class C < Scanner add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index 40aeb426..cd4d0941 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -49,7 +49,7 @@ class CPlusPlus < Scanner add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index 74a6c27a..a2a6fccf 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -109,7 +109,7 @@ def scan_tokens encoder, options for deleted_line, inserted_line in deleted_lines.zip(inserted_lines) pre, deleted_part, inserted_part, post = diff deleted_line, inserted_line content_scanner_entry_state = content_scanner.state - deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new) + deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new) content_scanner.state = content_scanner_entry_state || :initial inserted_lines_tokenized << content_scanner.tokenize([pre, inserted_part, post], :tokens => Tokens.new) end @@ -212,7 +212,7 @@ def diff a, b # does not precede the leftmost one from the left. j = -1 j -= 1 while j >= j_min && a[j] == b[j] - return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j+1..-1] : '' + return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j + 1..-1] : '' end end diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index c64454f0..c52ce8d3 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -22,8 +22,8 @@ class Groovy < Java add(GROOVY_MAGIC_VARIABLES, :local_variable) # :nodoc: ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8} - REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8} + REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x # :nodoc: # TODO: interpretation inside ', ", / STRING_CONTENT_PATTERN = { diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index a490ec60..7dd1919e 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -38,7 +38,7 @@ class Java < Scanner add(DIRECTIVES, :directive) # :nodoc: ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: STRING_CONTENT_PATTERN = { "'" => /[^\\']+/, '"' => /[^\\"]+/, diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 5e278137..8c13d4ff 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -40,8 +40,8 @@ class JavaScript < Scanner add(KEYWORDS, :keyword) # :nodoc: ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - REGEXP_ESCAPE = / [bBdDsSwW] /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + REGEXP_ESCAPE = / [bBdDsSwW] /x # :nodoc: STRING_CONTENT_PATTERN = { "'" => /[^\\']+/, '"' => /[^\\"]+/, diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 09c8b6e7..5da553a6 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -63,7 +63,7 @@ class Python < Scanner NAME = / [[:alpha:]_] \w* /x # :nodoc: ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x # :nodoc: OPERATOR = / \.\.\. | # ellipsis diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index e5a156d8..cd942d0d 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -60,7 +60,7 @@ module Ruby::Patterns # :nodoc: all QUOTE_TO_TYPE = { '`' => :shell, - '/'=> :regexp, + '/' => :regexp, } QUOTE_TO_TYPE.default = :string diff --git a/rake_tasks/code_statistics.rb b/rake_tasks/code_statistics.rb index 0a2016bd..32eb3f06 100644 --- a/rake_tasks/code_statistics.rb +++ b/rake_tasks/code_statistics.rb @@ -156,7 +156,7 @@ def print_code_test_stats code = calculate_code tests = calculate_tests - puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f/code)}]" + puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f / code)}]" puts "" end From 3dbf995d6d09430a0d3ae9f24b38d7bd7314574e Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:31:40 +0100 Subject: [PATCH 202/236] enforce UselessAccessModifier --- .rubocop_todo.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 17f16e57..2b0c3708 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -412,13 +412,6 @@ Lint/UnusedBlockArgument: Lint/UnusedMethodArgument: Enabled: false -# Offense count: 2 -# Configuration parameters: ContextCreatingMethods, MethodCreatingMethods. -Lint/UselessAccessModifier: - Exclude: - - 'lib/coderay/scanners/java_script.rb' - - 'lib/coderay/scanners/php.rb' - # Offense count: 8 Lint/UselessAssignment: Exclude: From 668f7fb8d8fa105638155973b73606aca16e3dc4 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:44:32 +0100 Subject: [PATCH 203/236] add RSpec --- Gemfile | 1 + spec/coderay_spec.rb | 7 +++ spec/spec_helper.rb | 100 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 spec/coderay_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/Gemfile b/Gemfile index 10dc31c2..12eeccc0 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,7 @@ group :development do gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' + gem 'rspec', '~> 3.9.0' gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' diff --git a/spec/coderay_spec.rb b/spec/coderay_spec.rb new file mode 100644 index 00000000..85e66606 --- /dev/null +++ b/spec/coderay_spec.rb @@ -0,0 +1,7 @@ +RSpec.describe CodeRay do + describe 'version' do + it "returns the Gem's version" do + expect(CodeRay::VERSION).to match(/\A\d\.\d\.\d?\z/) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..251aa510 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,100 @@ +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # This setting enables warnings. It's recommended, but in some cases may + # be too noisy due to issues in dependencies. + config.warnings = true + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end From 25de07df92d9c0ecf535d60052c6afd307c0f972 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:51:16 +0100 Subject: [PATCH 204/236] run specs on rake test --- rake_tasks/test.rake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index ce32a02a..277bd33b 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -79,4 +79,7 @@ Please rename or remove it and run again to use the GitHub repository: end end -task :test => %w(test:functional test:units test:exe) +require 'rspec/core/rake_task' +RSpec::Core::RakeTask.new(:spec) + +task :test => %w(test:functional test:units test:exe spec) From 0d373531da1231575345be1df8710cac7c0ab079 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 15:56:03 +0100 Subject: [PATCH 205/236] fix load path --- spec/spec_helper.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 251aa510..a63ebfee 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -98,3 +98,6 @@ Kernel.srand config.seed =end end + +$:.unshift File.expand_path('../lib', __FILE__) +require 'coderay' From a59099685e34c29438b11dc6eacaeea1215a150b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:01:53 +0100 Subject: [PATCH 206/236] still not loaded? --- spec/coderay_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/coderay_spec.rb b/spec/coderay_spec.rb index 85e66606..2c7b91e4 100644 --- a/spec/coderay_spec.rb +++ b/spec/coderay_spec.rb @@ -1,3 +1,5 @@ +require File.expand_path('../spec_helper', __FILE__) + RSpec.describe CodeRay do describe 'version' do it "returns the Gem's version" do From 1962f994113aec922cb3b1902ca1dc77f78de930 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:12:09 +0100 Subject: [PATCH 207/236] add SimpleCov --- Gemfile | 1 + spec/spec_helper.rb | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index 12eeccc0..0369afec 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,7 @@ group :development do gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' gem 'rspec', '~> 3.9.0' + gem 'simplecov', '~> 0.17.1' gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a63ebfee..9c1bc729 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,3 +1,6 @@ +require 'simplecov' +SimpleCov.start + # This file was generated by the `rspec --init` command. Conventionally, all # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. # The generated `.rspec` file contains `--require spec_helper` which will cause From 77734f6cfa1d90b80c53ac71c880dc5978e58dd7 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:14:18 +0100 Subject: [PATCH 208/236] fix tests for Ruby Enterprise Edition? --- spec/spec_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 9c1bc729..66f4127b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -29,7 +29,7 @@ # ...rather than: # # => "be bigger than 2" expectations.include_chain_clauses_in_custom_matcher_descriptions = true - end + end if RUBY_VERSION >= '1.9' # rspec-mocks config goes here. You can use an alternate test double # library (such as bogus or mocha) by changing the `mock_with` option here. From 8d25b7227f3eb3efb92de2d2ff57e83aed47e8b6 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:21:29 +0100 Subject: [PATCH 209/236] add spec for CodeRay.coderay_path --- spec/coderay_spec.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/spec/coderay_spec.rb b/spec/coderay_spec.rb index 2c7b91e4..8a299b3c 100644 --- a/spec/coderay_spec.rb +++ b/spec/coderay_spec.rb @@ -1,9 +1,16 @@ require File.expand_path('../spec_helper', __FILE__) RSpec.describe CodeRay do - describe 'version' do + describe '::VERSION' do it "returns the Gem's version" do expect(CodeRay::VERSION).to match(/\A\d\.\d\.\d?\z/) end end + + describe '.coderay_path' do + it 'returns an absolute file path to the given code file' do + base = File.expand_path('../..', __FILE__) + expect(CodeRay.coderay_path('file')).to eq("#{base}/lib/coderay/file") + end + end end From 69ec4d90ee666563d32341f81b388dd25c3cbbff Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:37:06 +0100 Subject: [PATCH 210/236] fix tests for Ruby 2.3 --- rake_tasks/test.rake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 277bd33b..51239fbb 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -79,7 +79,9 @@ Please rename or remove it and run again to use the GitHub repository: end end -require 'rspec/core/rake_task' -RSpec::Core::RakeTask.new(:spec) +unless RUBY_VERSION[/^2.3/] + require 'rspec/core/rake_task' + RSpec::Core::RakeTask.new(:spec) +end task :test => %w(test:functional test:units test:exe spec) From 70ea6b742137f97efd1ce02f0e16599cd1258f58 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:37:31 +0100 Subject: [PATCH 211/236] actually, we only need to disable SimpleCov --- rake_tasks/test.rake | 6 ++---- spec/spec_helper.rb | 6 ++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 51239fbb..277bd33b 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -79,9 +79,7 @@ Please rename or remove it and run again to use the GitHub repository: end end -unless RUBY_VERSION[/^2.3/] - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new(:spec) -end +require 'rspec/core/rake_task' +RSpec::Core::RakeTask.new(:spec) task :test => %w(test:functional test:units test:exe spec) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 66f4127b..78a60b29 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,7 @@ -require 'simplecov' -SimpleCov.start +unless RUBY_VERSION[/^2.3/] + require 'simplecov' + SimpleCov.start +end # This file was generated by the `rspec --init` command. Conventionally, all # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. From a5fe57486659b79a006d97489dbe2b4637543658 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:39:15 +0100 Subject: [PATCH 212/236] also disable for Ruby 1.8.7 --- spec/spec_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 78a60b29..282f576b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,4 @@ -unless RUBY_VERSION[/^2.3/] +if RUBY_VERSION >= '1.9' && !RUBY_VERSION[/^2.3/] require 'simplecov' SimpleCov.start end From a24c39336d85e3d41b709dac1ae1f0ae1cd2f658 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:41:38 +0100 Subject: [PATCH 213/236] also test with 2.7.0-preview3 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 19932b4e..cc067acb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ rvm: - 2.4 - 2.5 - 2.6 + - 2.7.0-preview3 - ruby-head - jruby matrix: From 951ea4fab6f9c8a984bd87d5abf77a84322bf011 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:43:11 +0100 Subject: [PATCH 214/236] reorder gems --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 0369afec..559648a2 100644 --- a/Gemfile +++ b/Gemfile @@ -12,8 +12,8 @@ group :development do gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' gem 'rspec', '~> 3.9.0' - gem 'simplecov', '~> 0.17.1' gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' + gem 'simplecov', '~> 0.17.1' gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' From e18aa32071f4ca83a622c9ed600b1cf4145edc06 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:52:50 +0100 Subject: [PATCH 215/236] maybe like this? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cc067acb..a8f407e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ rvm: - 2.4 - 2.5 - 2.6 - - 2.7.0-preview3 + - 2.7 - ruby-head - jruby matrix: From f3b1f3dc9dbf1145e3244c1cc6d81438c180ea29 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 16:54:03 +0100 Subject: [PATCH 216/236] disable specs for Ruby 1.8.7 --- rake_tasks/test.rake | 6 ++++-- spec/spec_helper.rb | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 277bd33b..e72c96b2 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -79,7 +79,9 @@ Please rename or remove it and run again to use the GitHub repository: end end -require 'rspec/core/rake_task' -RSpec::Core::RakeTask.new(:spec) +if RUBY_VERSION >= '1.9' + require 'rspec/core/rake_task' + RSpec::Core::RakeTask.new(:spec) +end task :test => %w(test:functional test:units test:exe spec) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 282f576b..78a60b29 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,4 @@ -if RUBY_VERSION >= '1.9' && !RUBY_VERSION[/^2.3/] +unless RUBY_VERSION[/^2.3/] require 'simplecov' SimpleCov.start end From e0b08d754b205f9204415c8d08b93a30cb92c04b Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 17:01:34 +0100 Subject: [PATCH 217/236] add simple spec for CodeRay.scan --- spec/coderay_spec.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/spec/coderay_spec.rb b/spec/coderay_spec.rb index 8a299b3c..88c9aece 100644 --- a/spec/coderay_spec.rb +++ b/spec/coderay_spec.rb @@ -13,4 +13,23 @@ expect(CodeRay.coderay_path('file')).to eq("#{base}/lib/coderay/file") end end + + describe '.scan' do + let(:code) { 'puts "Hello, World!"' } + let(:tokens) do + [ + ['puts', :ident], + [' ', :space], + [:begin_group, :string], + ['"', :delimiter], + ['Hello, World!', :content], + ['"', :delimiter], + [:end_group, :string] + ].flatten + end + + it 'returns tokens' do + expect(CodeRay.scan(code, :ruby).tokens).to eq(tokens) + end + end end From ae1c07408eb367ba4d72198e0f4c09efccf67153 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 17:18:35 +0100 Subject: [PATCH 218/236] merge coverage --- .simplecov | 4 ++++ spec/spec_helper.rb | 5 +---- test/executable/suite.rb | 1 + test/functional/for_redcloth.rb | 1 + test/functional/suite.rb | 1 + test/unit/suite.rb | 1 + 6 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 .simplecov diff --git a/.simplecov b/.simplecov new file mode 100644 index 00000000..f498df81 --- /dev/null +++ b/.simplecov @@ -0,0 +1,4 @@ +unless RUBY_VERSION[/^2.3/] + SimpleCov.command_name $0 + SimpleCov.start +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 78a60b29..4e2dac6e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,4 @@ -unless RUBY_VERSION[/^2.3/] - require 'simplecov' - SimpleCov.start -end +require 'simplecov' # This file was generated by the `rspec --init` command. Conventionally, all # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. diff --git a/test/executable/suite.rb b/test/executable/suite.rb index 997405ca..4eb86c1c 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -1,3 +1,4 @@ +require 'simplecov' require 'test/unit' require 'rubygems' unless defined? Gem require 'shoulda-context' diff --git a/test/functional/for_redcloth.rb b/test/functional/for_redcloth.rb index d2b53f80..05c6e2d6 100644 --- a/test/functional/for_redcloth.rb +++ b/test/functional/for_redcloth.rb @@ -1,3 +1,4 @@ +require 'simplecov' require 'test/unit' $:.unshift File.expand_path('../../../lib', __FILE__) diff --git a/test/functional/suite.rb b/test/functional/suite.rb index ec23eec0..f87ca0fe 100644 --- a/test/functional/suite.rb +++ b/test/functional/suite.rb @@ -1,3 +1,4 @@ +require 'simplecov' require 'test/unit' $VERBOSE = $CODERAY_DEBUG = true diff --git a/test/unit/suite.rb b/test/unit/suite.rb index 417dfed8..26ebe1b5 100755 --- a/test/unit/suite.rb +++ b/test/unit/suite.rb @@ -1,3 +1,4 @@ +require 'simplecov' require 'test/unit' require 'rubygems' From ac45fe740c0ad9f89f7cd0c3620815e9033cb1e9 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 24 Nov 2019 17:28:51 +0100 Subject: [PATCH 219/236] don't load simplecov on Ruby 1.8.7 --- test/executable/suite.rb | 2 +- test/functional/for_redcloth.rb | 2 +- test/functional/suite.rb | 2 +- test/unit/suite.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index 4eb86c1c..a6f40972 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -1,4 +1,4 @@ -require 'simplecov' +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' require 'rubygems' unless defined? Gem require 'shoulda-context' diff --git a/test/functional/for_redcloth.rb b/test/functional/for_redcloth.rb index 05c6e2d6..32a1a1b3 100644 --- a/test/functional/for_redcloth.rb +++ b/test/functional/for_redcloth.rb @@ -1,4 +1,4 @@ -require 'simplecov' +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' $:.unshift File.expand_path('../../../lib', __FILE__) diff --git a/test/functional/suite.rb b/test/functional/suite.rb index f87ca0fe..2bbc29c5 100644 --- a/test/functional/suite.rb +++ b/test/functional/suite.rb @@ -1,4 +1,4 @@ -require 'simplecov' +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' $VERBOSE = $CODERAY_DEBUG = true diff --git a/test/unit/suite.rb b/test/unit/suite.rb index 26ebe1b5..7d20dc0c 100755 --- a/test/unit/suite.rb +++ b/test/unit/suite.rb @@ -1,4 +1,4 @@ -require 'simplecov' +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' require 'rubygems' From 21b7ae87d67226a137cfa524ae623144c2296293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Sat, 18 Jan 2020 16:34:13 +0100 Subject: [PATCH 220/236] Fix rubygems deprecation ``` NOTE: Gem::Specification#rubyforge_project= is deprecated with no replacement. It will be removed on or after 2019-12-01. Gem::Specification#rubyforge_project= called from /home/deivid/.rbenv/versions/2.4.9/lib/ruby/gems/2.4.0/specifications/coderay-1.1.2.gemspec:21. ``` --- coderay.gemspec | 1 - 1 file changed, 1 deletion(-) diff --git a/coderay.gemspec b/coderay.gemspec index 50c195b5..14500ad9 100644 --- a/coderay.gemspec +++ b/coderay.gemspec @@ -28,7 +28,6 @@ Gem::Specification.new do |s| s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } s.require_paths = ['lib'] - s.rubyforge_project = s.name s.rdoc_options = '-SNw2', "-m#{readme_file}", '-t CodeRay Documentation' s.extra_rdoc_files = readme_file end From bef6209fba095c707c0592f4439e5af219d8f710 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 30 May 2020 07:58:35 +0200 Subject: [PATCH 221/236] add changelog --- Changes.textile | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes.textile b/Changes.textile index 99b79c8d..8c4f3e95 100644 --- a/Changes.textile +++ b/Changes.textile @@ -7,6 +7,7 @@ h2. Changes in 1.1.3 * Tokens: Ensure Ruby 2.6 compatibility. [#233, thanks to Jun Aruga] * SQL scanner: Add @numeric@ data type. [#223, thanks to m16a1] * Java scanner: Add @var@ as type. [#229, thanks to Davide Angelocola] +* Gem: Fix deprecation warning. [#246, thanks to David Rodríguez] h2. Changes in 1.1.2 From cf4025bf3d1a151e56626bea50e1ef7573f4e939 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 30 May 2020 07:58:43 +0200 Subject: [PATCH 222/236] trying to fix tests for 1.9.3 --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 559648a2..96f0d649 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,7 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem 'bundler' - gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' + gem 'json', '>= 1.8' if RUBY_VERSION < '2.0' gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' From 846c2f7d8a2ea99f45a3a0dedaf838d17a966ed2 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 30 May 2020 09:07:47 +0200 Subject: [PATCH 223/236] like this? --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 96f0d649..1851939d 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,7 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem 'bundler' - gem 'json', '>= 1.8' if RUBY_VERSION < '2.0' + gem 'json', '~> 1.8' if RUBY_VERSION < '2.0' gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' From d30855fe96e33fed39bd5aa7ba6879ba62306860 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 30 May 2020 09:20:17 +0200 Subject: [PATCH 224/236] bump version --- lib/coderay/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index f5e7a39d..3c68bd83 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.2' + VERSION = '1.1.3' end From a135917a983b99b3f0c07e9decf74e7a83bcc51c Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 30 May 2020 09:39:12 +0200 Subject: [PATCH 225/236] fix simplecov for Ruby 2.7 --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 1851939d..49ac338d 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ group :development do gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' gem 'rspec', '~> 3.9.0' gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' - gem 'simplecov', '~> 0.17.1' + gem 'simplecov', RUBY_VERSION < '2.7' ? '~> 0.17.1' : '>= 0.18.5' gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' From a283730a26c33431df11e5dcfa48124a74dcd732 Mon Sep 17 00:00:00 2001 From: Daniel Berger Date: Thu, 18 Jun 2020 09:18:52 -0400 Subject: [PATCH 226/236] Remove invalid -S option from rdoc_options. --- coderay.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderay.gemspec b/coderay.gemspec index 14500ad9..9aba34eb 100644 --- a/coderay.gemspec +++ b/coderay.gemspec @@ -28,6 +28,6 @@ Gem::Specification.new do |s| s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } s.require_paths = ['lib'] - s.rdoc_options = '-SNw2', "-m#{readme_file}", '-t CodeRay Documentation' + s.rdoc_options = '-Nw2', "-m#{readme_file}", '-t CodeRay Documentation' s.extra_rdoc_files = readme_file end From 228e87307c2faab06f854a79ab592a2737c20f65 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Wed, 24 Jun 2020 10:44:07 +0200 Subject: [PATCH 227/236] Update MIT-LICENSE --- MIT-LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MIT-LICENSE b/MIT-LICENSE index d8d009d1..9431e246 100644 --- a/MIT-LICENSE +++ b/MIT-LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2005-2012 Kornelius Kalnbach (@murphy_karasu) +Copyright (C) 2005 Kornelius Kalnbach (@murphy_karasu) http://coderay.rubychan.de/ @@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From c1c15034749684fcad91ad2bcb2fcd2056faf18d Mon Sep 17 00:00:00 2001 From: Mamoru TASAKA Date: Sun, 21 Feb 2021 23:04:03 +0900 Subject: [PATCH 228/236] Fix test suite for ruby 3.0 change for methods on subclass of Array With ruby 3.0, especially with https://github.com/ruby/ruby/pull/3690 , for subclass of Array, `flatten` method now returns the instance of Array, not of the subclass. To keep the object instance of the subclass, use `flatten!` instead. --- test/unit/debug.rb | 3 ++- test/unit/statistic.rb | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/unit/debug.rb b/test/unit/debug.rb index 88baf563..b694f21e 100644 --- a/test/unit/debug.rb +++ b/test/unit/debug.rb @@ -24,7 +24,8 @@ def test_creation [" \n", :space], ["[]", :method], [:end_line, :head], - ].flatten + ] + TEST_INPUT.flatten! TEST_OUTPUT = <<-'DEBUG'.chomp integer(10)operator((\\\))stringhead[ diff --git a/test/unit/statistic.rb b/test/unit/statistic.rb index 1326dca6..776774d4 100644 --- a/test/unit/statistic.rb +++ b/test/unit/statistic.rb @@ -24,7 +24,8 @@ def test_creation [" \n", :space], ["[]", :method], [:end_line, :test], - ].flatten + ] + TEST_INPUT.flatten! TEST_OUTPUT = <<-'DEBUG' Code Statistics @@ -56,4 +57,4 @@ def test_filtering_text_tokens assert_equal TEST_OUTPUT, TEST_INPUT.statistic end -end \ No newline at end of file +end From 050259de50e5dd744b193fac7823ce1b1c2be7ef Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 1 Mar 2021 16:07:34 +0100 Subject: [PATCH 229/236] test for ruby 3, too --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a8f407e5..8eaee3ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,13 @@ rvm: - 2.5 - 2.6 - 2.7 + - 3.0 - ruby-head - jruby matrix: allow_failures: - rvm: 1.8.7 + - rvm: ree - rvm: ruby-head - rvm: jruby branches: From c25e8ef53cef6e72b98547139a6a27bdd4f1aaf3 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Mon, 1 Mar 2021 16:12:22 +0100 Subject: [PATCH 230/236] fix Travis warnings --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8eaee3ff..45fb2441 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ env: - "JRUBY_OPTS=-Xcext.enabled=true" - "CC_TEST_REPORTER_ID=faa393209ff0a104cf37511a9a03510bcee37951971b1ca4ffc2af217851d47e" language: ruby +os: linux rvm: - 1.8.7 - ree @@ -18,7 +19,7 @@ rvm: - 3.0 - ruby-head - jruby -matrix: +jobs: allow_failures: - rvm: 1.8.7 - rvm: ree @@ -35,4 +36,3 @@ before_script: script: "rake test" # test:scanners" after_script: - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT -sudo: false From bd3a1676792aa2e370a308eca5753e6c52192d1a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 13 Nov 2022 20:30:17 +0100 Subject: [PATCH 231/236] add latest Rubies to test matrix --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 45fb2441..6c607d27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ rvm: - 2.6 - 2.7 - 3.0 + - 3.1 + - 3.2 - ruby-head - jruby jobs: From 286211777036fb67f82b3a24475a8fc61301bf53 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 13 Nov 2022 20:51:56 +0100 Subject: [PATCH 232/236] add CircleCI config --- .circleci/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..9dd6de46 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,12 @@ +jobs: + build: + docker: + - image: cimg/ruby:3.1.2 + environment: + RAILS_ENV: test + steps: + - checkout + - run: | + bundle install + - run: | + bundle exec rake test From 3e35c86617335f2e9edba018f51c0889f2362ebf Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 13 Nov 2022 20:55:40 +0100 Subject: [PATCH 233/236] replace Travis badge with CircleCI --- README.markdown | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index 410d1bff..da594599 100644 --- a/README.markdown +++ b/README.markdown @@ -1,7 +1,6 @@ # CodeRay -[![Build Status](https://travis-ci.org/rubychan/coderay.svg?branch=master)](https://travis-ci.org/rubychan/coderay) -[![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) +[![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) ## About From f71b25d3112ac7fcc43ca48055030319ecc7a840 Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sun, 13 Nov 2022 20:59:43 +0100 Subject: [PATCH 234/236] add missing badge token --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index da594599..e9263837 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,6 @@ # CodeRay -[![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) +[![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg&circle-token=cdc86dfde1b86067977f0fc1d3cbdd7e3171c873)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) ## About From 01144a3f9d311a3f7f8ee52ccded7b31d1d69b5f Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 2 Nov 2024 02:18:11 +0100 Subject: [PATCH 235/236] fix CircleCI status badge --- README.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index e9263837..a768ef10 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,6 @@ # CodeRay -[![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg&circle-token=cdc86dfde1b86067977f0fc1d3cbdd7e3171c873)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) +[![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg&circle-token=CCIPRJ_BbFff6nMhNtPdrCBNMDxNq_be00bbb00849a29d8d8d2e28e7b84cbf76a9ee5c)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) ## About From eabc13c2a17895dec54cfde2a2d1288e17b6722a Mon Sep 17 00:00:00 2001 From: Kornelius Kalnbach Date: Sat, 2 Nov 2024 02:22:07 +0100 Subject: [PATCH 236/236] update Ruby version in CircleCI config to 3.3.5 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9dd6de46..c76072a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ jobs: build: docker: - - image: cimg/ruby:3.1.2 + - image: cimg/ruby:3.3.5 environment: RAILS_ENV: test steps: