diff --git a/.travis.yml b/.travis.yml index 6d926f32..f8156982 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,10 @@ rvm: - 1.8.7 - ree - 1.9.3 - - 2.0.0 + - 2.0 + - 2.1 + - 2.2 + - 2.3.0 - ruby-head - jruby-18mode - jruby-19mode @@ -19,3 +22,4 @@ matrix: - rvm: rbx-18mode - rvm: rbx-19mode script: "rake test" # test:scanners" +sudo: false diff --git a/Changes.textile b/Changes.textile index 8e388e04..10f1d6eb 100644 --- a/Changes.textile +++ b/Changes.textile @@ -2,6 +2,19 @@ h1=. CodeRay Version History p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ +h2. Changes in 1.1.1 + +* SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson] +* SQL scanner: Fix open strings [#163, thanks to Adam] +* Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1) +* Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2) +* Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3) +* Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3) +* Diff scanner: Prevent running out of regexp stack. +* HTML encoder: You can keep tabs intact now by setting @tab_width: false@. +* Alpha style: Tweaked colors for @:function@ group with @:content@. +* File structure: One module per file, autoload CodeRay::Version, paths follow namespace hierarchy. + h2. Changes in 1.1 New scanners: diff --git a/Gemfile b/Gemfile index 15a71aea..6d3a176f 100644 --- a/Gemfile +++ b/Gemfile @@ -6,11 +6,13 @@ gemspec # Add dependencies to develop your gem here. # Include everything needed to run rake, tests, features, etc. group :development do - gem "bundler", ">= 1.0.0" + gem "bundler" gem "rake" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor", '~> 1.2.2' - gem "shoulda-context", "~> 1.1.2" + gem "term-ansicolor" + gem 'tins', '~> 1.6.0' + gem "shoulda-context" + gem "test-unit" gem "json" if RUBY_VERSION < '1.9' gem "rdoc" end diff --git a/README.markdown b/README.markdown index e23f6036..15b34470 100644 --- a/README.markdown +++ b/README.markdown @@ -16,7 +16,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co ### Dependencies -CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby. +CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby. ## Example Usage diff --git a/bin/coderay b/bin/coderay index 889ae726..130a50ba 100755 --- a/bin/coderay +++ b/bin/coderay @@ -35,7 +35,7 @@ defaults: common: coderay file.rb # highlight file to terminal - coderay file.rb > file.html # highlight file to HTML page + coderay file.rb -page > file.html # highlight file to HTML page coderay file.rb -div > file.html # highlight file to HTML snippet configure output: diff --git a/lib/coderay.rb b/lib/coderay.rb index f759ed63..c3de20b5 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -134,7 +134,7 @@ def self.coderay_path *path File.join CODERAY_PATH, *path end - require 'coderay/version' + autoload :VERSION, 'coderay/version' # helpers autoload :FileType, coderay_path('helpers', 'file_type') @@ -145,13 +145,13 @@ def self.coderay_path *path autoload :TokenKinds, coderay_path('token_kinds') # Plugin system - autoload :PluginHost, coderay_path('helpers', 'plugin') + autoload :PluginHost, coderay_path('helpers', 'plugin_host') autoload :Plugin, coderay_path('helpers', 'plugin') # Plugins - autoload :Scanners, coderay_path('scanner') - autoload :Encoders, coderay_path('encoder') - autoload :Styles, coderay_path('style') + autoload :Scanners, coderay_path('scanners') + autoload :Encoders, coderay_path('encoders') + autoload :Styles, coderay_path('styles') # convenience access and reusable Encoder/Scanner pair autoload :Duo, coderay_path('duo') @@ -166,7 +166,7 @@ class << self # # See also demo/demo_simple. def scan code, lang, options = {}, &block - TokensProxy.new code, lang, options, block + CodeRay::TokensProxy.new code, lang, options, block end # Scans +filename+ (a path to a code file) with the Scanner for +lang+. @@ -181,7 +181,7 @@ def scan code, lang, options = {}, &block # require 'coderay' # page = CodeRay.scan_file('some_c_code.c').html def scan_file filename, lang = :auto, options = {}, &block - lang = FileType.fetch filename, :text, true if lang == :auto + lang = CodeRay::FileType.fetch filename, :text, true if lang == :auto code = File.read filename scan code, lang, options, &block end @@ -258,7 +258,7 @@ def highlight_file filename, options = { :css => :class }, format = :div # ] # #-> 2 out of 4 tokens have the kind :integer. def encoder format, options = {} - Encoders[format].new options + CodeRay::Encoders[format].new options end # Finds the Scanner class for +lang+ and creates an instance, passing @@ -266,7 +266,7 @@ def encoder format, options = {} # # See Scanner.new. def scanner lang, options = {}, &block - Scanners[lang].new '', options, &block + CodeRay::Scanners[lang].new '', options, &block end # Extract the options for the scanner from the +options+ hash. diff --git a/lib/coderay/encoders.rb b/lib/coderay/encoders.rb new file mode 100644 index 00000000..6599186e --- /dev/null +++ b/lib/coderay/encoders.rb @@ -0,0 +1,18 @@ +module CodeRay + + # This module holds the Encoder class and its subclasses. + # For example, the HTML encoder is named CodeRay::Encoders::HTML + # can be found in coderay/encoders/html. + # + # Encoders also provides methods and constants for the register + # mechanism and the [] method that returns the Encoder class + # belonging to the given format. + module Encoders + + extend PluginHost + plugin_path File.dirname(__FILE__), 'encoders' + + autoload :Encoder, CodeRay.coderay_path('encoders', 'encoder') + + end +end diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoders/encoder.rb similarity index 93% rename from lib/coderay/encoder.rb rename to lib/coderay/encoders/encoder.rb index d2d6c7e6..fa5695d6 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoders/encoder.rb @@ -1,17 +1,6 @@ module CodeRay - - # This module holds the Encoder class and its subclasses. - # For example, the HTML encoder is named CodeRay::Encoders::HTML - # can be found in coderay/encoders/html. - # - # Encoders also provides methods and constants for the register - # mechanism and the [] method that returns the Encoder class - # belonging to the given format. module Encoders - extend PluginHost - plugin_path File.dirname(__FILE__), 'encoders' - # = Encoder # # The Encoder base class. Together with Scanner and diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index d2ebb5af..942b9c89 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -25,7 +25,8 @@ module Encoders # == Options # # === :tab_width - # Convert \t characters to +n+ spaces (a number.) + # Convert \t characters to +n+ spaces (a number or false.) + # false will keep tab characters untouched. # # Default: 8 # @@ -180,7 +181,7 @@ def setup options @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t") @opened = [] @last_opened = nil diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index 9a724fff..45679436 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -1,224 +1,5 @@ module CodeRay - # = PluginHost - # - # A simple subclass/subfolder plugin system. - # - # Example: - # class Generators - # extend PluginHost - # plugin_path 'app/generators' - # end - # - # class Generator - # extend Plugin - # PLUGIN_HOST = Generators - # end - # - # class FancyGenerator < Generator - # register_for :fancy - # end - # - # Generators[:fancy] #-> FancyGenerator - # # or - # CodeRay.require_plugin 'Generators/fancy' - # # or - # Generators::Fancy - module PluginHost - - # Raised if Encoders::[] fails because: - # * a file could not be found - # * the requested Plugin is not registered - PluginNotFound = Class.new LoadError - HostNotFound = Class.new LoadError - - PLUGIN_HOSTS = [] - PLUGIN_HOSTS_BY_ID = {} # dummy hash - - # Loads all plugins using list and load. - def load_all - for plugin in list - load plugin - end - end - - # Returns the Plugin for +id+. - # - # Example: - # yaml_plugin = MyPluginHost[:yaml] - def [] id, *args, &blk - plugin = validate_id(id) - begin - plugin = plugin_hash.[](plugin, *args, &blk) - end while plugin.is_a? String - plugin - end - - alias load [] - - # Tries to +load+ the missing plugin by translating +const+ to the - # underscore form (eg. LinesOfCode becomes lines_of_code). - def const_missing const - id = const.to_s. - gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). - gsub(/([a-z\d])([A-Z])/,'\1_\2'). - downcase - load id - end - - class << self - - # Adds the module/class to the PLUGIN_HOSTS list. - def extended mod - PLUGIN_HOSTS << mod - end - - end - - # The path where the plugins can be found. - def plugin_path *args - unless args.empty? - @plugin_path = File.expand_path File.join(*args) - end - @plugin_path ||= '' - end - - # Map a plugin_id to another. - # - # Usage: Put this in a file plugin_path/_map.rb. - # - # class MyColorHost < PluginHost - # map :navy => :dark_blue, - # :maroon => :brown, - # :luna => :moon - # end - def map hash - for from, to in hash - from = validate_id from - to = validate_id to - plugin_hash[from] = to unless plugin_hash.has_key? from - end - end - - # Define the default plugin to use when no plugin is found - # for a given id, or return the default plugin. - # - # See also map. - # - # class MyColorHost < PluginHost - # map :navy => :dark_blue - # default :gray - # end - # - # MyColorHost.default # loads and returns the Gray plugin - def default id = nil - if id - id = validate_id id - raise "The default plugin can't be named \"default\"." if id == :default - plugin_hash[:default] = id - else - load :default - end - end - - # Every plugin must register itself for +id+ by calling register_for, - # which calls this method. - # - # See Plugin#register_for. - def register plugin, id - plugin_hash[validate_id(id)] = plugin - end - - # A Hash of plugion_id => Plugin pairs. - def plugin_hash - @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } - end - - # Returns an array of all .rb files in the plugin path. - # - # The extension .rb is not included. - def list - Dir[path_to('*')].select do |file| - File.basename(file)[/^(?!_)\w+\.rb$/] - end.map do |file| - File.basename(file, '.rb').to_sym - end - end - - # Returns an array of all Plugins. - # - # Note: This loads all plugins using load_all. - def all_plugins - load_all - plugin_hash.values.grep(Class) - end - - # Loads the map file (see map). - # - # This is done automatically when plugin_path is called. - def load_plugin_map - mapfile = path_to '_map' - if File.exist? mapfile - require mapfile - true - else - false - end - end - - protected - - # Return a plugin hash that automatically loads plugins. - def make_plugin_hash - Hash.new do |h, plugin_id| - id = validate_id(plugin_id) - path = path_to id - begin - require path - rescue LoadError => boom - if h.has_key?(:default) - h[:default] - else - raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] - end - else - # Plugin should have registered by now - if h.has_key? id - h[id] - else - raise PluginNotFound, "No #{self.name} plugin for #{id.inspect} found in #{path}." - end - end - end - end - - # Returns the expected path to the plugin file for the given id. - def path_to plugin_id - File.join plugin_path, "#{plugin_id}.rb" - end - - # Converts +id+ to a valid plugin ID String, or returns +nil+. - # - # Raises +ArgumentError+ for all other objects, or if the - # given String includes non-alphanumeric characters (\W). - def validate_id id - case id - when Symbol - id.to_s - when String - if id[/\w+/] == id - id.downcase - else - raise ArgumentError, "Invalid id given: #{id}" - end - else - raise ArgumentError, "Symbol or String expected, but #{id.class} given." - end - end - - end - - # = Plugin # # Plugins have to include this module. diff --git a/lib/coderay/helpers/plugin_host.rb b/lib/coderay/helpers/plugin_host.rb new file mode 100644 index 00000000..e9bc17c1 --- /dev/null +++ b/lib/coderay/helpers/plugin_host.rb @@ -0,0 +1,221 @@ +module CodeRay + + # = PluginHost + # + # A simple subclass/subfolder plugin system. + # + # Example: + # class Generators + # extend PluginHost + # plugin_path 'app/generators' + # end + # + # class Generator + # extend Plugin + # PLUGIN_HOST = Generators + # end + # + # class FancyGenerator < Generator + # register_for :fancy + # end + # + # Generators[:fancy] #-> FancyGenerator + # # or + # CodeRay.require_plugin 'Generators/fancy' + # # or + # Generators::Fancy + module PluginHost + + # Raised if Encoders::[] fails because: + # * a file could not be found + # * the requested Plugin is not registered + PluginNotFound = Class.new LoadError + HostNotFound = Class.new LoadError + + PLUGIN_HOSTS = [] + PLUGIN_HOSTS_BY_ID = {} # dummy hash + + # Loads all plugins using list and load. + def load_all + for plugin in list + load plugin + end + end + + # Returns the Plugin for +id+. + # + # Example: + # yaml_plugin = MyPluginHost[:yaml] + def [] id, *args, &blk + plugin = validate_id(id) + begin + plugin = plugin_hash.[](plugin, *args, &blk) + end while plugin.is_a? String + plugin + end + + alias load [] + + # Tries to +load+ the missing plugin by translating +const+ to the + # underscore form (eg. LinesOfCode becomes lines_of_code). + def const_missing const + id = const.to_s. + gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). + gsub(/([a-z\d])([A-Z])/,'\1_\2'). + downcase + load id + end + + class << self + + # Adds the module/class to the PLUGIN_HOSTS list. + def extended mod + PLUGIN_HOSTS << mod + end + + end + + # The path where the plugins can be found. + def plugin_path *args + unless args.empty? + @plugin_path = File.expand_path File.join(*args) + end + @plugin_path ||= '' + end + + # Map a plugin_id to another. + # + # Usage: Put this in a file plugin_path/_map.rb. + # + # class MyColorHost < PluginHost + # map :navy => :dark_blue, + # :maroon => :brown, + # :luna => :moon + # end + def map hash + for from, to in hash + from = validate_id from + to = validate_id to + plugin_hash[from] = to unless plugin_hash.has_key? from + end + end + + # Define the default plugin to use when no plugin is found + # for a given id, or return the default plugin. + # + # See also map. + # + # class MyColorHost < PluginHost + # map :navy => :dark_blue + # default :gray + # end + # + # MyColorHost.default # loads and returns the Gray plugin + def default id = nil + if id + id = validate_id id + raise "The default plugin can't be named \"default\"." if id == :default + plugin_hash[:default] = id + else + load :default + end + end + + # Every plugin must register itself for +id+ by calling register_for, + # which calls this method. + # + # See Plugin#register_for. + def register plugin, id + plugin_hash[validate_id(id)] = plugin + end + + # A Hash of plugion_id => Plugin pairs. + def plugin_hash + @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } + end + + # Returns an array of all .rb files in the plugin path. + # + # The extension .rb is not included. + def list + Dir[path_to('*')].select do |file| + File.basename(file)[/^(?!_)\w+\.rb$/] + end.map do |file| + File.basename(file, '.rb').to_sym + end + end + + # Returns an array of all Plugins. + # + # Note: This loads all plugins using load_all. + def all_plugins + load_all + plugin_hash.values.grep(Class) + end + + # Loads the map file (see map). + # + # This is done automatically when plugin_path is called. + def load_plugin_map + mapfile = path_to '_map' + if File.exist? mapfile + require mapfile + true + else + false + end + end + + protected + + # Return a plugin hash that automatically loads plugins. + def make_plugin_hash + Hash.new do |h, plugin_id| + id = validate_id(plugin_id) + path = path_to id + begin + require path + rescue LoadError => boom + if h.has_key?(:default) + h[:default] + else + raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] + end + else + # Plugin should have registered by now + if h.has_key? id + h[id] + else + raise PluginNotFound, "No #{self.name} plugin for #{id.inspect} found in #{path}." + end + end + end + end + + # Returns the expected path to the plugin file for the given id. + def path_to plugin_id + File.join plugin_path, "#{plugin_id}.rb" + end + + # Converts +id+ to a valid plugin ID String, or returns +nil+. + # + # Raises +ArgumentError+ for all other objects, or if the + # given String includes non-alphanumeric characters (\W). + def validate_id id + case id + when Symbol + id.to_s + when String + if id[/\w+/] == id + id.downcase + else + raise ArgumentError, "Invalid id given: #{id}" + end + else + raise ArgumentError, "Symbol or String expected, but #{id.class} given." + end + end + + end + +end diff --git a/lib/coderay/scanners.rb b/lib/coderay/scanners.rb new file mode 100644 index 00000000..3c7e594d --- /dev/null +++ b/lib/coderay/scanners.rb @@ -0,0 +1,27 @@ +require 'strscan' + +module CodeRay + + autoload :WordList, coderay_path('helpers', 'word_list') + + # = Scanners + # + # This module holds the Scanner class and its subclasses. + # For example, the Ruby scanner is named CodeRay::Scanners::Ruby + # can be found in coderay/scanners/ruby. + # + # Scanner also provides methods and constants for the register + # mechanism and the [] method that returns the Scanner class + # belonging to the given lang. + # + # See PluginHost. + module Scanners + + extend PluginHost + plugin_path File.dirname(__FILE__), 'scanners' + + autoload :Scanner, CodeRay.coderay_path('scanners', 'scanner') + + end + +end diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index fd1aed67..74a6c27a 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -100,7 +100,7 @@ def scan_tokens encoder, options next elsif match = scan(/-/) deleted_lines_count += 1 - if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) + if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) deleted_lines = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) } inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/) ; scan(/.*/) } diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index b282864a..962154eb 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -36,7 +36,7 @@ class Java < Scanner add(BuiltinTypes::List, :predefined_type). add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception). add(DIRECTIVES, :directive) # :nodoc: - + ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: STRING_CONTENT_PATTERN = { diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 80165cae..5b8de42f 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -164,15 +164,19 @@ def scan_tokens encoder, options end elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx) - encoder.begin_group :string if match.size == 1 + kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string + encoder.begin_group kind encoder.text_token match, :delimiter - state = self.class::StringState.new :string, match == '"', match # important for streaming + state = self.class::StringState.new kind, match == '"', match # important for streaming else + kind = value_expected == true && scan(/:/) ? :key : :string + encoder.begin_group kind encoder.text_token match[0,1], :delimiter encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter - encoder.end_group :string + encoder.end_group kind + encoder.text_token ':', :operator if kind == :key value_expected = false end @@ -191,11 +195,14 @@ def scan_tokens encoder, options encoder.text_token match, :error method_call_expected = false else - encoder.text_token match, self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + kind = self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + match << 'r' if match !~ /e/i && scan(/r/) + match << 'i' if scan(/i/) + encoder.text_token match, kind end value_expected = false - elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x) + elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x) value_expected = true encoder.text_token match, :operator @@ -208,7 +215,7 @@ def scan_tokens encoder, options encoder.end_group kind heredocs ||= [] # create heredocs if empty heredocs << self.class::StringState.new(kind, quote != "'", delim, - self[1] == '-' ? :indented : :linestart) + self[1] ? :indented : :linestart) value_expected = false elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o) diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index 0b36e13b..e5a156d8 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -114,7 +114,7 @@ module Ruby::Patterns # :nodoc: all # NOTE: This is not completely correct, but # nobody needs heredoc delimiters ending with \n. HEREDOC_OPEN = / - << (-)? # $1 = float + << ([-~])? # $1 = float (?: ( [A-Za-z_0-9]+ ) # $2 = delim | diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 28ddd6c6..95f1e832 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -37,6 +37,14 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, end end + def self.simple_key_pattern delim + if delim == "'" + / (?> (?: [^\\']+ | \\. )* ) ' : /mx + else + / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx + end + end + def initialize kind, interpreted, delim, heredoc = false if heredoc pattern = heredoc_pattern delim, interpreted, heredoc == :indented diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanners/scanner.rb similarity index 94% rename from lib/coderay/scanner.rb rename to lib/coderay/scanners/scanner.rb index b3f7e175..efa710d9 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanners/scanner.rb @@ -1,25 +1,7 @@ # encoding: utf-8 -require 'strscan' module CodeRay - - autoload :WordList, coderay_path('helpers', 'word_list') - - # = Scanners - # - # This module holds the Scanner class and its subclasses. - # For example, the Ruby scanner is named CodeRay::Scanners::Ruby - # can be found in coderay/scanners/ruby. - # - # Scanner also provides methods and constants for the register - # mechanism and the [] method that returns the Scanner class - # belonging to the given lang. - # - # See PluginHost. module Scanners - extend PluginHost - plugin_path File.dirname(__FILE__), 'scanners' - # = Scanner # diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index 93aeaf39..7d57f773 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -57,6 +57,12 @@ class SQL < Scanner STRING_PREFIXES = /[xnb]|_\w+/i + STRING_CONTENT_PATTERN = { + '"' => / (?: [^\\"] | "" )+ /x, + "'" => / (?: [^\\'] | '' )+ /x, + '`' => / (?: [^\\`] | `` )+ /x, + } + def scan_tokens encoder, options state = :initial @@ -90,7 +96,7 @@ def scan_tokens encoder, options state = :string encoder.text_token match, :delimiter - elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x) + elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x) encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match]) name_expected = false @@ -115,40 +121,26 @@ def scan_tokens encoder, options end elsif state == :string - if match = scan(/[^\\"'`]+/) - string_content << match - next + if match = scan(STRING_CONTENT_PATTERN[string_type]) + encoder.text_token match, :content elsif match = scan(/["'`]/) if string_type == match if peek(1) == string_type # doubling means escape - string_content << string_type << getch - next - end - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' + encoder.text_token match + getch, :content + else + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + string_type = nil end - encoder.text_token match, :delimiter - encoder.end_group :string - state = :initial - string_type = nil else - string_content << match + encoder.text_token match, :content end elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :char elsif match = scan(/ \\ . /mox) - string_content << match - next + encoder.text_token match, :content elsif match = scan(/ \\ | $ /x) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :error unless match.empty? encoder.end_group :string state = :initial diff --git a/lib/coderay/styles.rb b/lib/coderay/styles.rb new file mode 100644 index 00000000..d8fa8aa7 --- /dev/null +++ b/lib/coderay/styles.rb @@ -0,0 +1,15 @@ +module CodeRay + + # This module holds the Style class and its subclasses. + # + # See Plugin. + module Styles + + extend PluginHost + plugin_path File.dirname(__FILE__), 'styles' + + autoload :Style, CodeRay.coderay_path('styles', 'style') + + end + +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index d304dc4a..f21cefed 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -82,7 +82,8 @@ class Alpha < Style .exception { color:#C00; font-weight:bold } .float { color:#60E } .function { color:#06B; font-weight:bold } -.function .delimiter { color:#024; font-weight:bold } +.function .delimiter { color:#059 } +.function .content { color:#037 } .global-variable { color:#d70 } .hex { color:#02b } .id { color:#33D; font-weight:bold } diff --git a/lib/coderay/style.rb b/lib/coderay/styles/style.rb similarity index 64% rename from lib/coderay/style.rb rename to lib/coderay/styles/style.rb index df4704f4..a3353861 100644 --- a/lib/coderay/style.rb +++ b/lib/coderay/styles/style.rb @@ -1,11 +1,6 @@ module CodeRay - - # This module holds the Style class and its subclasses. - # - # See Plugin. + module Styles - extend PluginHost - plugin_path File.dirname(__FILE__), 'styles' # Base class for styles. # diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f0850..7ea3f70c 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.1' end diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 371214a2..b15b9993 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -1,9 +1,4 @@ namespace :test do - desc 'run all sample tests' - task :samples do - ruby './test/samples/suite.rb' - end - desc 'run functional tests' task :functional do ruby './test/functional/suite.rb' @@ -85,4 +80,3 @@ Please rename or remove it and run again to use the GitHub repository: end task :test => %w(test:functional test:units test:exe) -task :samples => 'test:samples' \ No newline at end of file diff --git a/test/samples/README b/test/samples/README deleted file mode 100644 index b0ab6e44..00000000 --- a/test/samples/README +++ /dev/null @@ -1 +0,0 @@ -These demos rely on Ruby 1.8.5, so the tests might fail on other versions. \ No newline at end of file diff --git a/test/samples/cache.actual b/test/samples/cache.actual deleted file mode 100644 index c131857f..00000000 --- a/test/samples/cache.actual +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/test/samples/cache.expected b/test/samples/cache.expected deleted file mode 100644 index c131857f..00000000 --- a/test/samples/cache.expected +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/test/samples/cache.rb b/test/samples/cache.rb deleted file mode 100644 index 0c0b8478..00000000 --- a/test/samples/cache.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'coderay' - -html_encoder = CodeRay.encoder :html - -scanner = Hash.new do |h, lang| - h[lang] = CodeRay.scanner lang -end - -for lang in [:ruby, :html] - tokens = scanner[lang].tokenize 'test ' - puts html_encoder.encode_tokens(tokens) -end diff --git a/test/samples/count.expected b/test/samples/count.expected deleted file mode 100644 index 7f493b6c..00000000 --- a/test/samples/count.expected +++ /dev/null @@ -1 +0,0 @@ -2 out of 4 tokens have the kind :integer. diff --git a/test/samples/count.rb b/test/samples/count.rb deleted file mode 100644 index bcb7c2dc..00000000 --- a/test/samples/count.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'coderay' - -stats = CodeRay.encoder(:statistic) -stats.encode("puts 17 + 4\n", :ruby) - -puts '%d out of %d tokens have the kind :integer.' % [ - stats.type_stats[:integer].count, - stats.real_token_count -] -#-> 2 out of 4 tokens have the kind :integer. diff --git a/test/samples/css.actual b/test/samples/css.actual deleted file mode 100644 index be73a7f9..00000000 --- a/test/samples/css.actual +++ /dev/null @@ -1,127 +0,0 @@ -.CodeRay { - background-color: hsl(0,0%,95%); - border: 1px solid silver; - color: black; -} -.CodeRay pre { - margin: 0px; -} - -span.CodeRay { white-space: pre; border: 0px; padding: 2px; } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } -table.CodeRay td { padding: 2px 4px; vertical-align: top; } - -.CodeRay .line-numbers { - background-color: hsl(180,65%,90%); - color: gray; - text-align: right; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; -} -.CodeRay .line-numbers a { - background-color: hsl(180,65%,90%) !important; - color: gray !important; - text-decoration: none !important; -} -.CodeRay .line-numbers pre { - word-break: normal; -} -.CodeRay .line-numbers a:target { color: blue !important; } -.CodeRay .line-numbers .highlighted { color: red !important; } -.CodeRay .line-numbers .highlighted a { color: red !important; } -.CodeRay span.line-numbers { padding: 0px 4px; } -.CodeRay .line { display: block; float: left; width: 100%; } -.CodeRay .code { width: 100%; } - -.CodeRay .debug { color: white !important; background: blue !important; } - -.CodeRay .annotation { color:#007 } -.CodeRay .attribute-name { color:#b48 } -.CodeRay .attribute-value { color:#700 } -.CodeRay .binary { color:#549 } -.CodeRay .binary .char { color:#325 } -.CodeRay .binary .delimiter { color:#325 } -.CodeRay .char { color:#D20 } -.CodeRay .char .content { color:#D20 } -.CodeRay .char .delimiter { color:#710 } -.CodeRay .class { color:#B06; font-weight:bold } -.CodeRay .class-variable { color:#369 } -.CodeRay .color { color:#0A0 } -.CodeRay .comment { color:#777 } -.CodeRay .comment .char { color:#444 } -.CodeRay .comment .delimiter { color:#444 } -.CodeRay .constant { color:#036; font-weight:bold } -.CodeRay .decorator { color:#B0B } -.CodeRay .definition { color:#099; font-weight:bold } -.CodeRay .delimiter { color:black } -.CodeRay .directive { color:#088; font-weight:bold } -.CodeRay .docstring { color:#D42; } -.CodeRay .doctype { color:#34b } -.CodeRay .done { text-decoration: line-through; color: gray } -.CodeRay .entity { color:#800; font-weight:bold } -.CodeRay .error { color:#F00; background-color:#FAA } -.CodeRay .escape { color:#666 } -.CodeRay .exception { color:#C00; font-weight:bold } -.CodeRay .float { color:#60E } -.CodeRay .function { color:#06B; font-weight:bold } -.CodeRay .function .delimiter { color:#024; font-weight:bold } -.CodeRay .global-variable { color:#d70 } -.CodeRay .hex { color:#02b } -.CodeRay .id { color:#33D; font-weight:bold } -.CodeRay .include { color:#B44; font-weight:bold } -.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.CodeRay .inline-delimiter { font-weight: bold; color: #666 } -.CodeRay .instance-variable { color:#33B } -.CodeRay .integer { color:#00D } -.CodeRay .imaginary { color:#f00 } -.CodeRay .important { color:#D00 } -.CodeRay .key { color: #606 } -.CodeRay .key .char { color: #60f } -.CodeRay .key .delimiter { color: #404 } -.CodeRay .keyword { color:#080; font-weight:bold } -.CodeRay .label { color:#970; font-weight:bold } -.CodeRay .local-variable { color:#963 } -.CodeRay .namespace { color:#707; font-weight:bold } -.CodeRay .octal { color:#40E } -.CodeRay .operator { } -.CodeRay .predefined { color:#369; font-weight:bold } -.CodeRay .predefined-constant { color:#069 } -.CodeRay .predefined-type { color:#0a5; font-weight:bold } -.CodeRay .preprocessor { color:#579 } -.CodeRay .pseudo-class { color:#00C; font-weight:bold } -.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } -.CodeRay .regexp .content { color:#808 } -.CodeRay .regexp .delimiter { color:#404 } -.CodeRay .regexp .modifier { color:#C2C } -.CodeRay .reserved { color:#080; font-weight:bold } -.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } -.CodeRay .shell .content { color:#2B2 } -.CodeRay .shell .delimiter { color:#161 } -.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } -.CodeRay .string .char { color: #b0b } -.CodeRay .string .content { color: #D20 } -.CodeRay .string .delimiter { color: #710 } -.CodeRay .string .modifier { color: #E40 } -.CodeRay .symbol { color:#A60 } -.CodeRay .symbol .content { color:#A60 } -.CodeRay .symbol .delimiter { color:#630 } -.CodeRay .tag { color:#070 } -.CodeRay .type { color:#339; font-weight:bold } -.CodeRay .value { color: #088 } -.CodeRay .variable { color:#037 } - -.CodeRay .insert { background: hsla(120,100%,50%,0.12) } -.CodeRay .delete { background: hsla(0,100%,50%,0.12) } -.CodeRay .change { color: #bbf; background: #007 } -.CodeRay .head { color: #f8f; background: #505 } -.CodeRay .head .filename { color: white; } - -.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } -.CodeRay .change .change { color: #88f } -.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.expected b/test/samples/css.expected deleted file mode 100644 index be73a7f9..00000000 --- a/test/samples/css.expected +++ /dev/null @@ -1,127 +0,0 @@ -.CodeRay { - background-color: hsl(0,0%,95%); - border: 1px solid silver; - color: black; -} -.CodeRay pre { - margin: 0px; -} - -span.CodeRay { white-space: pre; border: 0px; padding: 2px; } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } -table.CodeRay td { padding: 2px 4px; vertical-align: top; } - -.CodeRay .line-numbers { - background-color: hsl(180,65%,90%); - color: gray; - text-align: right; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; -} -.CodeRay .line-numbers a { - background-color: hsl(180,65%,90%) !important; - color: gray !important; - text-decoration: none !important; -} -.CodeRay .line-numbers pre { - word-break: normal; -} -.CodeRay .line-numbers a:target { color: blue !important; } -.CodeRay .line-numbers .highlighted { color: red !important; } -.CodeRay .line-numbers .highlighted a { color: red !important; } -.CodeRay span.line-numbers { padding: 0px 4px; } -.CodeRay .line { display: block; float: left; width: 100%; } -.CodeRay .code { width: 100%; } - -.CodeRay .debug { color: white !important; background: blue !important; } - -.CodeRay .annotation { color:#007 } -.CodeRay .attribute-name { color:#b48 } -.CodeRay .attribute-value { color:#700 } -.CodeRay .binary { color:#549 } -.CodeRay .binary .char { color:#325 } -.CodeRay .binary .delimiter { color:#325 } -.CodeRay .char { color:#D20 } -.CodeRay .char .content { color:#D20 } -.CodeRay .char .delimiter { color:#710 } -.CodeRay .class { color:#B06; font-weight:bold } -.CodeRay .class-variable { color:#369 } -.CodeRay .color { color:#0A0 } -.CodeRay .comment { color:#777 } -.CodeRay .comment .char { color:#444 } -.CodeRay .comment .delimiter { color:#444 } -.CodeRay .constant { color:#036; font-weight:bold } -.CodeRay .decorator { color:#B0B } -.CodeRay .definition { color:#099; font-weight:bold } -.CodeRay .delimiter { color:black } -.CodeRay .directive { color:#088; font-weight:bold } -.CodeRay .docstring { color:#D42; } -.CodeRay .doctype { color:#34b } -.CodeRay .done { text-decoration: line-through; color: gray } -.CodeRay .entity { color:#800; font-weight:bold } -.CodeRay .error { color:#F00; background-color:#FAA } -.CodeRay .escape { color:#666 } -.CodeRay .exception { color:#C00; font-weight:bold } -.CodeRay .float { color:#60E } -.CodeRay .function { color:#06B; font-weight:bold } -.CodeRay .function .delimiter { color:#024; font-weight:bold } -.CodeRay .global-variable { color:#d70 } -.CodeRay .hex { color:#02b } -.CodeRay .id { color:#33D; font-weight:bold } -.CodeRay .include { color:#B44; font-weight:bold } -.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.CodeRay .inline-delimiter { font-weight: bold; color: #666 } -.CodeRay .instance-variable { color:#33B } -.CodeRay .integer { color:#00D } -.CodeRay .imaginary { color:#f00 } -.CodeRay .important { color:#D00 } -.CodeRay .key { color: #606 } -.CodeRay .key .char { color: #60f } -.CodeRay .key .delimiter { color: #404 } -.CodeRay .keyword { color:#080; font-weight:bold } -.CodeRay .label { color:#970; font-weight:bold } -.CodeRay .local-variable { color:#963 } -.CodeRay .namespace { color:#707; font-weight:bold } -.CodeRay .octal { color:#40E } -.CodeRay .operator { } -.CodeRay .predefined { color:#369; font-weight:bold } -.CodeRay .predefined-constant { color:#069 } -.CodeRay .predefined-type { color:#0a5; font-weight:bold } -.CodeRay .preprocessor { color:#579 } -.CodeRay .pseudo-class { color:#00C; font-weight:bold } -.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } -.CodeRay .regexp .content { color:#808 } -.CodeRay .regexp .delimiter { color:#404 } -.CodeRay .regexp .modifier { color:#C2C } -.CodeRay .reserved { color:#080; font-weight:bold } -.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } -.CodeRay .shell .content { color:#2B2 } -.CodeRay .shell .delimiter { color:#161 } -.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } -.CodeRay .string .char { color: #b0b } -.CodeRay .string .content { color: #D20 } -.CodeRay .string .delimiter { color: #710 } -.CodeRay .string .modifier { color: #E40 } -.CodeRay .symbol { color:#A60 } -.CodeRay .symbol .content { color:#A60 } -.CodeRay .symbol .delimiter { color:#630 } -.CodeRay .tag { color:#070 } -.CodeRay .type { color:#339; font-weight:bold } -.CodeRay .value { color: #088 } -.CodeRay .variable { color:#037 } - -.CodeRay .insert { background: hsla(120,100%,50%,0.12) } -.CodeRay .delete { background: hsla(0,100%,50%,0.12) } -.CodeRay .change { color: #bbf; background: #007 } -.CodeRay .head { color: #f8f; background: #505 } -.CodeRay .head .filename { color: white; } - -.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } -.CodeRay .change .change { color: #88f } -.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.rb b/test/samples/css.rb deleted file mode 100644 index 52e4bcc7..00000000 --- a/test/samples/css.rb +++ /dev/null @@ -1,4 +0,0 @@ -require 'coderay' - -# print the default stylesheet for HTML codes -puts CodeRay::Encoders[:html]::CSS.new.stylesheet diff --git a/test/samples/div.actual b/test/samples/div.actual deleted file mode 100644 index d1e692ab..00000000 --- a/test/samples/div.actual +++ /dev/null @@ -1,17 +0,0 @@ -
- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/test/samples/div.expected b/test/samples/div.expected deleted file mode 100644 index d1e692ab..00000000 --- a/test/samples/div.expected +++ /dev/null @@ -1,17 +0,0 @@ -
- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/test/samples/div.rb b/test/samples/div.rb deleted file mode 100644 index 27b6f328..00000000 --- a/test/samples/div.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'coderay' - -puts CodeRay.scan(DATA.read, :ruby).div - -__END__ -for a in 0..255 - a = a.chr - begin - x = eval("?\\#{a}") - if x == a[0] - next - else - print "#{a}: #{x}" - end - rescue SyntaxError => boom - print "#{a}: error" - end - puts -end diff --git a/test/samples/encoder.actual b/test/samples/encoder.actual deleted file mode 100644 index 8bd83a92..00000000 --- a/test/samples/encoder.actual +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" -compressed: 79 byte < 1200 byte - -Undump: - -Code Statistics - -Tokens 800 - Non-Whitespace 400 -Bytes Total 1200 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 800 100.00 % 1.5 - space 400 50.00 % 1.0 - integer 200 25.00 % 1.5 - ident 100 12.50 % 4.0 - operator 100 12.50 % 1.0 - diff --git a/test/samples/encoder.expected b/test/samples/encoder.expected deleted file mode 100644 index 8bd83a92..00000000 --- a/test/samples/encoder.expected +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" -compressed: 79 byte < 1200 byte - -Undump: - -Code Statistics - -Tokens 800 - Non-Whitespace 400 -Bytes Total 1200 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 800 100.00 % 1.5 - space 400 50.00 % 1.0 - integer 200 25.00 % 1.5 - ident 100 12.50 % 4.0 - operator 100 12.50 % 1.0 - diff --git a/test/samples/encoder.rb b/test/samples/encoder.rb deleted file mode 100644 index bfcfbfa4..00000000 --- a/test/samples/encoder.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'coderay' - -SAMPLE = "puts 17 + 4\n" -puts 'Encoders Demo: ' + SAMPLE -scanner = CodeRay::Scanners[:ruby].new SAMPLE -encoder = CodeRay::Encoders[:statistic].new - -tokens = scanner.tokenize -stats = encoder.encode_tokens tokens - -puts -puts 'Statistic:' -puts stats - -# alternative 1 -tokens = CodeRay.scan SAMPLE, :ruby -encoder = CodeRay.encoder(:json) -textual = encoder.encode_tokens tokens -puts -puts 'Original text:' -puts textual - -# alternative 2 -yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby -puts -puts 'YAML:' -puts yaml - -# alternative 3 -require 'zlib' -BIGSAMPLE = SAMPLE * 100 -dump = Zlib::Deflate.deflate(CodeRay.scan(BIGSAMPLE, :ruby).debug) -puts -puts 'Dump:' -p dump -puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size] - -puts -puts 'Undump:' -puts CodeRay.scan(Zlib::Inflate.inflate(dump), :debug).statistic diff --git a/test/samples/global_vars.actual b/test/samples/global_vars.actual deleted file mode 100644 index e69de29b..00000000 diff --git a/test/samples/global_vars.diff b/test/samples/global_vars.diff deleted file mode 100644 index 2ed33821..00000000 --- a/test/samples/global_vars.diff +++ /dev/null @@ -1,5 +0,0 @@ -1,3d0 -< <--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -< ODER -< TEXT = <--$BLA-->.TEST(...) -\ No newline at end of file diff --git a/test/samples/global_vars.expected b/test/samples/global_vars.expected deleted file mode 100644 index 0dc13c8f..00000000 --- a/test/samples/global_vars.expected +++ /dev/null @@ -1,3 +0,0 @@ -<--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -ODER -TEXT = <--$BLA-->.TEST(...) \ No newline at end of file diff --git a/test/samples/global_vars.rb b/test/samples/global_vars.rb deleted file mode 100644 index 8066d67d..00000000 --- a/test/samples/global_vars.rb +++ /dev/null @@ -1,13 +0,0 @@ -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE - -require 'coderay' - -tokens = CodeRay.scan code, :ruby -tokens.each_text_token { |text, kind| text.upcase! } -tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text } - -print tokens diff --git a/test/samples/global_vars2.expected b/test/samples/global_vars2.expected deleted file mode 100644 index 964cf504..00000000 --- a/test/samples/global_vars2.expected +++ /dev/null @@ -1,10 +0,0 @@ - - - - - -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) - - diff --git a/test/samples/global_vars2.rb b/test/samples/global_vars2.rb deleted file mode 100644 index 76468906..00000000 --- a/test/samples/global_vars2.rb +++ /dev/null @@ -1,28 +0,0 @@ -require 'coderay' -require 'erb' -include ERB::Util - -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE -puts < - - - - -HTML - -CodeRay.scan_stream code, :ruby do |text, kind| - next if text.is_a? Symbol - text = h(text) - text = '%s' % text if kind == :global_variable - print text -end - -puts < - -HTML diff --git a/test/samples/highlight.expected b/test/samples/highlight.expected deleted file mode 100644 index 6a9b2784..00000000 --- a/test/samples/highlight.expected +++ /dev/null @@ -1,175 +0,0 @@ -
- 
puts "Hello, World!"
-
- - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-
require 'coderay'
-
-puts CodeRay.highlight('puts "Hello, World!"', :ruby)
-
-output = CodeRay.highlight_file($0, :line_numbers => :table)
-puts <<HTML
-<html>
-<head>
-#{output.stylesheet true}
-<body>
-#{output}
-</body>
-</html>
-HTML
-
- - - diff --git a/test/samples/highlight.rb b/test/samples/highlight.rb deleted file mode 100644 index 846efa45..00000000 --- a/test/samples/highlight.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'coderay' - -puts CodeRay.highlight('puts "Hello, World!"', :ruby) - -output = CodeRay.highlight_file($0, :line_numbers => :table) -puts < - -#{output.stylesheet true} - -#{output} - - -HTML diff --git a/test/samples/html.expected b/test/samples/html.expected deleted file mode 100644 index e98d5897..00000000 --- a/test/samples/html.expected +++ /dev/null @@ -1,919 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-59
-60
-61
-62
-63
-64
-65
-66
-67
-68
-69
-70
-71
-72
-73
-74
-75
-76
-77
-78
-79
-80
-81
-82
-83
-84
-85
-86
-87
-88
-89
-90
-91
-92
-93
-94
-95
-96
-97
-98
-99
-100
-101
-102
-103
-104
-105
-106
-107
-108
-109
-110
-111
-112
-113
-114
-115
-116
-117
-118
-119
-120
-121
-122
-123
-124
-125
-126
-127
-128
-129
-130
-131
-132
-133
-134
-135
-136
-137
-138
-139
-140
-141
-142
-143
-144
-145
-146
-147
-148
-149
-150
-151
-152
-153
-154
-155
-156
-157
-158
-159
-160
-161
-162
-163
-164
-165
-166
-167
-168
-169
-170
-171
-172
-173
-174
-175
-176
-177
-178
-179
-180
-181
-182
-183
-184
-185
-186
-187
-188
-189
-190
-191
-192
-193
-194
-195
-196
-197
-198
-199
-200
-201
-202
-203
-204
-205
-206
-207
-208
-209
-210
-211
-212
-213
-214
-215
-216
-217
-218
-219
-220
-221
-222
-223
-224
-225
-226
-227
-228
-229
-230
-231
-232
-233
-234
-235
-236
-237
-238
-239
-240
-241
-242
-243
-244
-245
-246
-247
-248
-249
-250
-251
-252
-253
-254
-255
-256
-257
-258
-259
-260
-261
-262
-263
-264
-265
-266
-267
-268
-269
-270
-271
-272
-273
-274
-275
-276
-277
-278
-279
-280
-281
-282
-283
-284
-285
-286
-287
-288
-289
-290
-291
-292
-293
-294
-295
-296
-297
-298
-299
-300
-301
-302
-303
-304
-305
-306
-307
-308
-309
-310
-311
-312
-313
-314
-315
-316
-317
-318
-319
-320
-321
-322
-323
-324
-325
-326
-327
-328
-329
-330
-331
-332
-333
-334
-335
-336
-337
-338
-339
-340
-341
-342
-343
-344
-345
-346
-347
-348
-349
-350
-351
-352
-353
-354
-355
-356
-357
-358
-359
-360
-361
-362
-363
-364
-365
-366
-367
-368
-369
-370
-371
-372
-373
-374
-375
-376
-377
-378
-379
-380
-381
-382
-383
-384
-385
-
require 'scanner'
-
-module CodeRay
-  
-  class RubyScanner < Scanner
-    
-    RESERVED_WORDS = [
-      'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
-      'defined?', 'ensure', 'module', 'redo', 'super', 'until',
-      'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
-      'when', 'END', 'case', 'else', 'for', 'retry',
-      'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
-      'undef', 'yield',
-    ]
-
-    DEF_KEYWORDS = ['def']
-    MODULE_KEYWORDS = ['class', 'module']
-    DEF_NEW_STATE = WordList.new(:initial).
-      add(DEF_KEYWORDS, :def_expected).
-      add(MODULE_KEYWORDS, :module_expected)
-
-    WORDS_ALLOWING_REGEXP = [
-      'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
-    ]
-    REGEXP_ALLOWED = WordList.new(false).
-      add(WORDS_ALLOWING_REGEXP, :set)
-    
-    PREDEFINED_CONSTANTS = [
-      'nil', 'true', 'false', 'self',
-      'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
-    ]
-
-    IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant)
-
-    METHOD_NAME = / #{IDENT} [?!]? /xo
-    METHOD_NAME_EX = /
-     #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
-     | \*\*?         # multiplication and power
-     | [-+~]@?       # plus, minus
-     | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
-     | \[\]=?        # array getter and setter
-     | <=?>? | >=?   # comparison, rocket operator
-     | << | >>       # append or shift left, shift right
-     | ===?          # simple equality and case equality
-    /ox
-    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
-
-    DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
-    SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
-    STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
-    SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
-    REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
-    
-    DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
-    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
-    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
-    BINARY = /0b[01]+(?:_[01]+)*/
-
-    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
-    FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
-    INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
-    
-    def reset
-      super
-      @regexp_allowed = false
-    end
-    
-    def next_token
-      return if @scanner.eos?
-
-      kind = :error
-      if @scanner.scan(/\s+/)  # in every state
-        kind = :space
-        @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
-
-      elsif @state == :def_expected
-        if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
-          kind = :method
-          @state = :initial
-        else
-          @scanner.scan(/./)
-          kind = :error
-        end
-        @state = :initial
-        
-      elsif @state == :module_expected
-        if @scanner.scan(/<</)
-          kind = :operator
-        else
-          if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
-            kind = :method
-          else
-            @scanner.scan(/./)
-            kind = :error
-          end
-          @state = :initial
-        end
-        
-      elsif # state == :initial
-        # IDENTIFIERS, KEYWORDS
-        if @scanner.scan(GLOBAL_VARIABLE)
-          kind = :global_variable
-        elsif @scanner.scan(/ @@ #{IDENT} /ox)
-          kind = :class_variable
-        elsif @scanner.scan(/ @ #{IDENT} /ox)
-          kind = :instance_variable
-        elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
-          kind = :comment
-        elsif @scanner.scan(METHOD_NAME)
-          if @last_token_dot
-            kind = :ident
-          else
-            matched = @scanner.matched
-            kind = IDENT_KIND[matched]
-            if kind == :ident and matched =~ /^[A-Z]/
-              kind = :constant
-            elsif kind == :reserved
-              @state = DEF_NEW_STATE[matched]
-              @regexp_allowed = REGEXP_ALLOWED[matched]
-            end
-          end
-          
-        elsif @scanner.scan(STRING)
-          kind = :string
-        elsif @scanner.scan(SHELL)
-          kind = :shell
-        ## HEREDOCS
-        elsif @scanner.scan(/\//) and @regexp_allowed
-           @scanner.unscan
-           @scanner.scan(REGEXP)
-          kind = :regexp
-        ## %strings
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        elsif @scanner.scan(/
-          \? (?:
-            [^\s\\]
-          | 
-            \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
-          )
-        /ox)
-          kind = :integer
-          
-        elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
-          kind = :operator
-          @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
-        elsif @scanner.scan(FLOAT)
-          kind = :float
-        elsif @scanner.scan(INTEGER)
-          kind = :integer
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        else
-          @scanner.scan(/./m)
-        end
-      end
-      
-      token = Token.new @scanner.matched, kind
-
-      if kind == :regexp
-        token.text << @scanner.scan(/[eimnosux]*/)
-      end
-      
-      @regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
-
-      token
-    end
-  end
-  
-  ScannerList.register RubyScanner, 'ruby'
-
-end
-
-module CodeRay
-  require 'scanner'
-
-  class Highlighter
-
-    def initialize lang
-      @scanner = Scanner[lang].new
-    end
-
-    def highlight code
-      @scanner.feed code
-      @scanner.all_tokens.map { |t| t.inspect }.join "\n"
-    end
-
-  end
-
-  class HTMLHighlighter < Highlighter
-    
-    ClassOfKind = {
-      :attribute_name => 'an',
-      :attribute_name_fat => 'af',
-      :attribute_value => 'av',
-      :attribute_value_fat => 'aw',
-      :bin => 'bi',
-       :char => 'ch',
-      :class => 'cl',
-      :class_variable => 'cv',
-      :color => 'cr',
-      :comment => 'c',
-      :constant => 'co',
-      :definition => 'df',
-      :directive => 'di',
-      :doc => 'do',
-      :doc_string => 'ds',
-      :exception => 'ex',
-      :error => 'er',
-      :float => 'fl',
-      :function => 'fu',
-      :global_variable => 'gv',
-      :hex => 'hx',
-      :include => 'ic',
-      :instance_variable => 'iv',
-      :integer => 'i',
-      :interpreted => 'in',
-      :label => 'la',
-      :local_variable => 'lv',
-      :oct => 'oc',
-      :operator_name => 'on',
-      :pre_constant => 'pc',
-      :pre_type => 'pt',
-      :predefined => 'pd',
-      :preprocessor => 'pp',
-      :regexp => 'rx',
-      :reserved => 'r',
-      :shell => 'sh',
-      :string => 's',
-      :symbol => 'sy',
-      :tag => 'ta',
-      :tag_fat => 'tf',
-      :tag_special => 'ts',
-      :type => 'ty',
-      :variable => 'v',
-      :xml_text => 'xt',
-
-      :ident => :NO_HIGHLIGHT,
-      :operator => :NO_HIGHLIGHT,
-      :space => :NO_HIGHLIGHT,
-    }
-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
-    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
-    
-    def initialize lang, options = {}
-      super lang
-      
-      @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
-      case level = options.fetch(:level, 'xhtml')
-        when 'html'
-          @HTML_BR = "<BR>\n"
-        when 'xhtml'
-          @HTML_BR = "<br />\n"
-      else
-        raise "Unknown HTML level: #{level}"
-      end
-    end
-
-    def highlight code
-      @scanner.feed code
-      
-      out = ''
-      while t = @scanner.next_token
-        warn t.inspect if t.text.nil?
-        out << to_html(t)
-      end
-      TEMPLATE =~ /<%CONTENT%>/
-      $` + out + $'
-    end
-    
-  private
-    def to_html token
-      css_class = ClassOfKind[token.kind]
-      if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
-        warn "no token class found for :#{token.kind}"
-      end
-        
-      text = text_to_html token.text
-      if css_class == :NO_HIGHLIGHT
-        text
-      else
-        "<span class=\"#{css_class}\">#{text}</span>"
-      end
-    end
-    
-    def text_to_html text
-      return '' if text.empty?
-      text = text.dup  # important
-      if text.index(/["><&]/)
-        text.gsub!('&', '&amp;')
-        text.gsub!('"', '&quot;')
-        text.gsub!('>', '&gt;')
-        text.gsub!('<', '&lt;')
-      end
-      if text.index(/\s/)
-        text.gsub!("\n", @HTML_BR)
-        text.gsub!("\t", @HTML_TAB)
-        text.gsub!(/^ /, '&nbsp;')
-        text.gsub!('  ', ' &nbsp;')
-      end
-      text
-    end
-    
-    TEMPLATE = <<-'TEMPLATE'
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
-<html dir="ltr">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-<meta http-equiv="Content-Style-Type" content="text/css">
-
-<title>RubyBB BBCode</title>
-<style type="text/css">
-.code {
-  width: 100%;
-  background-color: #FAFAFA;
-  border: 1px solid #D1D7DC;
-  font-family: 'Courier New', 'Terminal', monospace;
-  font-size: 10pt;
-  color: black;
-  vertical-align: top;
-  text-align: left;
-}
-.code .af { color:#00C; }
-.code .an { color:#007; }
-.code .av { color:#700; }
-.code .aw { color:#C00; }
-.code .bi { color:#509; font-weight:bold; }
-.code .c  { color:#888; }
-.code .ch { color:#C28; font-weight:bold; }
-.code .cl { color:#B06; font-weight:bold; }
-.code .co { color:#036; font-weight:bold; }
-.code .cr { color:#0A0; }
-.code .cv { color:#369; }
-.code .df { color:#099; font-weight:bold; }
-.code .di { color:#088; font-weight:bold; }
-.code .do { color:#970; }
-.code .ds { color:#D42; font-weight:bold; }
-.code .er { color:#F00; background-color:#FAA; }
-.code .ex { color:#F00; font-weight:bold; }
-.code .fl { color:#60E; font-weight:bold; }
-.code .fu { color:#06B; font-weight:bold; }
-.code .gv { color:#800; font-weight:bold; }
-.code .hx { color:#058; font-weight:bold; }
-.code .i  { color:#00D; font-weight:bold; }
-.code .ic { color:#B44; font-weight:bold; }
-.code .in { color:#B2B; font-weight:bold; }
-.code .iv { color:#33B; }
-.code .la { color:#970; font-weight:bold; }
-.code .lv { color:#963; }
-.code .oc { color:#40E; font-weight:bold; }
-.code .on { color:#000; font-weight:bold; }
-.code .pc { color:#038; font-weight:bold; }
-.code .pd { color:#369; font-weight:bold; }
-.code .pp { color:#579; }
-.code .pt { color:#339; font-weight:bold; }
-.code .r  { color:#080; font-weight:bold; }
-.code .rx { color:#927; font-weight:bold; }
-.code .s  { color:#D42; font-weight:bold; }
-.code .sh { color:#B2B; font-weight:bold; }
-.code .sy { color:#A60; }
-.code .ta { color:#070; }
-.code .tf { color:#070; font-weight:bold; }
-.code .ts { color:#D70; font-weight:bold; }
-.code .ty { color:#339; font-weight:bold; }
-.code .v  { color:#036; }
-.code .xt { color:#444; }
-</style>
-</head>
-<body>
-<div class="code">
-<%CONTENT%>
-</div>
-<div class="validators">
-<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
-<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
-</div>    
-</body>
-</html>
-    TEMPLATE
-
-  end
-
-end
-
- - - diff --git a/test/samples/html.rb b/test/samples/html.rb deleted file mode 100644 index c18284a1..00000000 --- a/test/samples/html.rb +++ /dev/null @@ -1,394 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan DATA.read, :ruby -html = tokens.page(:tab_width => 2, :line_numbers => :table, :title => 'CodeRay HTML Encoder Example') - -puts html - -__END__ -require 'scanner' - -module CodeRay - - class RubyScanner < Scanner - - RESERVED_WORDS = [ - 'and', 'def', 'end', 'in', 'or', 'unless', 'begin', - 'defined?', 'ensure', 'module', 'redo', 'super', 'until', - 'BEGIN', 'break', 'do', 'next', 'rescue', 'then', - 'when', 'END', 'case', 'else', 'for', 'retry', - 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return', - 'undef', 'yield', - ] - - DEF_KEYWORDS = ['def'] - MODULE_KEYWORDS = ['class', 'module'] - DEF_NEW_STATE = WordList.new(:initial). - add(DEF_KEYWORDS, :def_expected). - add(MODULE_KEYWORDS, :module_expected) - - WORDS_ALLOWING_REGEXP = [ - 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when' - ] - REGEXP_ALLOWED = WordList.new(false). - add(WORDS_ALLOWING_REGEXP, :set) - - PREDEFINED_CONSTANTS = [ - 'nil', 'true', 'false', 'self', - 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__', - ] - - IDENT_KIND = WordList.new(:ident). - add(RESERVED_WORDS, :reserved). - add(PREDEFINED_CONSTANTS, :pre_constant) - - METHOD_NAME = / #{IDENT} [?!]? /xo - METHOD_NAME_EX = / - #{METHOD_NAME} # common methods: split, foo=, empty?, gsub! - | \*\*? # multiplication and power - | [-+~]@? # plus, minus - | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system` - | \[\]=? # array getter and setter - | <=?>? | >=? # comparison, rocket operator - | << | >> # append or shift left, shift right - | ===? # simple equality and case equality - /ox - GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox - - DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox - SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox - STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox - SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox - REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox - - DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error - OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ - HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ - BINARY = /0b[01]+(?:_[01]+)*/ - - EXPONENT = / [eE] [+-]? #{DECIMAL} /ox - FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) / - INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/ - - def reset - super - @regexp_allowed = false - end - - def next_token - return if @scanner.eos? - - kind = :error - if @scanner.scan(/\s+/) # in every state - kind = :space - @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting - - elsif @state == :def_expected - if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox) - kind = :method - @state = :initial - else - @scanner.scan(/./) - kind = :error - end - @state = :initial - - elsif @state == :module_expected - if @scanner.scan(/<;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x) - kind = :operator - @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/ - elsif @scanner.scan(FLOAT) - kind = :float - elsif @scanner.scan(INTEGER) - kind = :integer - elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox) - kind = :global_variable - else - @scanner.scan(/./m) - end - end - - token = Token.new @scanner.matched, kind - - if kind == :regexp - token.text << @scanner.scan(/[eimnosux]*/) - end - - @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting - - token - end - end - - ScannerList.register RubyScanner, 'ruby' - -end - -module CodeRay - require 'scanner' - - class Highlighter - - def initialize lang - @scanner = Scanner[lang].new - end - - def highlight code - @scanner.feed code - @scanner.all_tokens.map { |t| t.inspect }.join "\n" - end - - end - - class HTMLHighlighter < Highlighter - - ClassOfKind = { - :attribute_name => 'an', - :attribute_name_fat => 'af', - :attribute_value => 'av', - :attribute_value_fat => 'aw', - :bin => 'bi', - :char => 'ch', - :class => 'cl', - :class_variable => 'cv', - :color => 'cr', - :comment => 'c', - :constant => 'co', - :definition => 'df', - :directive => 'di', - :doc => 'do', - :doc_string => 'ds', - :exception => 'ex', - :error => 'er', - :float => 'fl', - :function => 'fu', - :global_variable => 'gv', - :hex => 'hx', - :include => 'ic', - :instance_variable => 'iv', - :integer => 'i', - :interpreted => 'in', - :label => 'la', - :local_variable => 'lv', - :oct => 'oc', - :operator_name => 'on', - :pre_constant => 'pc', - :pre_type => 'pt', - :predefined => 'pd', - :preprocessor => 'pp', - :regexp => 'rx', - :reserved => 'r', - :shell => 'sh', - :string => 's', - :symbol => 'sy', - :tag => 'ta', - :tag_fat => 'tf', - :tag_special => 'ts', - :type => 'ty', - :variable => 'v', - :xml_text => 'xt', - - :ident => :NO_HIGHLIGHT, - :operator => :NO_HIGHLIGHT, - :space => :NO_HIGHLIGHT, - } - ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function] - ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' - - def initialize lang, options = {} - super lang - - @HTML_TAB = ' ' * options.fetch(:tabs2space, 8) - case level = options.fetch(:level, 'xhtml') - when 'html' - @HTML_BR = "
\n" - when 'xhtml' - @HTML_BR = "
\n" - else - raise "Unknown HTML level: #{level}" - end - end - - def highlight code - @scanner.feed code - - out = '' - while t = @scanner.next_token - warn t.inspect if t.text.nil? - out << to_html(t) - end - TEMPLATE =~ /<%CONTENT%>/ - $` + out + $' - end - - private - def to_html token - css_class = ClassOfKind[token.kind] - if defined? ::DEBUG and not ClassOfKind.has_key? token.kind - warn "no token class found for :#{token.kind}" - end - - text = text_to_html token.text - if css_class == :NO_HIGHLIGHT - text - else - "#{text}" - end - end - - def text_to_html text - return '' if text.empty? - text = text.dup # important - if text.index(/["><&]/) - text.gsub!('&', '&') - text.gsub!('"', '"') - text.gsub!('>', '>') - text.gsub!('<', '<') - end - if text.index(/\s/) - text.gsub!("\n", @HTML_BR) - text.gsub!("\t", @HTML_TAB) - text.gsub!(/^ /, ' ') - text.gsub!(' ', '  ') - end - text - end - - TEMPLATE = <<-'TEMPLATE' - - - - - - -RubyBB BBCode - - - -
-<%CONTENT%> -
-
-Valid HTML 4.01! -Valid CSS! -
- - - TEMPLATE - - end - -end diff --git a/test/samples/html2.expected b/test/samples/html2.expected deleted file mode 100644 index c8ae56a7..00000000 --- a/test/samples/html2.expected +++ /dev/null @@ -1,185 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-
require 'coderay'
-
-# scan this file
-tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-
-# output it with two styles of line numbers
-out = tokens.div(:line_numbers => :table)
-out << '<hr />'
-out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-
-puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- 
 8 require 'coderay'
- 9 
-10 # scan this file
-11 tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-12 
-13 # output it with two styles of line numbers
-14 out = tokens.div(:line_numbers => :table)
-15 out << '<hr />'
-16 out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-17 
-18 puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- - - diff --git a/test/samples/html2.rb b/test/samples/html2.rb deleted file mode 100644 index 618d168d..00000000 --- a/test/samples/html2.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'coderay' - -# scan this file -tokens = CodeRay.scan(File.read($0) * 1, :ruby) - -# output it with two styles of line numbers -out = tokens.div(:line_numbers => :table) -out << '
' -out << tokens.div(:line_numbers => :inline, :line_number_start => 8) - -puts out.page(:title => 'CodeRay HTML Encoder Example') diff --git a/test/samples/html_list.expected b/test/samples/html_list.expected deleted file mode 100644 index a4092c8d..00000000 --- a/test/samples/html_list.expected +++ /dev/null @@ -1,160 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - -
- 
-1 $: << '..'
- 0 require 'coderay'
- 1 
- 2 tokens = CodeRay.scan File.read(__FILE__), :ruby
- 3 html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)
- 4 
- 5 puts html.page(:title => 'CodeRay HTML Encoder Example')
- 6 
- 7 commment = <<_
- 8 This code must be > 10 lines
- 9 because I want to test the correct adjustment of the line numbers.
-10 _
-
-
- - - diff --git a/test/samples/html_list.rb b/test/samples/html_list.rb deleted file mode 100644 index fdfa5123..00000000 --- a/test/samples/html_list.rb +++ /dev/null @@ -1,12 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan File.read(__FILE__), :ruby -html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1) - -puts html.page(:title => 'CodeRay HTML Encoder Example') - -commment = <<_ -This code must be > 10 lines -because I want to test the correct adjustment of the line numbers. -_ diff --git a/test/samples/load_encoder.expected b/test/samples/load_encoder.expected deleted file mode 100644 index 1cff356d..00000000 --- a/test/samples/load_encoder.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::YAML is not defined; you must load it first. -Now it is loaded: CodeRay::Encoders::YAML -See? -Require is also possible: CodeRay::Encoders::Tokens -See? -Now load some mapped encoders: stats and plain. -Require all Encoders: -[[:count, CodeRay::Encoders::Count], [:debug, CodeRay::Encoders::Debug], [:div, CodeRay::Encoders::Div], [:html, CodeRay::Encoders::HTML], [:null, CodeRay::Encoders::Null], [:page, CodeRay::Encoders::Page], [:plain, :text], [:span, CodeRay::Encoders::Span], [:statistic, CodeRay::Encoders::Statistic], [:stats, CodeRay::Encoders::Statistic], [:text, CodeRay::Encoders::Text], [:tokens, CodeRay::Encoders::Tokens], [:xml, CodeRay::Encoders::XML], [:yaml, CodeRay::Encoders::YAML]] diff --git a/test/samples/load_encoder.rb b/test/samples/load_encoder.rb deleted file mode 100644 index 9594bfa1..00000000 --- a/test/samples/load_encoder.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Encoders::YAML -rescue - puts 'CodeRay::Encoders::YAML is not defined; you must load it first.' -end - -yaml_encoder = CodeRay::Encoders[:yaml] -print 'Now it is loaded: ' -p yaml_encoder -puts 'See?' - -tokens_encoder = CodeRay.require_plugin 'CodeRay::Encoders/tokens' -print 'Require is also possible: ' -p tokens_encoder -puts 'See?' - -puts 'Now load some mapped encoders: stats and plain.' -CodeRay.require_plugin 'CodeRay::Encoders/stats' -CodeRay.require_plugin 'CodeRay::Encoders/plain' - -puts 'Require all Encoders:' -CodeRay::Encoders.load_all -p CodeRay::Encoders.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/load_scanner.expected b/test/samples/load_scanner.expected deleted file mode 100644 index a2d200d7..00000000 --- a/test/samples/load_scanner.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::Ruby is not defined; you must load it first. -Now it is loaded: CodeRay::Scanners::Ruby -See? -Require is also possible: CodeRay::Scanners::C -See? -Now load some mapped scanners: cpp and plain. -Require all Scanners: -[[nil, :plain], [:c, CodeRay::Scanners::C], [:cpp, :c], [:delphi, CodeRay::Scanners::Delphi], [:html, CodeRay::Scanners::HTML], [:irb, :ruby], [:nitro, :nitro_xhtml], [:nitro_xhtml, CodeRay::Scanners::NitroXHTML], [:pascal, :delphi], [:plain, CodeRay::Scanners::Plaintext], [:plaintext, CodeRay::Scanners::Plaintext], [:rhtml, CodeRay::Scanners::RHTML], [:ruby, CodeRay::Scanners::Ruby], [:xhtml, :nitro_xhtml], [:xml, :html]] diff --git a/test/samples/load_scanner.rb b/test/samples/load_scanner.rb deleted file mode 100644 index 23be8a29..00000000 --- a/test/samples/load_scanner.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Scanners::Ruby -rescue - puts 'CodeRay::Encoders::Ruby is not defined; you must load it first.' -end - -ruby_scanner = CodeRay::Scanners[:ruby] -print 'Now it is loaded: ' -p ruby_scanner -puts 'See?' - -c_scanner = CodeRay.require_plugin 'CodeRay::Scanners/c' -print 'Require is also possible: ' -p c_scanner -puts 'See?' - -puts 'Now load some mapped scanners: cpp and plain.' -CodeRay.require_plugin 'CodeRay::Scanners/cpp' -CodeRay.require_plugin 'CodeRay::Scanners/plain' - -puts 'Require all Scanners:' -CodeRay::Scanners.load_all -p CodeRay::Scanners.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/more.expected b/test/samples/more.expected deleted file mode 100644 index 196904d8..00000000 --- a/test/samples/more.expected +++ /dev/null @@ -1,2 +0,0 @@ -Input: 4983B, Output: 23484B -Take a look with your browser. diff --git a/test/samples/more.rb b/test/samples/more.rb deleted file mode 100644 index 0db7ba47..00000000 --- a/test/samples/more.rb +++ /dev/null @@ -1,205 +0,0 @@ -require 'coderay' - -c, ruby = DATA.read.split(/^---$/) -DATA.rewind -me = DATA.read[/.*^__END__$/m] -$input = c + ruby + me - -require 'benchmark' -time = Benchmark.realtime do - - # here CodeRay comes to play - hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div) - c = hl.highlight c, :c - ruby = hl.highlight ruby, :ruby - me = hl.highlight me, :ruby - - body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code| - "

#{title}

\n#{code}" - end.join - body = hl.class::Output.new(body, hl.css, :div).page! - - # CodeRay also provides a simple page generator - $output = body #hl.class.wrap_in_page body -end - -File.open('test.html', 'w') do |f| - f.write $output -end -puts 'Input: %dB, Output: %dB' % [$input.size, $output.size] -#puts 'Created "test.html" in %0.3f seconds (%d KB/s).' % [time, $input.size / 1024.0 / time] -puts 'Take a look with your browser.' - -__END__ -/********************************************************************** - - version.c - - - $Author: nobu $ - $Date: 2004/03/25 12:01:40 $ - created at: Thu Sep 30 20:08:01 JST 1993 - - Copyright (C) 1993-2003 Yukihiro Matsumoto - -**********************************************************************/ - -#include "ruby.h" -#include "version.h" -#include - -const char ruby_version[] = RUBY_VERSION; -const char ruby_release_date[] = RUBY_RELEASE_DATE; -const char ruby_platform[] = RUBY_PLATFORM; - -void -Init_version() -{ - VALUE v = rb_obj_freeze(rb_str_new2(ruby_version)); - VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date)); - VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform)); - - rb_define_global_const("RUBY_VERSION", v); - rb_define_global_const("RUBY_RELEASE_DATE", d); - rb_define_global_const("RUBY_PLATFORM", p); -} - -void -ruby_show_version() -{ - printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM); -} - -void -ruby_show_copyright() -{ - printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR); - exit(0); -} ---- -# -# = ostruct.rb: OpenStruct implementation -# -# Author:: Yukihiro Matsumoto -# Documentation:: Gavin Sinclair -# -# OpenStruct allows the creation of data objects with arbitrary attributes. -# See OpenStruct for an example. -# - -# -# OpenStruct allows you to create data objects and set arbitrary attributes. -# For example: -# -# require 'ostruct' -# -# record = OpenStruct.new -# record.name = "John Smith" -# record.age = 70 -# record.pension = 300 -# -# puts record.name # -> "John Smith" -# puts record.address # -> nil -# -# It is like a hash with a different way to access the data. In fact, it is -# implemented with a hash, and you can initialize it with one. -# -# hash = { "country" => "Australia", :population => 20_000_000 } -# data = OpenStruct.new(hash) -# -# p data # -> -# -class OpenStruct - # - # Create a new OpenStruct object. The optional +hash+, if given, will - # generate attributes and values. For example. - # - # require 'ostruct' - # hash = { "country" => "Australia", :population => 20_000_000 } - # data = OpenStruct.new(hash) - # - # p data # -> - # - # By default, the resulting OpenStruct object will have no attributes. - # - def initialize(hash=nil) - @table = {} - if hash - for k,v in hash - @table[k.to_sym] = v - new_ostruct_member(k) - end - end - end - - # Duplicate an OpenStruct object members. - def initialize_copy(orig) - super - @table = @table.dup - end - - def marshal_dump - @table - end - def marshal_load(x) - @table = x - @table.each_key{|key| new_ostruct_member(key)} - end - - def new_ostruct_member(name) - unless self.respond_to?(name) - self.instance_eval %{ - def #{name}; @table[:#{name}]; end - def #{name}=(x); @table[:#{name}] = x; end - } - end - end - - def method_missing(mid, *args) # :nodoc: - mname = mid.id2name - len = args.length - if mname =~ /=$/ - if len != 1 - raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1) - end - if self.frozen? - raise TypeError, "can't modify frozen #{self.class}", caller(1) - end - mname.chop! - @table[mname.intern] = args[0] - self.new_ostruct_member(mname) - elsif len == 0 - @table[mid] - else - raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1) - end - end - - # - # Remove the named field from the object. - # - def delete_field(name) - @table.delete name.to_sym - end - - # - # Returns a string containing a detailed summary of the keys and values. - # - def inspect - str = "<#{self.class}" - for k,v in @table - str << " #{k}=#{v.inspect}" - end - str << ">" - end - - attr_reader :table # :nodoc: - protected :table - - # - # Compare this object and +other+ for equality. - # - def ==(other) - return false unless(other.kind_of?(OpenStruct)) - return @table == other.table - end -end diff --git a/test/samples/scanner.expected b/test/samples/scanner.expected deleted file mode 100644 index 5015168f..00000000 --- a/test/samples/scanner.expected +++ /dev/null @@ -1,16 +0,0 @@ -C Code: if (*p == '{') nest++; - -> print only operators: -(*==)++; ------------------------------- - -Ruby Code: ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee) - -> has a string? -false - -> number of regexps? -2 - -> has a string? -"ruby_code" (ident), "(" (operator), ":can" (symbol), "," (operator), " " (space), "BE" (constant), "," (operator), " " (space), "%r[" (delimiter), "q" (content), "[" (nesting_delimiter), "ui" (content), "]" (nesting_delimiter), "te " (content), "#{" (inline_delimiter), " " (space), "/" (delimiter), "comple" (content), "/" (delimiter), "x" (modifier), " " (space), "}" (inline_delimiter), "," (content), "]" (delimiter), " " (space), "=" (operator), ">" (operator), " " (space), "$-s" (global_variable), "," (operator), " " (space), "&" (operator), "?\xee" (integer), ")" (operator) diff --git a/test/samples/scanner.rb b/test/samples/scanner.rb deleted file mode 100644 index 6a0245ea..00000000 --- a/test/samples/scanner.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'coderay' - -c_code = "if (*p == '{') nest++;" -puts 'C Code: ' + c_code -puts - -c_scanner = CodeRay::Scanners[:c].new c_code - -puts '> print only operators:' -for text, kind in c_scanner - print text if kind == :operator -end -puts -puts '-' * 30 -puts - -ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)! -puts 'Ruby Code: ' + ruby_code -puts - -ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code - -puts '> has a string?' -puts ruby_scanner. - any? { |text, kind| kind == :string } -puts - -puts '> number of regexps?' -puts ruby_scanner. - select { |token| token == [:open, :regexp] }.size -puts - -puts '> has a string?' -puts ruby_scanner. - reject { |text, kind| not text.is_a? String }. - map { |text, kind| %("#{text}" (#{kind})) }.join(', ') diff --git a/test/samples/server.rb b/test/samples/server.rb deleted file mode 100644 index ccdff324..00000000 --- a/test/samples/server.rb +++ /dev/null @@ -1,110 +0,0 @@ -# CodeRay dynamic highlighter - -unless ARGV.grep(/-[hv]|--(help|version)/).empty? - puts <<-USAGE -CodeRay Server 0.5 -$Id: demo_server.rb 113 2006-03-15 23:24:37Z murphy $ - -Usage: - 1) Start this and your browser. - 2) Go to http://localhost:2468/? - and you should get the highlighted version. - -Parameters: - -d Debug mode; reload CodeRay engine for every file. - (prepare for MANY "already initialized" and "method redefined" - messages - ingore it.) - - ... More to come. - USAGE - exit -end - -require 'webrick' -require 'pathname' - -class << File - alias dir? directory? -end - -require 'erb' -include ERB::Util -def url_decode s - s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' } -end - -class String - def to_link name = File.basename(self) - "#{name}" - end -end - -require 'coderay' -class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet - - STYLE = 'style="font-family: sans-serif; color: navy;"' - BANNER = '

Highlighted by CodeRay

' - - def do_GET req, res - q = req.query_string || '' - args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) } - path = args.fetch 'path', '.' - - backlinks = '

current path: %s
' % html_escape(path) + - (Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' + - '.'.to_link('current') + '

' - - res.body = - if File.dir? path - path = Pathname.new(path).cleanpath.to_s - dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p } - - page = "" - page << backlinks - - page << '
' - page << "
Directories
\n" + dirs.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
Files
\n" + files.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
\n" - page << "#{BANNER}" - - elsif File.exist? path - if $DEBUG - $".delete_if { |f| f =~ /coderay/ } - require 'coderay' - end - div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div, :hint => :info - div.replace <<-DIV -
- #{backlinks} -#{div} -
- #{BANNER} - DIV - div.page - end - - res['Content-Type'] = 'text/html' - end -end - -# This port is taken by "qip_msgd" - I don't know that. Do you? -module CodeRay - PORT = 0xC0DE / 20 -end - -server = WEBrick::HTTPServer.new :Port => CodeRay::PORT - -server.mount '/', CodeRayServlet - -server.mount_proc '/version' do |req, res| - res.body = 'CodeRay::Version = ' + CodeRay::Version - res['Content-Type'] = "text/plain" -end - -trap("INT") { server.shutdown } -server.start diff --git a/test/samples/simple.expected b/test/samples/simple.expected deleted file mode 100644 index b3d78753..00000000 --- a/test/samples/simple.expected +++ /dev/null @@ -1 +0,0 @@ -puts 'Hello, world!' diff --git a/test/samples/simple.rb b/test/samples/simple.rb deleted file mode 100644 index a3129b01..00000000 --- a/test/samples/simple.rb +++ /dev/null @@ -1,10 +0,0 @@ - -# Load CodeRay -# If this doesn't work, try ruby -rubygems. -require 'coderay' - -# Generate HTML page for Ruby code. -page = CodeRay.scan("puts 'Hello, world!'", :ruby).span - -# Print it -puts page diff --git a/test/samples/stream.rb b/test/samples/stream.rb deleted file mode 100644 index 7ed8a22b..00000000 --- a/test/samples/stream.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -code = File.read($0) * 500 -puts "Size of code: %d KB" % [code.size / 1024] - -puts "Use your system's memory tracker to see how much RAM this takes." -print 'Press some key to continue...'; gets - -require 'benchmark' -e = CodeRay.encoder(:div) -for do_stream in [true, false] - puts "Scanning and encoding in %s mode, please wait..." % - [do_stream ? 'streaming' : 'normal'] - output = '' - time = Benchmark.realtime do - if do_stream - output = e.encode_stream(code, :ruby) - else - output = e.encode_tokens(t = CodeRay.scan(code, :ruby)) - end - end - puts 'Finished after %4.2f seconds.' % time - puts "Size of output: %d KB" % [output.size / 1024] - print 'Press some key to continue...'; gets -end diff --git a/test/samples/stream2.expected b/test/samples/stream2.expected deleted file mode 100644 index 83aee987..00000000 --- a/test/samples/stream2.expected +++ /dev/null @@ -1,2 +0,0 @@ -kind: regexp, text size: 5. -kind: space, text size: 1. diff --git a/test/samples/stream2.rb b/test/samples/stream2.rb deleted file mode 100644 index d43cc9ad..00000000 --- a/test/samples/stream2.rb +++ /dev/null @@ -1,8 +0,0 @@ -require 'coderay' - -token_stream = CodeRay::TokenStream.new do |kind, text| - puts 'kind: %s, text size: %d.' % [kind, text.size] -end - -token_stream << [:regexp, '/\d+/'] << [:space, "\n"] -#-> kind: rexpexp, text size: 5. diff --git a/test/samples/suite.rb b/test/samples/suite.rb deleted file mode 100644 index cfe53c0f..00000000 --- a/test/samples/suite.rb +++ /dev/null @@ -1,86 +0,0 @@ -mydir = File.dirname(__FILE__) -$:.unshift mydir + '/../../lib/' - -$VERBOSE = true - -require 'test/unit' -include Test::Unit - -class CodeRaySuite < TestCase - - def self.dir &block - @dir ||= File.dirname(__FILE__) - if block - Dir.chdir @dir, &block - end - @dir - end - - def dir &block - self.class.dir(&block) - end - - def test_ALL - dir do - for input in Dir["*.rb"] - %w(server.rb stream.rb suite.rb) - next if input[/^load_/] - puts "[ testing #{input}... ]" - name = File.basename(input, ".rb") - output = name + '.expected' - code = File.open(input, 'rb') { |f| break f.read } - - result = `ruby -wI../../lib #{input}` - - diff = output.sub '.expected', '.diff' - File.delete diff if File.exist? diff - computed = output.sub '.expected', '.actual' - if File.exist? output - expected = File.read output - ok = expected == result - unless ok - File.open(computed, 'w') { |f| f.write result } - `diff #{output} #{computed} > #{diff}` - puts "Test failed; output written to #{diff}." - end - assert(ok, "Output error: #{computed} != #{output}") - else - File.open(output, 'w') do |f| f.write result end - puts "New test: #{output}" - end - - end - end - end - -end - -require 'test/unit/testsuite' -$suite = TestSuite.new 'CodeRay Demos Test' -$suite << CodeRaySuite.suite - -def load_suite name - begin - require name + '/suite.rb' - rescue LoadError - $stderr.puts <<-ERR - -!! Folder #{File.split(__FILE__).first + '/' + name} not found - - ERR - false - end -end - -if subsuite = ARGV.find { |a| break $1 if a[/^([^-].*)/] } - load_suite(subsuite) or exit -else - Dir[mydir + '/*/'].each { |suite| load_suite suite } -end - -if ARGV.include? '-f' - require 'test/unit/ui/fox/testrunner' - UI::Fox::TestRunner.run $suite -else - require 'test/unit/ui/console/testrunner' - UI::Console::TestRunner.run $suite -end diff --git a/test/samples/tokens.expected b/test/samples/tokens.expected deleted file mode 100644 index 747904e5..00000000 --- a/test/samples/tokens.expected +++ /dev/null @@ -1 +0,0 @@ -[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]] diff --git a/test/samples/tokens.rb b/test/samples/tokens.rb deleted file mode 100644 index 91b8abbf..00000000 --- a/test/samples/tokens.rb +++ /dev/null @@ -1,3 +0,0 @@ -require 'coderay' - -p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby)