diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..c76072a8 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,12 @@ +jobs: + build: + docker: + - image: cimg/ruby:3.3.5 + environment: + RAILS_ENV: test + steps: + - checkout + - run: | + bundle install + - run: | + bundle exec rake test diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 00000000..c01311f6 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,26 @@ +version: "2" +checks: + argument-count: + enabled: false + complex-logic: + enabled: false + file-lines: + enabled: false + identical-code: + enabled: false + method-complexity: + enabled: false + method-count: + enabled: false + method-lines: + enabled: false + nested-control-flow: + enabled: false + return-statements: + enabled: false + similar-code: + enabled: false +plugins: + rubocop: + enabled: true + channel: rubocop-0-76 diff --git a/.gitignore b/.gitignore index dd001c8a..4d962c0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,12 @@ -.DS_Store -.*~ +.* +bench/example.* coverage -pkg -spec/reports doc Gemfile.lock -.rvmrc -.ruby-gemset -.ruby-version +old-stuff +pkg +spec/examples.txt +spec/reports test/executable/source.rb.html test/executable/source.rb.json test/scanners -bench/test.div.html -old-stuff diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..e248a433 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,32 @@ +inherit_from: .rubocop_todo.yml + +AllCops: + TargetRubyVersion: 2.3 + DisplayStyleGuide: true + Exclude: + - 'test/scanners/**/*' + - 'bench/example.ruby' + - 'old-stuff/**/*' + - 'test/lib/**/*' + +Gemspec/RequiredRubyVersion: + Enabled: false + +Gemspec/DuplicatedAssignment: + Enabled: false + +Layout/AccessModifierIndentation: + Enabled: false + +Layout/AlignArguments: + Enabled: false + +Layout/AlignArray: + Enabled: false + +Layout/AlignHash: + Enabled: false + +Layout/SpaceInsideBlockBraces: + EnforcedStyle: space + EnforcedStyleForEmptyBraces: space diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml new file mode 100644 index 00000000..2b0c3708 --- /dev/null +++ b/.rubocop_todo.yml @@ -0,0 +1,1183 @@ +# This configuration was generated by +# `rubocop --auto-gen-config` +# on 2019-11-24 03:18:41 +0100 using RuboCop version 0.76.0. +# The point is for the user to remove these configuration records +# one by one as the offenses are removed from the code base. +# Note that changes in the inspected code, or installation of new +# versions of RuboCop, may require this file to be generated again. + +# Offense count: 26 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentOneStep, IndentationWidth. +# SupportedStyles: case, end +Layout/CaseIndentation: + Exclude: + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +Layout/CommentIndentation: + Exclude: + - 'bin/coderay' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/lua.rb' + +# Offense count: 82 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: leading, trailing +Layout/DotPosition: + Enabled: false + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: AllowBorderComment, AllowMarginComment. +Layout/EmptyComment: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 30 +# Cop supports --auto-correct. +Layout/EmptyLineAfterGuardClause: + Exclude: + - 'lib/coderay/encoders/debug_lint.rb' + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/encoders/lint.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'lib/coderay/scanners/yaml.rb' + - 'lib/coderay/tokens.rb' + - 'rake_tasks/generator.rake' + - 'rake_tasks/test.rake' + +# Offense count: 6 +# Cop supports --auto-correct. +Layout/EmptyLineAfterMagicComment: + Exclude: + - 'lib/coderay.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + - 'test/functional/basic.rb' + +# Offense count: 6 +# Cop supports --auto-correct. +# Configuration parameters: AllowAdjacentOneLineDefs, NumberOfEmptyLines. +Layout/EmptyLineBetweenDefs: + Exclude: + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/tokens.rb' + +# Offense count: 9 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: around, only_before +Layout/EmptyLinesAroundAccessModifier: + Exclude: + - 'lib/coderay/encoders/filter.rb' + - 'lib/coderay/encoders/json.rb' + - 'lib/coderay/encoders/text.rb' + - 'lib/coderay/encoders/token_kind_filter.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/encoders/yaml.rb' + +# Offense count: 18 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines, beginning_only, ending_only +Layout/EmptyLinesAroundClassBody: + Exclude: + - 'lib/coderay/duo.rb' + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/delphi.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/xml.rb' + - 'rake_tasks/code_statistics.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Layout/EmptyLinesAroundMethodBody: + Exclude: + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/java.rb' + +# Offense count: 16 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: empty_lines, empty_lines_except_namespace, empty_lines_special, no_empty_lines +Layout/EmptyLinesAroundModuleBody: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/delphi.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/xml.rb' + - 'lib/coderay/styles.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleAlignWith, AutoCorrect, Severity. +# SupportedStylesAlignWith: keyword, variable, start_of_line +Layout/EndAlignment: + Exclude: + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 2 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: native, lf, crlf +Layout/EndOfLine: + Exclude: + - 'rake_tasks/documentation.rake' + - 'rake_tasks/statistic.rake' + +# Offense count: 140 +# Cop supports --auto-correct. +# Configuration parameters: AllowForAlignment, AllowBeforeTrailingComments, ForceEqualSignAlignment. +Layout/ExtraSpacing: + Enabled: false + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: IndentationWidth. +# SupportedStyles: special_inside_parentheses, consistent, align_brackets +Layout/IndentFirstArrayElement: + EnforcedStyle: consistent + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: IndentationWidth. +# SupportedStyles: special_inside_parentheses, consistent, align_braces +Layout/IndentFirstHashElement: + EnforcedStyle: consistent + +# Offense count: 48 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: squiggly, active_support, powerpack, unindent +Layout/IndentHeredoc: + Enabled: false + +# Offense count: 53 +# Cop supports --auto-correct. +# Configuration parameters: Width, IgnoredPatterns. +Layout/IndentationWidth: + Enabled: false + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: AllowDoxygenCommentStyle. +Layout/LeadingCommentSpace: + Exclude: + - 'lib/coderay/scanners/html.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: symmetrical, new_line, same_line +Layout/MultilineArrayBraceLayout: + Exclude: + - 'lib/coderay/scanners/lua.rb' + +# Offense count: 78 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented, indented_relative_to_receiver +Layout/MultilineMethodCallIndentation: + Enabled: false + +# Offense count: 9 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, IndentationWidth. +# SupportedStyles: aligned, indented +Layout/MultilineOperationIndentation: + Exclude: + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/encoders/token_kind_filter.rb' + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'test/functional/basic.rb' + +# Offense count: 17 +# Cop supports --auto-correct. +Layout/SpaceAfterComma: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + - 'lib/coderay/token_kinds.rb' + +# Offense count: 37 +# Cop supports --auto-correct. +# Configuration parameters: AllowForAlignment. +Layout/SpaceAroundOperators: + AllowForAlignment: true + +# Offense count: 2 +# Cop supports --auto-correct. +Layout/SpaceBeforeComment: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/token_kinds.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Layout/SpaceBeforeSemicolon: + Exclude: + - 'lib/coderay/scanners/diff.rb' + +# Offense count: 17 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces. +# SupportedStyles: space, no_space, compact +# SupportedStylesForEmptyBraces: space, no_space +Layout/SpaceInsideHashLiteralBraces: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'lib/coderay/styles/style.rb' + - 'test/unit/json_encoder.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: space, no_space +Layout/SpaceInsideParens: + Exclude: + - 'rake_tasks/code_statistics.rb' + +# Offense count: 32 +# Cop supports --auto-correct. +Layout/SpaceInsidePercentLiteralDelimiters: + Exclude: + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/sql.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Layout/SpaceInsideRangeLiteral: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/encoders/html/numbering.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBrackets. +# SupportedStyles: space, no_space +# SupportedStylesForEmptyBrackets: space, no_space +Layout/SpaceInsideReferenceBrackets: + Exclude: + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 6 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: space, no_space +Layout/SpaceInsideStringInterpolation: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 12 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: final_newline, final_blank_line +Layout/TrailingBlankLines: + Exclude: + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'test/executable/source.rb' + - 'test/functional/for_redcloth.rb' + - 'test/unit/comment_filter.rb' + - 'test/unit/count.rb' + - 'test/unit/json_encoder.rb' + - 'test/unit/lines_of_code.rb' + - 'test/unit/null.rb' + - 'test/unit/statistic.rb' + - 'test/unit/text.rb' + - 'test/unit/tokens.rb' + +# Offense count: 1680 +# Cop supports --auto-correct. +# Configuration parameters: AllowInHeredoc. +Layout/TrailingWhitespace: + Enabled: false + +# Offense count: 485 +# Configuration parameters: AllowSafeAssignment. +Lint/AssignmentInCondition: + Enabled: false + +# Offense count: 2 +# Configuration parameters: AllowComments. +Lint/HandleExceptions: + Exclude: + - 'bin/coderay' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 2 +Lint/IneffectiveAccessModifier: + Exclude: + - 'lib/coderay/encoders/html.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: runtime_error, standard_error +Lint/InheritException: + Exclude: + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 3 +Lint/LiteralAsCondition: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/scanners/haml.rb' + - 'test/executable/suite.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Lint/LiteralInInterpolation: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 2 +Lint/Loop: + Exclude: + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/tokens.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Lint/RedundantSplatExpansion: + Exclude: + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Lint/SendWithMixinArgument: + Exclude: + - 'lib/coderay/for_redcloth.rb' + +# Offense count: 1 +# Configuration parameters: IgnoreImplicitReferences. +Lint/ShadowedArgument: + Exclude: + - 'lib/coderay/for_redcloth.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments. +Lint/UnusedBlockArgument: + Exclude: + - 'lib/coderay/encoders/statistic.rb' + - 'lib/coderay/scanners/diff.rb' + - 'rake_tasks/code_statistics.rb' + +# Offense count: 38 +# Cop supports --auto-correct. +# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods. +Lint/UnusedMethodArgument: + Enabled: false + +# Offense count: 8 +Lint/UselessAssignment: + Exclude: + - 'lib/coderay/scanners/sql.rb' + - 'lib/coderay/scanners/yaml.rb' + - 'rake_tasks/code_statistics.rb' + - 'test/executable/suite.rb' + +# Offense count: 7 +# Configuration parameters: CheckForMethodsWithNoSideEffects. +Lint/Void: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 49 +Metrics/AbcSize: + Max: 361 + +# Offense count: 5 +# Configuration parameters: CountComments, ExcludedMethods. +# ExcludedMethods: refine +Metrics/BlockLength: + Max: 71 + +# Offense count: 183 +# Configuration parameters: CountBlocks. +Metrics/BlockNesting: + Max: 8 + +# Offense count: 27 +# Configuration parameters: CountComments. +Metrics/ClassLength: + Max: 380 + +# Offense count: 34 +Metrics/CyclomaticComplexity: + Max: 148 + +# Offense count: 63 +# Configuration parameters: CountComments, ExcludedMethods. +Metrics/MethodLength: + Max: 366 + +# Offense count: 5 +# Configuration parameters: CountComments. +Metrics/ModuleLength: + Max: 410 + +# Offense count: 34 +Metrics/PerceivedComplexity: + Max: 161 + +# Offense count: 2 +Naming/AccessorMethodName: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 24 +Naming/ConstantName: + Exclude: + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/java/builtin_types.rb' + +# Offense count: 1 +# Configuration parameters: ExpectMatchingDefinition, Regex, IgnoreExecutableScripts, AllowedAcronyms. +# AllowedAcronyms: CLI, DSL, ACL, API, ASCII, CPU, CSS, DNS, EOF, GUID, HTML, HTTP, HTTPS, ID, IP, JSON, LHS, QPS, RAM, RHS, RPC, SLA, SMTP, SQL, SSH, TCP, TLS, TTL, UDP, UI, UID, UUID, URI, URL, UTF8, VM, XML, XMPP, XSRF, XSS +Naming/FileName: + Exclude: + - 'ideosyncratic-ruby.rb' + +# Offense count: 1 +# Configuration parameters: EnforcedStyleForLeadingUnderscores. +# SupportedStylesForLeadingUnderscores: disallowed, required, optional +Naming/MemoizedInstanceVariableName: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: PreferredName. +Naming/RescuedExceptionsVariableName: + Exclude: + - 'bin/coderay' + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 5 +# Configuration parameters: MinNameLength, AllowNamesEndingInNumbers, AllowedNames, ForbiddenNames. +# AllowedNames: io, id, to, by, on, in, at, ip, db, os +Naming/UncommunicativeMethodParamName: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 8 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: snake_case, camelCase +Naming/VariableName: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html.rb' + - 'test/functional/basic.rb' + +# Offense count: 2 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: snake_case, normalcase, non_integer +Naming/VariableNumber: + Exclude: + - 'test/unit/tokens.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect. +Security/JSONLoad: + Exclude: + - 'test/unit/json_encoder.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Security/YAMLLoad: + Exclude: + - 'test/unit/duo.rb' + +# Offense count: 1 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: inline, group +Style/AccessModifierDeclarations: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + +# Offense count: 8 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: always, conditionals +Style/AndOr: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/erb.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 9 +# Configuration parameters: AllowedChars. +Style/AsciiComments: + Exclude: + - 'lib/coderay/scanners/lua.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/Attr: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, IgnoredMethods, AllowBracesOnProceduralOneLiners. +# SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces +# ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object +# FunctionalMethods: let, let!, subject, watch +# IgnoredMethods: lambda, proc, it +Style/BlockDelimiters: + Exclude: + - 'lib/coderay/scanners/python.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: braces, no_braces, context_dependent +Style/BracesAroundHashParameters: + Exclude: + - 'lib/coderay/scanners/ruby/patterns.rb' + +# Offense count: 3 +Style/CaseEquality: + Exclude: + - 'bin/coderay' + - 'rake_tasks/generator.rake' + - 'test/executable/suite.rb' + +# Offense count: 35 +# Cop supports --auto-correct. +Style/CharacterLiteral: + Exclude: + - 'ideosyncratic-ruby.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/go.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'lib/coderay/scanners/sql.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, EnforcedStyle. +# SupportedStyles: nested, compact +Style/ClassAndModuleChildren: + Exclude: + - 'lib/coderay/helpers/word_list.rb' + - 'lib/coderay/scanners/java/builtin_types.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/ClassMethods: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + +# Offense count: 2 +Style/ClassVars: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + +# Offense count: 2 +Style/CommentedKeyword: + Exclude: + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 16 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, SingleLineConditionsOnly, IncludeTernaryExpressions. +# SupportedStyles: assign_to_condition, assign_inside_condition +Style/ConditionalAssignment: + Exclude: + - 'bin/coderay' + - 'coderay.gemspec' + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/scanners/debug.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/raydebug.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'rake_tasks/code_statistics.rb' + - 'test/executable/suite.rb' + +# Offense count: 21 +Style/Documentation: + Enabled: false + +# Offense count: 2 +Style/DoubleNegation: + Exclude: + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +Style/EachWithObject: + Exclude: + - 'bin/coderay' + - 'lib/coderay/helpers/plugin.rb' + - 'rake_tasks/code_statistics.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +Style/EmptyCaseCondition: + Exclude: + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Style/EmptyLiteral: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: compact, expanded +Style/EmptyMethod: + Exclude: + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 8 +# Cop supports --auto-correct. +Style/Encoding: + Exclude: + - 'lib/coderay.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'test/functional/basic.rb' + +# Offense count: 12 +# Cop supports --auto-correct. +Style/ExpandPathArguments: + Exclude: + - 'bench/bench.rb' + - 'coderay.gemspec' + - 'lib/coderay.rb' + - 'test/executable/suite.rb' + - 'test/functional/basic.rb' + - 'test/functional/examples.rb' + - 'test/functional/for_redcloth.rb' + - 'test/functional/suite.rb' + - 'test/unit/file_type.rb' + - 'test/unit/lines_of_code.rb' + - 'test/unit/plugin.rb' + +# Offense count: 22 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: each, for +Style/For: + Exclude: + - 'Rakefile' + - 'lib/coderay/encoders/encoder.rb' + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/tokens.rb' + - 'rake_tasks/generator.rake' + - 'rake_tasks/test.rake' + - 'test/functional/basic.rb' + - 'test/functional/suite.rb' + - 'test/unit/html.rb' + - 'test/unit/json_encoder.rb' + - 'test/unit/suite.rb' + - 'test/unit/token_kind_filter.rb' + +# Offense count: 62 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: format, sprintf, percent +Style/FormatString: + Enabled: false + +# Offense count: 87 +# Configuration parameters: EnforcedStyle. +# SupportedStyles: annotated, template, unannotated +Style/FormatStringToken: + Enabled: false + +# Offense count: 112 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: always, never +Style/FrozenStringLiteralComment: + Enabled: false + +# Offense count: 9 +# Configuration parameters: AllowedVariables. +Style/GlobalVars: + Exclude: + - 'bin/coderay' + - 'lib/coderay.rb' + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'test/functional/suite.rb' + - 'test/unit/suite.rb' + +# Offense count: 16 +# Configuration parameters: MinBodyLength. +Style/GuardClause: + Exclude: + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/encoders/terminal.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/haml.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'test/executable/suite.rb' + - 'test/unit/file_type.rb' + +# Offense count: 306 +# Cop supports --auto-correct. +# Configuration parameters: UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols. +# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys +Style/HashSyntax: + EnforcedStyle: hash_rockets + +# Offense count: 4 +Style/IdenticalConditionalBranches: + Exclude: + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 2 +# Configuration parameters: AllowIfModifier. +Style/IfInsideElse: + Exclude: + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/sass.rb' + +# Offense count: 42 +# Cop supports --auto-correct. +Style/IfUnlessModifier: + Enabled: false + +# Offense count: 3 +Style/IfUnlessModifierOfIfUnless: + Exclude: + - 'lib/coderay/encoders/text.rb' + - 'lib/coderay/scanners/erb.rb' + - 'rake_tasks/test.rake' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/InfiniteLoop: + Exclude: + - 'lib/coderay/scanners/haml.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Style/LineEndConcatenation: + Exclude: + - 'lib/coderay/for_redcloth.rb' + - 'test/functional/basic.rb' + +# Offense count: 221 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: require_parentheses, require_no_parentheses, require_no_parentheses_except_multiline +Style/MethodDefParentheses: + Enabled: false + +# Offense count: 1 +Style/MethodMissingSuper: + Exclude: + - 'lib/coderay/tokens_proxy.rb' + +# Offense count: 2 +Style/MissingRespondToMissing: + Exclude: + - 'lib/coderay/tokens.rb' + - 'lib/coderay/tokens_proxy.rb' + +# Offense count: 1 +Style/MultilineBlockChain: + Exclude: + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +Style/MultilineIfModifier: + Exclude: + - 'lib/coderay/encoders/text.rb' + - 'lib/coderay/scanners/erb.rb' + - 'rake_tasks/documentation.rake' + - 'rake_tasks/test.rake' + - 'test/functional/for_redcloth.rb' + +# Offense count: 10 +Style/MultilineTernaryOperator: + Exclude: + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 7 +Style/MultipleComparison: + Exclude: + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/yaml.rb' + +# Offense count: 247 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: literals, strict +Style/MutableConstant: + Enabled: false + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: both, prefix, postfix +Style/NegatedIf: + Exclude: + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + +# Offense count: 6 +Style/NestedTernaryOperator: + Exclude: + - 'Gemfile' + - 'lib/coderay/scanners/php.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/sass.rb' + - 'lib/coderay/scanners/sql.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: predicate, comparison +Style/NilComparison: + Exclude: + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/scanners/python.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Style/Not: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/scanners/clojure.rb' + - 'lib/coderay/scanners/erb.rb' + +# Offense count: 12 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods. +# SupportedStyles: predicate, comparison +Style/NumericPredicate: + Exclude: + - 'spec/**/*' + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/html/numbering.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/haml.rb' + - 'lib/coderay/scanners/lua.rb' + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/tokens.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/OneLineConditional: + Exclude: + - 'rake_tasks/code_statistics.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/OrAssignment: + Exclude: + - 'lib/coderay/scanners/groovy.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +Style/ParallelAssignment: + Exclude: + - 'lib/coderay/encoders/statistic.rb' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 30 +# Cop supports --auto-correct. +# Configuration parameters: PreferredDelimiters. +Style/PercentLiteralDelimiters: + Enabled: false + +# Offense count: 6 +# Cop supports --auto-correct. +Style/PerlBackrefs: + Exclude: + - 'bin/coderay' + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/for_redcloth.rb' + +# Offense count: 5 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: short, verbose +Style/PreferredHashMethods: + Exclude: + - 'lib/coderay/encoders/debug_lint.rb' + - 'lib/coderay/encoders/lint.rb' + - 'lib/coderay/helpers/plugin_host.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: AllowMultipleReturnValues. +Style/RedundantReturn: + Exclude: + - 'lib/coderay/encoders/html/css.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Style/RedundantSelf: + Exclude: + - 'lib/coderay/encoders/html/output.rb' + - 'lib/coderay/helpers/plugin_host.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +Style/RedundantSort: + Exclude: + - 'test/unit/plugin.rb' + +# Offense count: 58 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, AllowInnerSlashes. +# SupportedStyles: slashes, percent_r, mixed +Style/RegexpLiteral: + Enabled: false + +# Offense count: 4 +# Cop supports --auto-correct. +Style/RescueModifier: + Exclude: + - 'rake_tasks/code_statistics.rb' + - 'rake_tasks/test.rake' + - 'test/functional/for_redcloth.rb' + +# Offense count: 2 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: implicit, explicit +Style/RescueStandardError: + Exclude: + - 'lib/coderay/scanners/ruby.rb' + - 'lib/coderay/scanners/scanner.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: ConvertCodeThatCanStartToReturnNil, Whitelist. +# Whitelist: present?, blank?, presence, try, try! +Style/SafeNavigation: + Exclude: + - 'bin/coderay' + - 'lib/coderay/scanners/ruby.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +# Configuration parameters: AllowAsExpressionSeparator. +Style/Semicolon: + Exclude: + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: AllowIfMethodIsEmpty. +Style/SingleLineMethods: + Exclude: + - 'lib/coderay/tokens.rb' + +# Offense count: 24 +# Cop supports --auto-correct. +# Configuration parameters: . +# SupportedStyles: use_perl_names, use_english_names +Style/SpecialGlobalVars: + EnforcedStyle: use_perl_names + +# Offense count: 1 +# Cop supports --auto-correct. +Style/StderrPuts: + Exclude: + - 'lib/coderay/encoders/json.rb' + +# Offense count: 131 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline. +# SupportedStyles: single_quotes, double_quotes +Style/StringLiterals: + Enabled: false + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle. +# SupportedStyles: single_quotes, double_quotes +Style/StringLiteralsInInterpolation: + Exclude: + - 'rake_tasks/code_statistics.rb' + +# Offense count: 1 +Style/StructInheritance: + Exclude: + - 'lib/coderay/scanners/ruby/string_state.rb' + +# Offense count: 37 +# Cop supports --auto-correct. +# Configuration parameters: MinSize. +# SupportedStyles: percent, brackets +Style/SymbolArray: + EnforcedStyle: brackets + +# Offense count: 4 +# Cop supports --auto-correct. +# Configuration parameters: IgnoredMethods. +# IgnoredMethods: respond_to, define_method +Style/SymbolProc: + Exclude: + - 'bin/coderay' + - 'lib/coderay/scanners/scanner.rb' + - 'test/unit/plugin.rb' + +# Offense count: 1 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyle, AllowSafeAssignment. +# SupportedStyles: require_parentheses, require_no_parentheses, require_parentheses_when_complex +Style/TernaryParentheses: + Exclude: + - 'lib/coderay/scanners/diff.rb' + +# Offense count: 21 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInArrayLiteral: + Exclude: + - 'lib/coderay/scanners/c.rb' + - 'lib/coderay/scanners/cpp.rb' + - 'lib/coderay/scanners/css.rb' + - 'lib/coderay/scanners/delphi.rb' + - 'lib/coderay/scanners/go.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/json.rb' + - 'lib/coderay/scanners/python.rb' + - 'lib/coderay/scanners/scanner.rb' + - 'test/unit/json_encoder.rb' + +# Offense count: 26 +# Cop supports --auto-correct. +# Configuration parameters: EnforcedStyleForMultiline. +# SupportedStylesForMultiline: comma, consistent_comma, no_comma +Style/TrailingCommaInHashLiteral: + Exclude: + - 'lib/coderay/encoders/html.rb' + - 'lib/coderay/encoders/terminal.rb' + - 'lib/coderay/encoders/xml.rb' + - 'lib/coderay/for_redcloth.rb' + - 'lib/coderay/helpers/file_type.rb' + - 'lib/coderay/scanners/diff.rb' + - 'lib/coderay/scanners/groovy.rb' + - 'lib/coderay/scanners/html.rb' + - 'lib/coderay/scanners/java.rb' + - 'lib/coderay/scanners/java_script.rb' + - 'lib/coderay/scanners/ruby/patterns.rb' + - 'lib/coderay/scanners/sql.rb' + +# Offense count: 3 +# Cop supports --auto-correct. +Style/VariableInterpolation: + Exclude: + - 'bin/coderay' + - 'ideosyncratic-ruby.rb' + +# Offense count: 10 +# Cop supports --auto-correct. +# Configuration parameters: WordRegex. +# SupportedStyles: percent, brackets +Style/WordArray: + EnforcedStyle: percent + MinSize: 69 + +# Offense count: 1 +# Cop supports --auto-correct. +Style/ZeroLengthPredicate: + Exclude: + - 'lib/coderay/encoders/html.rb' + +# Offense count: 813 +# Cop supports --auto-correct. +# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. +# URISchemes: http, https +Metrics/LineLength: + Max: 266 diff --git a/.simplecov b/.simplecov new file mode 100644 index 00000000..f498df81 --- /dev/null +++ b/.simplecov @@ -0,0 +1,4 @@ +unless RUBY_VERSION[/^2.3/] + SimpleCov.command_name $0 + SimpleCov.start +end diff --git a/.travis.yml b/.travis.yml index 59bb791d..6c607d27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,40 @@ +env: + global: + - "JRUBY_OPTS=-Xcext.enabled=true" + - "CC_TEST_REPORTER_ID=faa393209ff0a104cf37511a9a03510bcee37951971b1ca4ffc2af217851d47e" +language: ruby +os: linux rvm: - 1.8.7 - ree - 1.9.3 - - 2.0.0 + - 2.0 + - 2.1 + - 2.2 + - 2.3 + - 2.4 + - 2.5 + - 2.6 + - 2.7 + - 3.0 + - 3.1 + - 3.2 - ruby-head - - jruby-18mode - - jruby-19mode - - jruby-head - - rbx-18mode - - rbx-19mode + - jruby +jobs: + allow_failures: + - rvm: 1.8.7 + - rvm: ree + - rvm: ruby-head + - rvm: jruby branches: only: - master -matrix: - allow_failures: - - rvm: rbx-18mode - - rvm: rbx-19mode +before_script: + - if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT + - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + - chmod +x ./cc-test-reporter + - ./cc-test-reporter before-build script: "rake test" # test:scanners" +after_script: + - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT diff --git a/Changes.textile b/Changes.textile index fcee29bc..8c4f3e95 100644 --- a/Changes.textile +++ b/Changes.textile @@ -2,26 +2,96 @@ h1=. CodeRay Version History p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ +h2. Changes in 1.1.3 + +* Tokens: Ensure Ruby 2.6 compatibility. [#233, thanks to Jun Aruga] +* SQL scanner: Add @numeric@ data type. [#223, thanks to m16a1] +* Java scanner: Add @var@ as type. [#229, thanks to Davide Angelocola] +* Gem: Fix deprecation warning. [#246, thanks to David Rodríguez] + +h2. Changes in 1.1.2 + +* Ruby future: Add support for frozen string literals. [#211, thanks to Pat Allan] +* C++ scanner: Add C++11 keywords. [#195, thanks to Johnny Willemsen] +* Haml scanner: Allow @-@ in tags. +* Java scanner: Allow Unicode characters in identifiers. [#212, thanks to t-gergely] + +h2. Changes in 1.1.1 + +* SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson] +* SQL scanner: Fix open strings [#163, thanks to Adam] +* Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1) +* Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2) +* Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3) +* Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3) +* Diff scanner: Prevent running out of regexp stack. +* HTML encoder: You can keep tabs intact now by setting @tab_width: false@. +* Alpha style: Tweaked colors for @:function@ group with @:content@. +* File structure: One module per file, autoload CodeRay::Version, paths follow namespace hierarchy. + h2. Changes in 1.1 -* New scanner: Sass [#93] -* New scanner: Taskpaper [#39, thanks to shimomura] +New scanners: + +* Go [#28, thanks to Eric Guo and Nathan Youngman] +* Lua [#21, #22, thanks to Quintus] +* Sass [#93] +* Taskpaper [#39, thanks to shimomura] + +More new stuff: + +* @.xaml@ file type [#121, thanks to Kozman Bálint] +* recognize @Guardfile@, @Vagrantfile@, and @Appraisals@ as Ruby files [#121, thanks to Kozman Bálint] +* new token kind @:id@ for CSS/Sass [#27] +* new token kind @:done@ for Taskpaper [#39] +* new token kind @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman] +* new token kind @:unknown@ for Debug scanner +* new DebugLint encoder that checks for empty tokens and correct nesting + +Improvements: + +* CSS scanner uses @:id@ and @:tag@ now [#27] * Diff scanner: Highlight inline changes in multi-line changes [#99] * JavaScript scanner: Highlight multi-line comments in diff correctly -* Ruby scanner: Accept keywords as Ruby 1.9 hash keys [#126] -* Remove double-click toggle handler from HTML table output -* Fixes to CSS scanner (floats, pseudoclasses) -* CSS scanner uses @:id@ and @:tag@ now [#27] -* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right? -* Add .xaml file type [#121, thanks to Kozman Bálint] -* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] -* New token type @:id@ for CSS/Sass [#27] -* New token type @:done@ for Taskpaper [#39] -* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] -* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] -* Fixed @:docstring@ token type style -* @Plugin@ does not warn about fallback when default is defined -* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now +* JSON scanner: simplify key/value heuristic, using look-ahead instead of a stack +* HTML scanner displays style tags and attributes now [#145] +* Ruby scanner: Accept @%i(…)@ and @%I(…)@ symbol lists (Ruby 2.0) [thanks to Nathan Youngman] +* Ruby scanner: Accept keywords as Ruby hash keys [#126] +* performance improvements to several scanners and encoders, especially Terminal and HTML +* added @:keep_state@ functionality to more scanners so they work nicely with diff now [#116] +* refactoring and cleanup to achieve better "Code Climate" ratings (but I don't really care) +* updated and cleaned up the documentation, +* documented list of TokenKinds +* Alpha style: tweaked colors for @.binary@, @.local-variable@, and @.predefined-type@ +* @rake generate@ supports Git now instead of Subversion + +Removed: + +* @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency +* double-click toggle handler from HTML table output +* @rake_helpers@, @sample@ directories and several other ancient garbage + +Fixes: + +* fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143] +* fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144] +* fixed @:docstring@ token type style +* fixed several infinite Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148] +* fixed HTML encoder when output is a StringIO (eg. when using @-HTML@ as a command line parameter) +* TokenKinds should not be frozen [#130, thanks to Gavin Kistner] +* display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] +* override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05] +* HTML encoder will not warn about unclosed token groups at the end of the stream +* fixed problem with coderay/version.rb being loaded twice + +Internals: + +* The Debug scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input). +* The Raydebug scanner highlights unknown token kinds as @:plain@. +* The Debug encoder refactored; use DebugLint if you want strict checking now.. +* The Debug encoder will not warn about errors in the token stream. +* Plugin does not warn about fallback when default is defined. +* PluginHost now works with Strings instead of Symbols internally (to avoid using @#to_sym@). h2. Changes in 1.0.9 @@ -431,6 +501,3 @@ The helper classes were cleaned up; see above for details. * *CHANGED* @Plugin@ API was simplified and stripped of all unnecessary features. * *CHANGED* Moved @GZip@ and @FileType@ libraries into @CodeRay@; cleaned them up. - - - diff --git a/FOLDERS b/FOLDERS index f29255ae..1709d08a 100644 --- a/FOLDERS +++ b/FOLDERS @@ -1,48 +1,48 @@ -= CodeRay - folder structure - -== bench - Benchmarking system - -All benchmarking stuff goes here. - -Test inputs are stored in files named example.. -Test outputs go to bench/test.. - -Run bench/bench.rb to get a usage description. - -Run rake bench to perform an example benchmark. - - -== bin - Scripts - -Executional files for CodeRay. - -coderay:: The CodeRay executable. - -== demo - Demos and functional tests - -Demonstrational scripts to show of CodeRay's features. - -Run them as functional tests with rake test:demos. - - -== etc - Lots of stuff - -Some addidtional files for CodeRay, mainly graphics and Vim scripts. - - -== lib - CodeRay library code - -This is the base directory for the CodeRay library. - - -== rake_helpers - Rake helper libraries - -Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. - - -== test - Tests - -In the subfolder scanners/ are the scanners tests. -Each language has its own subfolder and sub-suite. - -Run with rake test. += CodeRay - folder structure + +== bench - Benchmarking system + +All benchmarking stuff goes here. + +Test inputs are stored in files named example.. +Test outputs go to bench/test.. + +Run bench/bench.rb to get a usage description. + +Run rake bench to perform an example benchmark. + + +== bin - Scripts + +Executional files for CodeRay. + +coderay:: The CodeRay executable. + +== demo - Demos and functional tests + +Demonstrational scripts to show of CodeRay's features. + +Run them as functional tests with rake test:demos. + + +== etc - Lots of stuff + +Some additional files for CodeRay, mainly graphics and Vim scripts. + + +== lib - CodeRay library code + +This is the base directory for the CodeRay library. + + +== rake_helpers - Rake helper libraries + +Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. + + +== test - Tests + +In the subfolder scanners/ are the scanners tests. +Each language has its own subfolder and sub-suite. + +Run with rake test. diff --git a/Gemfile b/Gemfile index 15a71aea..49ac338d 100644 --- a/Gemfile +++ b/Gemfile @@ -6,11 +6,15 @@ gemspec # Add dependencies to develop your gem here. # Include everything needed to run rake, tests, features, etc. group :development do - gem "bundler", ">= 1.0.0" - gem "rake" - gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor", '~> 1.2.2' - gem "shoulda-context", "~> 1.1.2" - gem "json" if RUBY_VERSION < '1.9' - gem "rdoc" + gem 'bundler' + gem 'json', '~> 1.8' if RUBY_VERSION < '2.0' + gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' + gem 'rdoc', Gem::Version.new(RUBY_VERSION) < Gem::Version.new('1.9.3') ? '~> 4.2.2' : Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2') ? '< 6' : '>= 6' + gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' + gem 'rspec', '~> 3.9.0' + gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' + gem 'simplecov', RUBY_VERSION < '2.7' ? '~> 0.17.1' : '>= 0.18.5' + gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' + gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' + gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' end diff --git a/MIT-LICENSE.txt b/MIT-LICENSE similarity index 87% rename from MIT-LICENSE.txt rename to MIT-LICENSE index d8d009d1..9431e246 100644 --- a/MIT-LICENSE.txt +++ b/MIT-LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2005-2012 Kornelius Kalnbach (@murphy_karasu) +Copyright (C) 2005 Kornelius Kalnbach (@murphy_karasu) http://coderay.rubychan.de/ @@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.markdown b/README.markdown index f3336552..a768ef10 100644 --- a/README.markdown +++ b/README.markdown @@ -1,8 +1,6 @@ # CodeRay -[![Build Status](https://travis-ci.org/rubychan/coderay.png)](https://travis-ci.org/rubychan/coderay) -[![Gem Version](https://badge.fury.io/rb/coderay.png)](http://badge.fury.io/rb/coderay) -[![Dependency Status](https://gemnasium.com/rubychan/coderay.png)](https://gemnasium.com/rubychan/coderay) +[![CircleCI](https://dl.circleci.com/status-badge/img/gh/rubychan/coderay/tree/master.svg?style=svg&circle-token=CCIPRJ_BbFff6nMhNtPdrCBNMDxNq_be00bbb00849a29d8d8d2e28e7b84cbf76a9ee5c)](https://dl.circleci.com/status-badge/redirect/gh/rubychan/coderay/tree/master) [![Gem Version](https://badge.fury.io/rb/coderay.svg)](https://badge.fury.io/rb/coderay) [![Maintainability](https://api.codeclimate.com/v1/badges/e015bbd5eab45d948b6b/maintainability)](https://codeclimate.com/github/rubychan/coderay/maintainability) ## About @@ -16,7 +14,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co ### Dependencies -CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby. +CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby. ## Example Usage @@ -28,4 +26,4 @@ html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table) ## Documentation -See [http://coderay.rubychan.de/doc/](http://coderay.rubychan.de/doc/). +See [rubydoc](http://rubydoc.info/gems/coderay). diff --git a/Rakefile b/Rakefile index ba6c34ef..c9b1e8a3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,3 +1,5 @@ +require 'bundler/gem_tasks' + $:.unshift File.dirname(__FILE__) unless $:.include? '.' ROOT = '.' @@ -32,4 +34,4 @@ else rd.rdoc_dir = 'doc' end -end \ No newline at end of file +end diff --git a/bench/bench.rb b/bench/bench.rb index 1958c73a..a47721e3 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -1,142 +1,50 @@ -# The most ugly test script I've ever written! -# Shame on me! - -require 'pathname' -require 'profile' if ARGV.include? '-p' - -MYDIR = File.dirname(__FILE__) -LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s -$:.unshift MYDIR, LIBDIR +require 'benchmark' +$: << File.expand_path('../../lib', __FILE__) require 'coderay' -@size = ARGV.fetch(2, 100).to_i * 1000 - -lang = ARGV.fetch(0) do - puts <<-HELP -Usage: - ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream] - - SIZE defaults to 100 kB (= 100,000 bytes). - SIZE = 0 means the whole input. - --p generates a profile (slow! use with SIZE = 1) --o shows the output -stream enabled streaming mode - -Sorry for the strange interface. I will improve it in the next release. - HELP +if ARGV.include? '-h' + puts DATA.read exit end -format = ARGV.fetch(1, 'html').downcase - -$stream = ARGV.include? 'stream' -$optimize = ARGV.include? 'opt' -$style = ARGV.include? 'style' - -require 'benchmark' -require 'fileutils' +lang = ARGV.fetch(0, 'ruby') +data = nil +File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read } +raise 'Example file is empty.' if data.empty? -if format == 'comp' - format = 'page' - begin - require 'syntax' - require 'syntax/convertors/html.rb' - rescue LoadError - puts 'Syntax no found!! (Try % gem install syntax)' - end -end +format = ARGV.fetch(1, 'html').downcase +encoder = CodeRay.encoder(format) -def here fn = nil - return MYDIR unless fn - File.join here, fn +size = ARGV.fetch(2, 3000).to_i * 1000 +unless size.zero? + data += data until data.size >= size + data = data[0, size] end +size = data.size +puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] -n = ARGV.find { |a| a[/^N/] } -N = if n then n[/\d+/].to_i else 1 end -$filename = ARGV.include?('strange') ? 'strange' : 'example' - -Benchmark.bm(20) do |bm| -N.times do - - data = nil - File.open(here("#$filename." + lang), 'rb') { |f| data = f.read } - raise 'Example file is empty.' if data.empty? - unless @size.zero? - data += data until data.size >= @size - data = data[0, @size] - end - @size = data.size - - options = { - :tab_width => 2, - # :line_numbers => :inline, - :css => $style ? :style : :class, - } - $hl = CodeRay.encoder(format, options) - time = bm.report('CodeRay') do - if $stream || true - $o = $hl.encode(data, lang, options) - else - tokens = CodeRay.scan(data, lang) - tokens.optimize! if $optimize - $o = tokens.encode($hl) - end - end - $file_created = here('test.' + $hl.file_extension) - File.open($file_created, 'wb') do |f| - # f.write $o - end - - time_real = time.real - - puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000] - puts $o if ARGV.include? '-o' - -end +n = ARGV.fetch(3, 10).to_s[/\d+/].to_i +require 'profile' if ARGV.include? '-p' +times = [] +n.times do |i| + time = Benchmark.realtime { encoder.encode(data, lang) } + puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] + times << time end -puts "Files created: #$file_created" -STDIN.gets if ARGV.include? 'wait' +times_sum = times.inject(0) { |time, sum| sum + time } +puts 'Average time: %5.2f s, %4.0f kB/s' % [times_sum / times.size, (size * n) / times_sum / 1000.0] +puts 'Best time: %5.2f s, %4.0f kB/s' % [times.min, size / times.min / 1000.0] __END__ -.ruby .normal {} -.ruby .comment { color: #005; font-style: italic; } -.ruby .keyword { color: #A00; font-weight: bold; } -.ruby .method { color: #077; } -.ruby .class { color: #074; } -.ruby .module { color: #050; } -.ruby .punct { color: #447; font-weight: bold; } -.ruby .symbol { color: #099; } -.ruby .string { color: #944; background: #FFE; } -.ruby .char { color: #F07; } -.ruby .ident { color: #004; } -.ruby .constant { color: #07F; } -.ruby .regex { color: #B66; background: #FEF; } -.ruby .number { color: #F99; } -.ruby .attribute { color: #7BB; } -.ruby .global { color: #7FB; } -.ruby .expr { color: #227; } -.ruby .escape { color: #277; } +Usage: + ruby bench.rb [lang] [format] [size in kB] [number of runs] -.xml .normal {} -.xml .namespace { color: #B66; font-weight: bold; } -.xml .tag { color: #F88; } -.xml .comment { color: #005; font-style: italic; } -.xml .punct { color: #447; font-weight: bold; } -.xml .string { color: #944; } -.xml .number { color: #F99; } -.xml .attribute { color: #BB7; } + - lang defaults to ruby. + - format defaults to html. + - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input. + - number of runs defaults to 5. -.yaml .normal {} -.yaml .document { font-weight: bold; color: #07F; } -.yaml .type { font-weight: bold; color: #05C; } -.yaml .key { color: #F88; } -.yaml .comment { color: #005; font-style: italic; } -.yaml .punct { color: #447; font-weight: bold; } -.yaml .string { color: #944; } -.yaml .number { color: #F99; } -.yaml .time { color: #F99; } -.yaml .date { color: #F99; } -.yaml .ref { color: #944; } -.yaml .anchor { color: #944; } +-h prints this help +-p generates a profile (slow, use with SIZE = 1) +-w waits after the benchmark (for debugging memory usw) diff --git a/bin/coderay b/bin/coderay index 889ae726..130a50ba 100755 --- a/bin/coderay +++ b/bin/coderay @@ -35,7 +35,7 @@ defaults: common: coderay file.rb # highlight file to terminal - coderay file.rb > file.html # highlight file to HTML page + coderay file.rb -page > file.html # highlight file to HTML page coderay file.rb -div > file.html # highlight file to HTML snippet configure output: diff --git a/coderay.gemspec b/coderay.gemspec index 328b94c1..9aba34eb 100644 --- a/coderay.gemspec +++ b/coderay.gemspec @@ -24,12 +24,10 @@ Gem::Specification.new do |s| readme_file = 'README_INDEX.rdoc' - s.files = `git ls-files -- lib/* test/functional/* Rakefile #{readme_file} MIT-LICENSE`.split("\n") - s.test_files = `git ls-files -- test/functional/*`.split("\n") + s.files = `git ls-files -- lib/* #{readme_file} MIT-LICENSE`.split("\n") s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } s.require_paths = ['lib'] - s.rubyforge_project = s.name - s.rdoc_options = '-SNw2', "-m#{readme_file}", '-t CodeRay Documentation' + s.rdoc_options = '-Nw2', "-m#{readme_file}", '-t CodeRay Documentation' s.extra_rdoc_files = readme_file end diff --git a/lib/coderay.rb b/lib/coderay.rb index 24ae5a2c..c3de20b5 100644 --- a/lib/coderay.rb +++ b/lib/coderay.rb @@ -127,14 +127,14 @@ module CodeRay $CODERAY_DEBUG ||= false - CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay' + CODERAY_PATH = File.expand_path('../coderay', __FILE__) # Assuming the path is a subpath of lib/coderay/ def self.coderay_path *path File.join CODERAY_PATH, *path end - require coderay_path('version') + autoload :VERSION, 'coderay/version' # helpers autoload :FileType, coderay_path('helpers', 'file_type') @@ -145,13 +145,13 @@ def self.coderay_path *path autoload :TokenKinds, coderay_path('token_kinds') # Plugin system - autoload :PluginHost, coderay_path('helpers', 'plugin') + autoload :PluginHost, coderay_path('helpers', 'plugin_host') autoload :Plugin, coderay_path('helpers', 'plugin') # Plugins - autoload :Scanners, coderay_path('scanner') - autoload :Encoders, coderay_path('encoder') - autoload :Styles, coderay_path('style') + autoload :Scanners, coderay_path('scanners') + autoload :Encoders, coderay_path('encoders') + autoload :Styles, coderay_path('styles') # convenience access and reusable Encoder/Scanner pair autoload :Duo, coderay_path('duo') @@ -166,7 +166,7 @@ class << self # # See also demo/demo_simple. def scan code, lang, options = {}, &block - TokensProxy.new code, lang, options, block + CodeRay::TokensProxy.new code, lang, options, block end # Scans +filename+ (a path to a code file) with the Scanner for +lang+. @@ -181,7 +181,7 @@ def scan code, lang, options = {}, &block # require 'coderay' # page = CodeRay.scan_file('some_c_code.c').html def scan_file filename, lang = :auto, options = {}, &block - lang = FileType.fetch filename, :text, true if lang == :auto + lang = CodeRay::FileType.fetch filename, :text, true if lang == :auto code = File.read filename scan code, lang, options, &block end @@ -258,7 +258,7 @@ def highlight_file filename, options = { :css => :class }, format = :div # ] # #-> 2 out of 4 tokens have the kind :integer. def encoder format, options = {} - Encoders[format].new options + CodeRay::Encoders[format].new options end # Finds the Scanner class for +lang+ and creates an instance, passing @@ -266,7 +266,7 @@ def encoder format, options = {} # # See Scanner.new. def scanner lang, options = {}, &block - Scanners[lang].new '', options, &block + CodeRay::Scanners[lang].new '', options, &block end # Extract the options for the scanner from the +options+ hash. diff --git a/lib/coderay/encoders.rb b/lib/coderay/encoders.rb new file mode 100644 index 00000000..6599186e --- /dev/null +++ b/lib/coderay/encoders.rb @@ -0,0 +1,18 @@ +module CodeRay + + # This module holds the Encoder class and its subclasses. + # For example, the HTML encoder is named CodeRay::Encoders::HTML + # can be found in coderay/encoders/html. + # + # Encoders also provides methods and constants for the register + # mechanism and the [] method that returns the Encoder class + # belonging to the given format. + module Encoders + + extend PluginHost + plugin_path File.dirname(__FILE__), 'encoders' + + autoload :Encoder, CodeRay.coderay_path('encoders', 'encoder') + + end +end diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb index eeb2a92b..a4eba2c7 100644 --- a/lib/coderay/encoders/debug_lint.rb +++ b/lib/coderay/encoders/debug_lint.rb @@ -1,6 +1,8 @@ module CodeRay module Encoders + load :lint + # = Debug Lint Encoder # # Debug encoder with additional checks for: @@ -15,17 +17,9 @@ class DebugLint < Debug register_for :debug_lint - InvalidTokenStream = Class.new StandardError - EmptyToken = Class.new InvalidTokenStream - IncorrectTokenGroupNesting = Class.new InvalidTokenStream - - def initialize options = {} - super - @opened = [] - end - def text_token text, kind - raise EmptyToken, 'empty token' if text.empty? + raise Lint::EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise Lint::UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind super end @@ -35,7 +29,8 @@ def begin_group kind end def end_group kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop super end @@ -45,7 +40,20 @@ def begin_line kind end def end_line kind - raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind + raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + super + end + + protected + + def setup options + super + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? super end diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoders/encoder.rb similarity index 92% rename from lib/coderay/encoder.rb rename to lib/coderay/encoders/encoder.rb index d2d6c7e6..2baeedb6 100644 --- a/lib/coderay/encoder.rb +++ b/lib/coderay/encoders/encoder.rb @@ -1,17 +1,6 @@ module CodeRay - - # This module holds the Encoder class and its subclasses. - # For example, the HTML encoder is named CodeRay::Encoders::HTML - # can be found in coderay/encoders/html. - # - # Encoders also provides methods and constants for the register - # mechanism and the [] method that returns the Encoder class - # belonging to the given format. module Encoders - extend PluginHost - plugin_path File.dirname(__FILE__), 'encoders' - # = Encoder # # The Encoder base class. Together with Scanner and @@ -157,7 +146,7 @@ def setup options end def get_output options - options[:out] || '' + options[:out] || ''.dup end # Append data.to_s to the output. Returns the argument. diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index b897f5e2..1b33e921 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -25,7 +25,8 @@ module Encoders # == Options # # === :tab_width - # Convert \t characters to +n+ spaces (a number.) + # Convert \t characters to +n+ spaces (a number or false.) + # false will keep tab characters untouched. # # Default: 8 # @@ -175,12 +176,12 @@ def setup options if options[:wrap] || options[:line_numbers] @real_out = @out - @out = '' + @out = ''.dup end @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t") @opened = [] @last_opened = nil @@ -193,18 +194,19 @@ def setup options def finish options unless @opened.empty? - warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG @out << '' while @opened.pop @last_opened = nil end - @out.extend Output - @out.css = @css - if options[:line_numbers] - Numbering.number! @out, options[:line_numbers], options + if @out.respond_to? :to_str + @out.extend Output + @out.css = @css + if options[:line_numbers] + Numbering.number! @out, options[:line_numbers], options + end + @out.wrap! options[:wrap] + @out.apply_title! options[:title] end - @out.wrap! options[:wrap] - @out.apply_title! options[:title] if defined?(@real_out) && @real_out @real_out << @out @@ -286,7 +288,7 @@ def style_for_kinds kinds def make_span_for_kinds method, hint Hash.new do |h, kinds| - h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + begin css_class = css_class_for_kinds(kinds) title = HTML.token_path_to_hint hint, kinds if hint @@ -298,6 +300,9 @@ def make_span_for_kinds method, hint "" end end + end.tap do |span| + h.clear if h.size >= 100 + h[kinds] = span end end end @@ -309,9 +314,9 @@ def check_group_nesting name, kind end def break_lines text, style - reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '') + reopen = ''.dup + @opened.each_with_index do |kind, index| + reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '') end text.gsub("\n", "#{'' * @opened.size}#{'' if style}\n#{reopen}#{style}") end diff --git a/lib/coderay/encoders/html/output.rb b/lib/coderay/encoders/html/output.rb index de6f6ea1..ee87fea5 100644 --- a/lib/coderay/encoders/html/output.rb +++ b/lib/coderay/encoders/html/output.rb @@ -76,8 +76,6 @@ def wrap! element, *args apply_title! title end self - when nil - return self else raise "Unknown value %p for :wrap" % element end diff --git a/lib/coderay/encoders/lint.rb b/lib/coderay/encoders/lint.rb new file mode 100644 index 00000000..88c8bd1d --- /dev/null +++ b/lib/coderay/encoders/lint.rb @@ -0,0 +1,59 @@ +module CodeRay +module Encoders + + # = Lint Encoder + # + # Checks for: + # + # - empty tokens + # - incorrect nesting + # + # It will raise an InvalidTokenStream exception when any of the above occurs. + # + # See also: Encoders::DebugLint + class Lint < Debug + + register_for :lint + + InvalidTokenStream = Class.new StandardError + EmptyToken = Class.new InvalidTokenStream + UnknownTokenKind = Class.new InvalidTokenStream + IncorrectTokenGroupNesting = Class.new InvalidTokenStream + + def text_token text, kind + raise EmptyToken, 'empty token for %p' % [kind] if text.empty? + raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind + end + + def begin_group kind + @opened << kind + end + + def end_group kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + def begin_line kind + @opened << kind + end + + def end_line kind + raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind + @opened.pop + end + + protected + + def setup options + @opened = [] + end + + def finish options + raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty? + end + + end + +end +end diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 19f27acd..7de34d58 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -38,7 +38,7 @@ def [] filename, read_shebang = false (TypeFromExt[ext2.downcase] if ext2) || TypeFromName[name] || TypeFromName[name.downcase] - type ||= shebang(filename) if read_shebang + type ||= type_from_shebang(filename) if read_shebang type end @@ -63,7 +63,7 @@ def fetch filename, default = nil, read_shebang = false protected - def shebang filename + def type_from_shebang filename return unless File.exist? filename File.open filename, 'r' do |f| if first_line = f.gets @@ -86,6 +86,7 @@ def shebang filename 'dpr' => :delphi, 'erb' => :erb, 'gemspec' => :ruby, + 'go' => :go, 'groovy' => :groovy, 'gvy' => :groovy, 'h' => :c, @@ -96,6 +97,7 @@ def shebang filename 'java' => :java, 'js' => :java_script, 'json' => :json, + 'lua' => :lua, 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, @@ -115,7 +117,7 @@ def shebang filename 'rhtml' => :erb, 'rjs' => :ruby, 'rpdf' => :ruby, - 'ru' => :ruby, + 'ru' => :ruby, # config.ru 'rxml' => :ruby, 'sass' => :sass, 'sql' => :sql, @@ -139,6 +141,9 @@ def shebang filename 'Rakefile' => :ruby, 'Rantfile' => :ruby, 'Gemfile' => :ruby, + 'Guardfile' => :ruby, + 'Vagrantfile' => :ruby, + 'Appraisals' => :ruby } end diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index d14c5a94..45679436 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -1,224 +1,5 @@ module CodeRay - # = PluginHost - # - # A simple subclass/subfolder plugin system. - # - # Example: - # class Generators - # extend PluginHost - # plugin_path 'app/generators' - # end - # - # class Generator - # extend Plugin - # PLUGIN_HOST = Generators - # end - # - # class FancyGenerator < Generator - # register_for :fancy - # end - # - # Generators[:fancy] #-> FancyGenerator - # # or - # CodeRay.require_plugin 'Generators/fancy' - # # or - # Generators::Fancy - module PluginHost - - # Raised if Encoders::[] fails because: - # * a file could not be found - # * the requested Plugin is not registered - PluginNotFound = Class.new LoadError - HostNotFound = Class.new LoadError - - PLUGIN_HOSTS = [] - PLUGIN_HOSTS_BY_ID = {} # dummy hash - - # Loads all plugins using list and load. - def load_all - for plugin in list - load plugin - end - end - - # Returns the Plugin for +id+. - # - # Example: - # yaml_plugin = MyPluginHost[:yaml] - def [] id, *args, &blk - plugin = validate_id(id) - begin - plugin = plugin_hash.[] plugin, *args, &blk - end while plugin.is_a? Symbol - plugin - end - - alias load [] - - # Tries to +load+ the missing plugin by translating +const+ to the - # underscore form (eg. LinesOfCode becomes lines_of_code). - def const_missing const - id = const.to_s. - gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). - gsub(/([a-z\d])([A-Z])/,'\1_\2'). - downcase - load id - end - - class << self - - # Adds the module/class to the PLUGIN_HOSTS list. - def extended mod - PLUGIN_HOSTS << mod - end - - end - - # The path where the plugins can be found. - def plugin_path *args - unless args.empty? - @plugin_path = File.expand_path File.join(*args) - end - @plugin_path ||= '' - end - - # Map a plugin_id to another. - # - # Usage: Put this in a file plugin_path/_map.rb. - # - # class MyColorHost < PluginHost - # map :navy => :dark_blue, - # :maroon => :brown, - # :luna => :moon - # end - def map hash - for from, to in hash - from = validate_id from - to = validate_id to - plugin_hash[from] = to unless plugin_hash.has_key? from - end - end - - # Define the default plugin to use when no plugin is found - # for a given id, or return the default plugin. - # - # See also map. - # - # class MyColorHost < PluginHost - # map :navy => :dark_blue - # default :gray - # end - # - # MyColorHost.default # loads and returns the Gray plugin - def default id = nil - if id - id = validate_id id - raise "The default plugin can't be named \"default\"." if id == :default - plugin_hash[:default] = id - else - load :default - end - end - - # Every plugin must register itself for +id+ by calling register_for, - # which calls this method. - # - # See Plugin#register_for. - def register plugin, id - plugin_hash[validate_id(id)] = plugin - end - - # A Hash of plugion_id => Plugin pairs. - def plugin_hash - @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } - end - - # Returns an array of all .rb files in the plugin path. - # - # The extension .rb is not included. - def list - Dir[path_to('*')].select do |file| - File.basename(file)[/^(?!_)\w+\.rb$/] - end.map do |file| - File.basename(file, '.rb').to_sym - end - end - - # Returns an array of all Plugins. - # - # Note: This loads all plugins using load_all. - def all_plugins - load_all - plugin_hash.values.grep(Class) - end - - # Loads the map file (see map). - # - # This is done automatically when plugin_path is called. - def load_plugin_map - mapfile = path_to '_map' - if File.exist? mapfile - require mapfile - true - else - false - end - end - - protected - - # Return a plugin hash that automatically loads plugins. - def make_plugin_hash - Hash.new do |h, plugin_id| - id = validate_id(plugin_id) - path = path_to id - begin - require path - rescue LoadError => boom - if h.has_key?(:default) - h[:default] - else - raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] - end - else - # Plugin should have registered by now - if h.has_key? id - h[id] - else - raise PluginNotFound, "No #{self.name} plugin for #{id.inspect} found in #{path}." - end - end - end - end - - # Returns the expected path to the plugin file for the given id. - def path_to plugin_id - File.join plugin_path, "#{plugin_id}.rb" - end - - # Converts +id+ to a Symbol if it is a String, - # or returns +id+ if it already is a Symbol. - # - # Raises +ArgumentError+ for all other objects, or if the - # given String includes non-alphanumeric characters (\W). - def validate_id id - if id.is_a? Symbol or id.nil? - id - elsif id.is_a? String - if id[/\w+/] == id - id.downcase.to_sym - else - raise ArgumentError, "Invalid id given: #{id}" - end - else - raise ArgumentError, "String or Symbol expected, but #{id.class} given." - end - end - - end - - # = Plugin # # Plugins have to include this module. diff --git a/lib/coderay/helpers/plugin_host.rb b/lib/coderay/helpers/plugin_host.rb new file mode 100644 index 00000000..e9bc17c1 --- /dev/null +++ b/lib/coderay/helpers/plugin_host.rb @@ -0,0 +1,221 @@ +module CodeRay + + # = PluginHost + # + # A simple subclass/subfolder plugin system. + # + # Example: + # class Generators + # extend PluginHost + # plugin_path 'app/generators' + # end + # + # class Generator + # extend Plugin + # PLUGIN_HOST = Generators + # end + # + # class FancyGenerator < Generator + # register_for :fancy + # end + # + # Generators[:fancy] #-> FancyGenerator + # # or + # CodeRay.require_plugin 'Generators/fancy' + # # or + # Generators::Fancy + module PluginHost + + # Raised if Encoders::[] fails because: + # * a file could not be found + # * the requested Plugin is not registered + PluginNotFound = Class.new LoadError + HostNotFound = Class.new LoadError + + PLUGIN_HOSTS = [] + PLUGIN_HOSTS_BY_ID = {} # dummy hash + + # Loads all plugins using list and load. + def load_all + for plugin in list + load plugin + end + end + + # Returns the Plugin for +id+. + # + # Example: + # yaml_plugin = MyPluginHost[:yaml] + def [] id, *args, &blk + plugin = validate_id(id) + begin + plugin = plugin_hash.[](plugin, *args, &blk) + end while plugin.is_a? String + plugin + end + + alias load [] + + # Tries to +load+ the missing plugin by translating +const+ to the + # underscore form (eg. LinesOfCode becomes lines_of_code). + def const_missing const + id = const.to_s. + gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2'). + gsub(/([a-z\d])([A-Z])/,'\1_\2'). + downcase + load id + end + + class << self + + # Adds the module/class to the PLUGIN_HOSTS list. + def extended mod + PLUGIN_HOSTS << mod + end + + end + + # The path where the plugins can be found. + def plugin_path *args + unless args.empty? + @plugin_path = File.expand_path File.join(*args) + end + @plugin_path ||= '' + end + + # Map a plugin_id to another. + # + # Usage: Put this in a file plugin_path/_map.rb. + # + # class MyColorHost < PluginHost + # map :navy => :dark_blue, + # :maroon => :brown, + # :luna => :moon + # end + def map hash + for from, to in hash + from = validate_id from + to = validate_id to + plugin_hash[from] = to unless plugin_hash.has_key? from + end + end + + # Define the default plugin to use when no plugin is found + # for a given id, or return the default plugin. + # + # See also map. + # + # class MyColorHost < PluginHost + # map :navy => :dark_blue + # default :gray + # end + # + # MyColorHost.default # loads and returns the Gray plugin + def default id = nil + if id + id = validate_id id + raise "The default plugin can't be named \"default\"." if id == :default + plugin_hash[:default] = id + else + load :default + end + end + + # Every plugin must register itself for +id+ by calling register_for, + # which calls this method. + # + # See Plugin#register_for. + def register plugin, id + plugin_hash[validate_id(id)] = plugin + end + + # A Hash of plugion_id => Plugin pairs. + def plugin_hash + @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } + end + + # Returns an array of all .rb files in the plugin path. + # + # The extension .rb is not included. + def list + Dir[path_to('*')].select do |file| + File.basename(file)[/^(?!_)\w+\.rb$/] + end.map do |file| + File.basename(file, '.rb').to_sym + end + end + + # Returns an array of all Plugins. + # + # Note: This loads all plugins using load_all. + def all_plugins + load_all + plugin_hash.values.grep(Class) + end + + # Loads the map file (see map). + # + # This is done automatically when plugin_path is called. + def load_plugin_map + mapfile = path_to '_map' + if File.exist? mapfile + require mapfile + true + else + false + end + end + + protected + + # Return a plugin hash that automatically loads plugins. + def make_plugin_hash + Hash.new do |h, plugin_id| + id = validate_id(plugin_id) + path = path_to id + begin + require path + rescue LoadError => boom + if h.has_key?(:default) + h[:default] + else + raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] + end + else + # Plugin should have registered by now + if h.has_key? id + h[id] + else + raise PluginNotFound, "No #{self.name} plugin for #{id.inspect} found in #{path}." + end + end + end + end + + # Returns the expected path to the plugin file for the given id. + def path_to plugin_id + File.join plugin_path, "#{plugin_id}.rb" + end + + # Converts +id+ to a valid plugin ID String, or returns +nil+. + # + # Raises +ArgumentError+ for all other objects, or if the + # given String includes non-alphanumeric characters (\W). + def validate_id id + case id + when Symbol + id.to_s + when String + if id[/\w+/] == id + id.downcase + else + raise ArgumentError, "Invalid id given: #{id}" + end + else + raise ArgumentError, "Symbol or String expected, but #{id.class} given." + end + end + + end + +end diff --git a/lib/coderay/scanners.rb b/lib/coderay/scanners.rb new file mode 100644 index 00000000..3c7e594d --- /dev/null +++ b/lib/coderay/scanners.rb @@ -0,0 +1,27 @@ +require 'strscan' + +module CodeRay + + autoload :WordList, coderay_path('helpers', 'word_list') + + # = Scanners + # + # This module holds the Scanner class and its subclasses. + # For example, the Ruby scanner is named CodeRay::Scanners::Ruby + # can be found in coderay/scanners/ruby. + # + # Scanner also provides methods and constants for the register + # mechanism and the [] method that returns the Scanner class + # belonging to the given lang. + # + # See PluginHost. + module Scanners + + extend PluginHost + plugin_path File.dirname(__FILE__), 'scanners' + + autoload :Scanner, CodeRay.coderay_path('scanners', 'scanner') + + end + +end diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb index 84b6e8ec..fb2f30db 100644 --- a/lib/coderay/scanners/c.rb +++ b/lib/coderay/scanners/c.rb @@ -37,7 +37,7 @@ class C < Scanner add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: protected diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index e61f56f4..cd4d0941 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -2,14 +2,14 @@ module CodeRay module Scanners # Scanner for C++. - # + # # Aliases: +cplusplus+, c++ class CPlusPlus < Scanner register_for :cpp file_extension 'cpp' title 'C++' - + #-- http://www.cppreference.com/wiki/keywords/start KEYWORDS = [ 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break', @@ -17,28 +17,30 @@ class CPlusPlus < Scanner 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else', 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new', 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return', - 'sizeof', 'static_cast', 'struct', 'switch', 'template', - 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', + 'sizeof', 'static_assert', 'static_cast', 'struct', 'switch', + 'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', 'while', 'xor', 'xor_eq', ] # :nodoc: - + PREDEFINED_TYPES = [ - 'bool', 'char', 'double', 'float', 'int', 'long', - 'short', 'signed', 'unsigned', 'wchar_t', 'string', + 'bool', 'char', 'char16_t', 'char32_t', 'double', 'float', + 'int', 'long', 'short', 'signed', 'unsigned', + 'wchar_t', 'string', ] # :nodoc: PREDEFINED_CONSTANTS = [ 'false', 'true', - 'EOF', 'NULL', + 'EOF', 'NULL', 'nullptr' ] # :nodoc: PREDEFINED_VARIABLES = [ 'this', ] # :nodoc: DIRECTIVES = [ - 'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator', - 'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void', - 'volatile', + 'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit', + 'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator', + 'override', 'private', 'protected', 'public', 'register', 'static', + 'thread_local', 'using', 'virtual', 'void', 'volatile', ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). @@ -47,10 +49,10 @@ class CPlusPlus < Scanner add(PREDEFINED_CONSTANTS, :predefined_constant) # :nodoc: ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + protected - + def scan_tokens encoder, options state = :initial @@ -107,7 +109,7 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -180,7 +182,7 @@ def scan_tokens encoder, options state = :initial end - + when :class_name_expected if match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) encoder.text_token match, :class @@ -194,7 +196,7 @@ def scan_tokens encoder, options state = :initial end - + else raise_inspect 'Unknown state', encoder diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 732f9c59..55d52397 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -25,7 +25,7 @@ module RE # :nodoc: HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/ - Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/ + Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/ Name = /#{NMChar}+/ Ident = /-?#{NMStart}#{NMChar}*/ AtKeyword = /@#{Ident}/ @@ -53,7 +53,7 @@ def setup end def scan_tokens encoder, options - states = Array(options[:state] || @state) + states = Array(options[:state] || @state).dup value_expected = @value_expected until eos? @@ -145,10 +145,10 @@ def scan_tokens encoder, options start = match[/^\w+\(/] encoder.text_token start, :delimiter if match[-1] == ?) - encoder.text_token match[start.size..-2], :content + encoder.text_token match[start.size..-2], :content if match.size > start.size + 1 encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if match.size > start.size end encoder.end_group :function diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb index 566bfa77..83ede9a5 100644 --- a/lib/coderay/scanners/debug.rb +++ b/lib/coderay/scanners/debug.rb @@ -1,9 +1,11 @@ +require 'set' + module CodeRay module Scanners # = Debug Scanner # - # Interprets the output of the Encoders::Debug encoder. + # Interprets the output of the Encoders::Debug encoder (basically the inverse function). class Debug < Scanner register_for :debug @@ -11,6 +13,11 @@ class Debug < Scanner protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options opened_tokens = [] @@ -21,16 +28,19 @@ def scan_tokens encoder, options encoder.text_token match, :space elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x) - kind = self[1].to_sym - match = self[2].gsub(/\\(.)/m, '\1') - unless TokenKinds.has_key? kind - kind = :error - match = matched + if @known_token_kinds.include? self[1] + encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym + else + encoder.text_token matched, :unknown end - encoder.text_token match, kind elsif match = scan(/ (\w+) ([<\[]) /x) - kind = self[1].to_sym + if @known_token_kinds.include? self[1] + kind = self[1].to_sym + else + kind = :unknown + end + opened_tokens << kind case self[2] when '<' diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index af0f7557..a2a6fccf 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -69,7 +69,7 @@ def scan_tokens encoder, options state = :added elsif match = scan(/\\ .*/) encoder.text_token match, :comment - elsif match = scan(/@@(?>[^@\n]*)@@/) + elsif match = scan(/@@(?>[^@\n]+)@@/) content_scanner.state = :initial unless match?(/\n\+/) content_scanner_entry_state = nil if check(/\n|$/) @@ -100,7 +100,7 @@ def scan_tokens encoder, options next elsif match = scan(/-/) deleted_lines_count += 1 - if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) + if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) deleted_lines = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) } inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/) ; scan(/.*/) } @@ -109,7 +109,7 @@ def scan_tokens encoder, options for deleted_line, inserted_line in deleted_lines.zip(inserted_lines) pre, deleted_part, inserted_part, post = diff deleted_line, inserted_line content_scanner_entry_state = content_scanner.state - deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new) + deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new) content_scanner.state = content_scanner_entry_state || :initial inserted_lines_tokenized << content_scanner.tokenize([pre, inserted_part, post], :tokens => Tokens.new) end @@ -212,7 +212,7 @@ def diff a, b # does not precede the leftmost one from the left. j = -1 j -= 1 while j >= j_min && a[j] == b[j] - return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j+1..-1] : '' + return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j + 1..-1] : '' end end diff --git a/lib/coderay/scanners/go.rb b/lib/coderay/scanners/go.rb new file mode 100644 index 00000000..99fdd638 --- /dev/null +++ b/lib/coderay/scanners/go.rb @@ -0,0 +1,208 @@ +module CodeRay +module Scanners + + class Go < Scanner + + register_for :go + file_extension 'go' + + # http://golang.org/ref/spec#Keywords + KEYWORDS = [ + 'break', 'default', 'func', 'interface', 'select', + 'case', 'defer', 'go', 'map', 'struct', + 'chan', 'else', 'goto', 'package', 'switch', + 'const', 'fallthrough', 'if', 'range', 'type', + 'continue', 'for', 'import', 'return', 'var', + ] # :nodoc: + + # http://golang.org/ref/spec#Types + PREDEFINED_TYPES = [ + 'bool', + 'uint8', 'uint16', 'uint32', 'uint64', + 'int8', 'int16', 'int32', 'int64', + 'float32', 'float64', + 'complex64', 'complex128', + 'byte', 'rune', 'string', 'error', + 'uint', 'int', 'uintptr', + ] # :nodoc: + + PREDEFINED_CONSTANTS = [ + 'nil', 'iota', + 'true', 'false', + ] # :nodoc: + + PREDEFINED_FUNCTIONS = %w[ + append cap close complex copy delete imag len + make new panic print println real recover + ] # :nodoc: + + IDENT_KIND = WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_TYPES, :predefined_type). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_FUNCTIONS, :predefined) # :nodoc: + + ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + + protected + + def scan_tokens encoder, options + + state = :initial + label_expected = true + case_expected = false + label_expected_before_preproc_line = nil + in_preproc_line = false + + until eos? + + case state + + when :initial + + if match = scan(/ \s+ | \\\n /x) + if in_preproc_line && match != "\\\n" && match.index(?\n) + in_preproc_line = false + case_expected = false + label_expected = label_expected_before_preproc_line + end + encoder.text_token match, :space + + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + encoder.text_token match, :comment + + elsif match = scan(/ ?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x) + if case_expected + label_expected = true if match == ':' + case_expected = false + end + encoder.text_token match, :operator + + elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) + kind = IDENT_KIND[match] + if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/) + kind = :label + label_expected = false + match << matched + else + label_expected = false + if kind == :keyword + case match + when 'case', 'default' + case_expected = true + end + end + end + encoder.text_token match, kind + + elsif match = scan(/L?"/) + encoder.begin_group :string + if match[0] == ?L + encoder.text_token 'L', :modifier + match = '"' + end + encoder.text_token match, :delimiter + state = :string + + elsif match = scan(/ ` ([^`]+)? (`)? /x) + encoder.begin_group :shell + encoder.text_token '`', :delimiter + encoder.text_token self[1], :content if self[1] + encoder.text_token self[2], :delimiter if self[2] + encoder.end_group :shell + + elsif match = scan(/ \# \s* if \s* 0 /x) + match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos? + encoder.text_token match, :comment + + elsif match = scan(/#[ \t]*(\w*)/) + encoder.text_token match, :preprocessor + in_preproc_line = true + label_expected_before_preproc_line = label_expected + state = :include_expected if self[1] == 'include' + + elsif match = scan(/ L?' (?: [^\'\n\\] | \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) )? '? /ox) + label_expected = false + encoder.text_token match, :char + + elsif match = scan(/\$/) + encoder.text_token match, :ident + + elsif match = scan(/-?\d*(\.\d*)?([eE][+-]?\d+)?i/) + label_expected = false + encoder.text_token match, :imaginary + + elsif match = scan(/-?0[xX][0-9A-Fa-f]+/) + label_expected = false + encoder.text_token match, :hex + + elsif match = scan(/-?(?:0[0-7]+)(?![89.eEfF])/) + label_expected = false + encoder.text_token match, :octal + + elsif match = scan(/-?(?:\d*\.\d+|\d+\.)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/) + label_expected = false + encoder.text_token match, :float + + elsif match = scan(/-?(?:\d+)(?![.eEfF])L?L?/) + label_expected = false + encoder.text_token match, :integer + + else + encoder.text_token getch, :error + + end + + when :string + if match = scan(/[^\\\n"]+/) + encoder.text_token match, :content + elsif match = scan(/"/) + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + label_expected = false + elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) + encoder.text_token match, :char + elsif match = scan(/ \\ /x) + encoder.text_token match, :error + elsif match = scan(/$/) + encoder.end_group :string + state = :initial + label_expected = false + else + raise_inspect "else case \" reached; %p not handled." % peek(1), encoder + end + + when :include_expected + if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/) + encoder.text_token match, :include + state = :initial + + elsif match = scan(/\s+/) + encoder.text_token match, :space + state = :initial if match.index ?\n + + else + state = :initial + + end + + else + raise_inspect 'Unknown state', encoder + + end + + end + + if state == :string + encoder.end_group :string + end + + encoder + end + + end + +end +end diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb index cf55daf2..c52ce8d3 100644 --- a/lib/coderay/scanners/groovy.rb +++ b/lib/coderay/scanners/groovy.rb @@ -22,8 +22,8 @@ class Groovy < Java add(GROOVY_MAGIC_VARIABLES, :local_variable) # :nodoc: ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8} - REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8} + REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x # :nodoc: # TODO: interpretation inside ', ", / STRING_CONTENT_PATTERN = { @@ -36,9 +36,12 @@ class Groovy < Java protected + def setup + @state = :initial + end + def scan_tokens encoder, options - - state = :initial + state = options[:state] || @state inline_block_stack = [] inline_block_paren_depth = nil string_delimiter = nil @@ -223,7 +226,7 @@ def scan_tokens encoder, options encoder.text_token match, :content # TODO: Shouldn't this be :error? elsif match = scan(/ \\ | \n /x) - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string encoder.text_token match, :error after_def = value_expected = false state = :initial @@ -243,7 +246,17 @@ def scan_tokens encoder, options end if [:multiline_string, :string, :regexp].include? state - encoder.end_group state + encoder.end_group state == :regexp ? :regexp : :string + end + + if options[:keep_state] + @state = state + end + + until inline_block_stack.empty? + state, = *inline_block_stack.pop + encoder.end_group :inline + encoder.end_group state == :regexp ? :regexp : :string end encoder diff --git a/lib/coderay/scanners/haml.rb b/lib/coderay/scanners/haml.rb index 5433790a..d516ba9e 100644 --- a/lib/coderay/scanners/haml.rb +++ b/lib/coderay/scanners/haml.rb @@ -75,7 +75,7 @@ def scan_tokens encoder, options tag = false - if match = scan(/%[\w:]+\/?/) + if match = scan(/%[-\w:]+\/?/) encoder.text_token match, :tag # if match = scan(/( +)(.+)/) # encoder.text_token self[1], :space diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 3ba3b795..ebe7b01d 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -1,13 +1,13 @@ module CodeRay module Scanners - + # HTML Scanner # # Alias: +xhtml+ # # See also: Scanners::XML class HTML < Scanner - + register_for :html KINDS_NOT_LOC = [ @@ -33,7 +33,8 @@ class HTML < Scanner ) IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil). - add(EVENT_ATTRIBUTES, :script) + add(EVENT_ATTRIBUTES, :script). + add(['style'], :style) ATTR_NAME = /[\w.:-]+/ # :nodoc: TAG_END = /\/?>/ # :nodoc: @@ -75,9 +76,14 @@ def setup def scan_java_script encoder, code if code && !code.empty? @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true - # encoder.begin_group :inline @java_script_scanner.tokenize code, :tokens => encoder - # encoder.end_group :inline + end + end + + def scan_css encoder, code, state = [:initial] + if code && !code.empty? + @css_scanner ||= Scanners::CSS.new '', :keep_tokens => true + @css_scanner.tokenize code, :tokens => encoder, :state => state end end @@ -99,7 +105,15 @@ def scan_tokens encoder, options case state when :initial - if match = scan(/|.*)/m) + if match = scan(//m) + encoder.text_token match[0..-4], :plain + encoder.text_token ']]>', :inline_delimiter + elsif match = scan(/.+/) + encoder.text_token match, :error + end + elsif match = scan(/|.*)/m) encoder.text_token match, :comment elsif match = scan(/|.*)|\]>/m) encoder.text_token match, :doctype @@ -110,7 +124,7 @@ def scan_tokens encoder, options elsif match = scan(/<\/[-\w.:]*>?/m) in_tag = nil encoder.text_token match, :tag - elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m) + elsif match = scan(/<(?:(script|style)|[-\w.:]+)(>)?/m) encoder.text_token match, :tag in_tag = self[1] if self[2] @@ -161,17 +175,21 @@ def scan_tokens encoder, options encoder.text_token match, :attribute_value state = :attribute elsif match = scan(/["']/) - if in_attribute == :script - encoder.begin_group :inline - encoder.text_token match, :inline_delimiter + if in_attribute == :script || in_attribute == :style + encoder.begin_group :string + encoder.text_token match, :delimiter if scan(/javascript:[ \t]*/) encoder.text_token matched, :comment end code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/) - scan_java_script encoder, code + if in_attribute == :script + scan_java_script encoder, code + else + scan_css encoder, code, [:block] + end match = scan(/["']/) - encoder.text_token match, :inline_delimiter if match - encoder.end_group :inline + encoder.text_token match, :delimiter if match + encoder.end_group :string state = :attribute in_attribute = nil else @@ -206,19 +224,23 @@ def scan_tokens encoder, options when :in_special_tag case in_tag - when 'script' + when 'script', 'style' encoder.text_token match, :space if match = scan(/[ \t]*\n/) if scan(/(\s*)|(.*))/m) code = self[2] || self[4] closing = self[3] encoder.text_token self[1], :comment else - code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/) + code = scan_until(/(?=(?:\n\s*)?<\/#{in_tag}>)|\z/) closing = false end unless code.empty? encoder.begin_group :inline - scan_java_script encoder, code + if in_tag == 'script' + scan_java_script encoder, code + else + scan_css encoder, code + end encoder.end_group :inline end encoder.text_token closing, :comment if closing diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index b282864a..7dd1919e 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -20,7 +20,7 @@ class Java < Scanner MAGIC_VARIABLES = %w[ this super ] # :nodoc: TYPES = %w[ boolean byte char class double enum float int interface long - short void + short void var ] << '[]' # :nodoc: because int[] should be highlighted as a type DIRECTIVES = %w[ abstract extends final implements native private protected public @@ -36,15 +36,15 @@ class Java < Scanner add(BuiltinTypes::List, :predefined_type). add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception). add(DIRECTIVES, :directive) # :nodoc: - + ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: STRING_CONTENT_PATTERN = { "'" => /[^\\']+/, '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } # :nodoc: - IDENT = /[a-zA-Z_][A-Za-z_0-9]*/ # :nodoc: + IDENT = RUBY_VERSION < '1.9' ? /[a-zA-Z_][A-Za-z_0-9]*/ : Regexp.new('[[[:alpha:]]_][[[:alnum:]]_]*') # :nodoc: protected diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 9eb0a0a1..8c13d4ff 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -40,8 +40,8 @@ class JavaScript < Scanner add(KEYWORDS, :keyword) # :nodoc: ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - REGEXP_ESCAPE = / [bBdDsSwW] /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: + REGEXP_ESCAPE = / [bBdDsSwW] /x # :nodoc: STRING_CONTENT_PATTERN = { "'" => /[^\\']+/, '"' => /[^\\"]+/, @@ -100,7 +100,6 @@ def scan_tokens encoder, options # TODO: scan over nested tags xml_scanner.tokenize match, :tokens => encoder value_expected = false - next elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x) value_expected = true diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 3754a9b4..b09970c2 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -26,6 +26,10 @@ def setup def scan_tokens encoder, options state = options[:state] || @state + if [:string, :key].include? state + encoder.begin_group state + end + until eos? case state diff --git a/lib/coderay/scanners/lua.rb b/lib/coderay/scanners/lua.rb new file mode 100644 index 00000000..81d7dae4 --- /dev/null +++ b/lib/coderay/scanners/lua.rb @@ -0,0 +1,280 @@ +# encoding: utf-8 + +module CodeRay +module Scanners + + # Scanner for the Lua[http://lua.org] programming lanuage. + # + # The language’s complete syntax is defined in + # {the Lua manual}[http://www.lua.org/manual/5.2/manual.html], + # which is what this scanner tries to conform to. + class Lua < Scanner + + register_for :lua + file_extension 'lua' + title 'Lua' + + # Keywords used in Lua. + KEYWORDS = %w[and break do else elseif end + for function goto if in + local not or repeat return + then until while + ] + + # Constants set by the Lua core. + PREDEFINED_CONSTANTS = %w[false true nil] + + # The expressions contained in this array are parts of Lua’s `basic' + # library. Although it’s not entirely necessary to load that library, + # it is highly recommended and one would have to provide own implementations + # of some of these expressions if one does not do so. They however aren’t + # keywords, neither are they constants, but nearly predefined, so they + # get tagged as `predefined' rather than anything else. + # + # This list excludes values of form `_UPPERCASE' because the Lua manual + # requires such identifiers to be reserved by Lua anyway and they are + # highlighted directly accordingly, without the need for specific + # identifiers to be listed here. + PREDEFINED_EXPRESSIONS = %w[ + assert collectgarbage dofile error getmetatable + ipairs load loadfile next pairs pcall print + rawequal rawget rawlen rawset select setmetatable + tonumber tostring type xpcall + ] + + # Automatic token kind selection for normal words. + IDENT_KIND = CodeRay::WordList.new(:ident). + add(KEYWORDS, :keyword). + add(PREDEFINED_CONSTANTS, :predefined_constant). + add(PREDEFINED_EXPRESSIONS, :predefined) + + protected + + # Scanner initialization. + def setup + @state = :initial + @brace_depth = 0 + end + + # CodeRay entry hook. Starts parsing. + def scan_tokens(encoder, options) + state = options[:state] || @state + brace_depth = @brace_depth + num_equals = nil + + until eos? + case state + + when :initial + if match = scan(/\-\-\[\=*\[/) #--[[ long (possibly multiline) comment ]] + num_equals = match.count("=") # Number must match for comment end + encoder.begin_group(:comment) + encoder.text_token(match, :delimiter) + state = :long_comment + + elsif match = scan(/--.*$/) # --Lua comment + encoder.text_token(match, :comment) + + elsif match = scan(/\[=*\[/) # [[ long (possibly multiline) string ]] + num_equals = match.count("=") # Number must match for string end + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + state = :long_string + + elsif match = scan(/::\s*[a-zA-Z_][a-zA-Z0-9_]+\s*::/) # ::goto_label:: + encoder.text_token(match, :label) + + elsif match = scan(/_[A-Z]+/) # _UPPERCASE are names reserved for Lua + encoder.text_token(match, :predefined) + + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # Normal letters (or letters followed by digits) + kind = IDENT_KIND[match] + + # Extra highlighting for entities following certain keywords + if kind == :keyword and match == "function" + state = :function_expected + elsif kind == :keyword and match == "goto" + state = :goto_label_expected + elsif kind == :keyword and match == "local" + state = :local_var_expected + end + + encoder.text_token(match, kind) + + elsif match = scan(/\{/) # Opening table brace { + encoder.begin_group(:map) + encoder.text_token(match, brace_depth >= 1 ? :inline_delimiter : :delimiter) + brace_depth += 1 + state = :map + + elsif match = scan(/\}/) # Closing table brace } + if brace_depth == 1 + brace_depth = 0 + encoder.text_token(match, :delimiter) + encoder.end_group(:map) + elsif brace_depth == 0 # Mismatched brace + encoder.text_token(match, :error) + else + brace_depth -= 1 + encoder.text_token(match, :inline_delimiter) + encoder.end_group(:map) + state = :map + end + + elsif match = scan(/["']/) # String delimiters " and ' + encoder.begin_group(:string) + encoder.text_token(match, :delimiter) + start_delim = match + state = :string + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h* \. \h+ (?:p[+\-]?\d+)? | \d*\.\d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :float) + + # ↓Prefix hex number ←|→ decimal number + elsif match = scan(/-? (?:0x\h+ (?:p[+\-]?\d+)? | \d+ (?:e[+\-]?\d+)?)/ix) # hexadecimal constants have no E power, decimal ones no P power + encoder.text_token(match, :integer) + + elsif match = scan(/[\+\-\*\/%^\#=~<>\(\)\[\]:;,] | \.(?!\d)/x) # Operators + encoder.text_token(match, :operator) + + elsif match = scan(/\s+/) # Space + encoder.text_token(match, :space) + + else # Invalid stuff. Note that Lua doesn’t accept multibyte chars outside of strings, hence these are also errors. + encoder.text_token(getch, :error) + end + + # It may be that we’re scanning a full-blown subexpression of a table + # (tables can contain full expressions in parts). + # If this is the case, return to :map scanning state. + state = :map if state == :initial && brace_depth >= 1 + + when :function_expected + if match = scan(/\(.*?\)/m) # x = function() # "Anonymous" function without explicit name + encoder.text_token(match, :operator) + state = :initial + elsif match = scan(/[a-zA-Z_] (?:[a-zA-Z0-9_\.] (?!\.\d))* [\.\:]/x) # function tbl.subtbl.foo() | function tbl:foo() # Colon only allowed as last separator + encoder.text_token(match, :ident) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) # function foo() + encoder.text_token(match, :function) + state = :initial + elsif match = scan(/\s+/) # Between the `function' keyword and the ident may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + state = :initial + end + + when :goto_label_expected + if match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :label) + state = :initial + elsif match = scan(/\s+/) # Between the `goto' keyword and the label may be any amount of whitespace + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :local_var_expected + if match = scan(/function/) # local function ... + encoder.text_token(match, :keyword) + state = :function_expected + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + encoder.text_token(match, :local_variable) + elsif match = scan(/,/) + encoder.text_token(match, :operator) + elsif match = scan(/\=/) + encoder.text_token(match, :operator) + # After encountering the equal sign, arbitrary expressions are + # allowed again, so just return to the main state for further + # parsing. + state = :initial + elsif match = scan(/\n/) + encoder.text_token(match, :space) + state = :initial + elsif match = scan(/\s+/) + encoder.text_token(match, :space) + else + encoder.text_token(getch, :error) + end + + when :long_comment + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) + encoder.text_token(match, :content) + + delim = scan(/\]={#{num_equals}}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:comment) + state = :initial + + when :long_string + if match = scan(/.*?(?=\]={#{num_equals}}\])/m) # Long strings do not interpret any escape sequences + encoder.text_token(match, :content) + + delim = scan(/\]={#{num_equals}}\]/) + encoder.text_token(delim, :delimiter) + else # No terminator found till EOF + encoder.text_token(rest, :error) + terminate + end + encoder.end_group(:string) + state = :initial + + when :string + if match = scan(/[^\\#{start_delim}\n]+/) # Everything except \ and the start delimiter character is string content (newlines are only allowed if preceeded by \ or \z) + encoder.text_token(match, :content) + elsif match = scan(/\\(?:['"abfnrtv\\]|z\s*|x\h\h|\d{1,3}|\n)/m) + encoder.text_token(match, :char) + elsif match = scan(Regexp.compile(start_delim)) + encoder.text_token(match, :delimiter) + encoder.end_group(:string) + state = :initial + elsif match = scan(/\n/) # Lua forbids unescaped newlines in normal non-long strings + encoder.text_token("\\n\n", :error) # Visually appealing error indicator--otherwise users may wonder whether the highlighter cannot highlight multine strings + encoder.end_group(:string) + state = :initial + else + encoder.text_token(getch, :error) + end + + when :map + if match = scan(/[,;]/) + encoder.text_token(match, :operator) + elsif match = scan(/[a-zA-Z_][a-zA-Z0-9_]* (?=\s*=)/x) + encoder.text_token(match, :key) + encoder.text_token(scan(/\s+/), :space) if check(/\s+/) + encoder.text_token(scan(/\=/), :operator) + state = :initial + elsif match = scan(/\s+/m) + encoder.text_token(match, :space) + else + # Note this clause doesn’t advance the scan pointer, it’s a kind of + # "retry with other options" (the :initial state then of course + # advances the pointer). + state = :initial + end + else + raise + end + + end + + if options[:keep_state] + @state = state + end + + encoder.end_group :string if [:string].include? state + brace_depth.times { encoder.end_group :map } + + encoder + end + + end + +end +end diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 6c688343..7a8d75d9 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -265,7 +265,7 @@ def scan_tokens encoder, options @html_scanner.tokenize match unless match.empty? end - when :php + when :php, :php_inline if match = scan(/\s+/) encoder.text_token match, :space @@ -332,7 +332,7 @@ def scan_tokens encoder, options if states.size == 1 encoder.text_token match, :error else - states.pop + state = states.pop if states.last.is_a?(::Array) delimiter = states.last[1] states[-1] = states.last[0] @@ -340,6 +340,7 @@ def scan_tokens encoder, options encoder.end_group :inline else encoder.text_token match, :operator + encoder.end_group :inline if state == :php_inline label_expected = true end end @@ -350,7 +351,14 @@ def scan_tokens encoder, options elsif match = scan(RE::PHP_END) encoder.text_token match, :inline_delimiter - states = [:initial] + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end + end + states << :initial elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o) encoder.begin_group :string @@ -400,6 +408,7 @@ def scan_tokens encoder, options elsif match = scan(/\\/) encoder.text_token match, :error else + encoder.end_group :string states.pop end @@ -459,7 +468,7 @@ def scan_tokens encoder, options encoder.begin_group :inline states[-1] = [states.last, delimiter] delimiter = nil - states.push :php + states.push :php_inline encoder.text_token match, :delimiter else encoder.text_token match, :content @@ -469,6 +478,7 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :content else + encoder.end_group :string states.pop end @@ -500,6 +510,14 @@ def scan_tokens encoder, options end + while state = states.pop + encoder.end_group :string if [:sqstring, :dqstring].include? state + if state.is_a? Array + encoder.end_group :inline + encoder.end_group :string if [:sqstring, :dqstring].include? state.first + end + end + encoder end diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index 09c8b6e7..5da553a6 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -63,7 +63,7 @@ class Python < Scanner NAME = / [[:alpha:]_] \w* /x # :nodoc: ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: - UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x # :nodoc: + UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x # :nodoc: OPERATOR = / \.\.\. | # ellipsis diff --git a/lib/coderay/scanners/raydebug.rb b/lib/coderay/scanners/raydebug.rb index 7a21354c..1effdc85 100644 --- a/lib/coderay/scanners/raydebug.rb +++ b/lib/coderay/scanners/raydebug.rb @@ -1,23 +1,30 @@ +require 'set' + module CodeRay module Scanners - - # = Debug Scanner + + # = Raydebug Scanner # - # Parses the output of the Encoders::Debug encoder. + # Highlights the output of the Encoders::Debug encoder. class Raydebug < Scanner - + register_for :raydebug file_extension 'raydebug' title 'CodeRay Token Dump' protected + def setup + super + @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set + end + def scan_tokens encoder, options - + opened_tokens = [] - + until eos? - + if match = scan(/\s+/) encoder.text_token match, :space @@ -26,20 +33,22 @@ def scan_tokens encoder, options encoder.text_token kind, :class encoder.text_token '(', :operator match = self[2] - encoder.text_token match, kind.to_sym + unless match.empty? + if @known_token_kinds.include? kind + encoder.text_token match, kind.to_sym + else + encoder.text_token match, :plain + end + end encoder.text_token match, :operator if match = scan(/\)/) elsif match = scan(/ (\w+) ([<\[]) /x) - kind = self[1] - case self[2] - when '<' - encoder.text_token kind, :class - when '[' - encoder.text_token kind, :class + encoder.text_token self[1], :class + if @known_token_kinds.include? self[1] + kind = self[1].to_sym else - raise 'CodeRay bug: This case should not be reached.' + kind = :unknown end - kind = kind.to_sym opened_tokens << kind encoder.begin_group kind encoder.text_token self[2], :operator @@ -59,8 +68,8 @@ def scan_tokens encoder, options encoder end - + end - + end end diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index c282f31b..5b8de42f 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -164,15 +164,19 @@ def scan_tokens encoder, options end elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx) - encoder.begin_group :string if match.size == 1 + kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string + encoder.begin_group kind encoder.text_token match, :delimiter - state = self.class::StringState.new :string, match == '"', match # important for streaming + state = self.class::StringState.new kind, match == '"', match # important for streaming else + kind = value_expected == true && scan(/:/) ? :key : :string + encoder.begin_group kind encoder.text_token match[0,1], :delimiter encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter - encoder.end_group :string + encoder.end_group kind + encoder.text_token ':', :operator if kind == :key value_expected = false end @@ -191,11 +195,14 @@ def scan_tokens encoder, options encoder.text_token match, :error method_call_expected = false else - encoder.text_token match, self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + kind = self[1] ? :float : :integer # TODO: send :hex/:octal/:binary + match << 'r' if match !~ /e/i && scan(/r/) + match << 'i' if scan(/i/) + encoder.text_token match, kind end value_expected = false - elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x) + elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x) value_expected = true encoder.text_token match, :operator @@ -208,7 +215,7 @@ def scan_tokens encoder, options encoder.end_group kind heredocs ||= [] # create heredocs if empty heredocs << self.class::StringState.new(kind, quote != "'", delim, - self[1] == '-' ? :indented : :linestart) + self[1] ? :indented : :linestart) value_expected = false elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o) @@ -269,7 +276,7 @@ def scan_tokens encoder, options end if last_state - state = last_state + state = last_state unless state.is_a?(StringState) # otherwise, a simple 'def"' results in unclosed tokens last_state = nil end diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index ed071d2b..cd942d0d 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -60,7 +60,7 @@ module Ruby::Patterns # :nodoc: all QUOTE_TO_TYPE = { '`' => :shell, - '/'=> :regexp, + '/' => :regexp, } QUOTE_TO_TYPE.default = :string @@ -114,7 +114,7 @@ module Ruby::Patterns # :nodoc: all # NOTE: This is not completely correct, but # nobody needs heredoc delimiters ending with \n. HEREDOC_OPEN = / - << (-)? # $1 = float + << ([-~])? # $1 = float (?: ( [A-Za-z_0-9]+ ) # $2 = delim | @@ -157,13 +157,16 @@ module Ruby::Patterns # :nodoc: all yield ]) - FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x + FANCY_STRING_START = / % ( [iIqQrswWx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x FANCY_STRING_KIND = Hash.new(:string).merge({ + 'i' => :symbol, + 'I' => :symbol, 'r' => :regexp, 's' => :symbol, 'x' => :shell, }) FANCY_STRING_INTERPRETED = Hash.new(true).merge({ + 'i' => false, 'q' => false, 's' => false, 'w' => false, diff --git a/lib/coderay/scanners/ruby/string_state.rb b/lib/coderay/scanners/ruby/string_state.rb index 2f398d1e..95f1e832 100644 --- a/lib/coderay/scanners/ruby/string_state.rb +++ b/lib/coderay/scanners/ruby/string_state.rb @@ -16,7 +16,6 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, STRING_PATTERN = Hash.new do |h, k| delim, interpreted = *k - # delim = delim.dup # workaround for old Ruby delim_pattern = Regexp.escape(delim) if closing_paren = CLOSING_PAREN[delim] delim_pattern << Regexp.escape(closing_paren) @@ -29,12 +28,21 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc, # '| [|?*+(){}\[\].^$]' # end - h[k] = - if interpreted && delim != '#' - / (?= [#{delim_pattern}] | \# [{$@] ) /mx - else - / (?= [#{delim_pattern}] ) /mx - end + if interpreted && delim != '#' + / (?= [#{delim_pattern}] | \# [{$@] ) /mx + else + / (?= [#{delim_pattern}] ) /mx + end.tap do |pattern| + h[k] = pattern if (delim.respond_to?(:ord) ? delim.ord : delim[0]) < 256 + end + end + + def self.simple_key_pattern delim + if delim == "'" + / (?> (?: [^\\']+ | \\. )* ) ' : /mx + else + / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx + end end def initialize kind, interpreted, delim, heredoc = false diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb index 167051d0..e3296b90 100644 --- a/lib/coderay/scanners/sass.rb +++ b/lib/coderay/scanners/sass.rb @@ -7,11 +7,6 @@ class Sass < CSS register_for :sass file_extension 'sass' - STRING_CONTENT_PATTERN = { - "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, - } - protected def setup @@ -19,8 +14,9 @@ def setup end def scan_tokens encoder, options - states = Array(options[:state] || @state) - string_delimiter = nil + states = Array(options[:state] || @state).dup + + encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring until eos? @@ -48,7 +44,7 @@ def scan_tokens encoder, options elsif case states.last when :initial, :media, :sass_inline if match = scan(/(?>#{RE::Ident})(?!\()/ox) - encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag) + encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag) next elsif !value_expected && (match = scan(/\*/)) encoder.text_token match, :tag @@ -91,24 +87,23 @@ def scan_tokens encoder, options next end - when :string - if match = scan(STRING_CONTENT_PATTERN[string_delimiter]) + when :sqstring, :dqstring + if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o) encoder.text_token match, :content elsif match = scan(/['"]/) encoder.text_token match, :delimiter encoder.end_group :string - string_delimiter = nil states.pop elsif match = scan(/#\{/) encoder.begin_group :inline encoder.text_token match, :inline_delimiter states.push :sass_inline elsif match = scan(/ \\ | $ /x) - encoder.end_group :string + encoder.end_group states.last encoder.text_token match, :error unless match.empty? states.pop else - raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder + raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder end when :include @@ -119,7 +114,7 @@ def scan_tokens encoder, options else #:nocov: - raise_inspect 'Unknown state', encoder + raise_inspect 'Unknown state: %p' % [states.last], encoder #:nocov: end @@ -157,15 +152,15 @@ def scan_tokens encoder, options elsif match = scan(/['"]/) encoder.begin_group :string - string_delimiter = match encoder.text_token match, :delimiter if states.include? :sass_inline - content = scan_until(/(?=#{string_delimiter}|\}|\z)/) + # no nesting, just scan the string until delimiter + content = scan_until(/(?=#{match}|\}|\z)/) encoder.text_token content, :content unless content.empty? - encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/) + encoder.text_token match, :delimiter if scan(/#{match}/) encoder.end_group :string else - states.push :string + states.push match == "'" ? :sqstring : :dqstring end elsif match = scan(/#{RE::Function}/o) @@ -176,7 +171,7 @@ def scan_tokens encoder, options encoder.text_token match[start.size..-2], :content encoder.text_token ')', :delimiter else - encoder.text_token match[start.size..-1], :content + encoder.text_token match[start.size..-1], :content if start.size < match.size end encoder.end_group :function @@ -195,7 +190,7 @@ def scan_tokens encoder, options elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) encoder.text_token match, :color - elsif match = scan(/@else if\b|#{RE::AtKeyword}/) + elsif match = scan(/@else if\b|#{RE::AtKeyword}/o) encoder.text_token match, :directive value_expected = true @@ -214,8 +209,18 @@ def scan_tokens encoder, options end + states.pop if states.last == :include + if options[:keep_state] - @state = states + @state = states.dup + end + + while state = states.pop + if state == :sass_inline + encoder.end_group :inline + elsif state == :sqstring || state == :dqstring + encoder.end_group :string + end end encoder diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanners/scanner.rb similarity index 94% rename from lib/coderay/scanner.rb rename to lib/coderay/scanners/scanner.rb index b3f7e175..efa710d9 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanners/scanner.rb @@ -1,25 +1,7 @@ # encoding: utf-8 -require 'strscan' module CodeRay - - autoload :WordList, coderay_path('helpers', 'word_list') - - # = Scanners - # - # This module holds the Scanner class and its subclasses. - # For example, the Ruby scanner is named CodeRay::Scanners::Ruby - # can be found in coderay/scanners/ruby. - # - # Scanner also provides methods and constants for the register - # mechanism and the [] method that returns the Scanner class - # belonging to the given lang. - # - # See PluginHost. module Scanners - extend PluginHost - plugin_path File.dirname(__FILE__), 'scanners' - # = Scanner # diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index b7572789..c8725a8f 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -1,8 +1,9 @@ -module CodeRay module Scanners +module CodeRay +module Scanners # by Josh Goebel class SQL < Scanner - + register_for :sql KEYWORDS = %w( @@ -28,7 +29,7 @@ class SQL < Scanner char varchar varchar2 enum binary text tinytext mediumtext longtext blob tinyblob mediumblob longblob timestamp date time datetime year double decimal float int - integer tinyint mediumint bigint smallint unsigned bit + integer tinyint mediumint bigint smallint unsigned bit numeric bool boolean hex bin oct ) @@ -56,6 +57,12 @@ class SQL < Scanner STRING_PREFIXES = /[xnb]|_\w+/i + STRING_CONTENT_PATTERN = { + '"' => / (?: [^\\"] | "" )+ /x, + "'" => / (?: [^\\'] | '' )+ /x, + '`' => / (?: [^\\`] | `` )+ /x, + } + def scan_tokens encoder, options state = :initial @@ -89,7 +96,7 @@ def scan_tokens encoder, options state = :string encoder.text_token match, :delimiter - elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x) + elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x) encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match]) name_expected = false @@ -114,41 +121,28 @@ def scan_tokens encoder, options end elsif state == :string - if match = scan(/[^\\"'`]+/) - string_content << match - next + if match = scan(STRING_CONTENT_PATTERN[string_type]) + encoder.text_token match, :content elsif match = scan(/["'`]/) if string_type == match if peek(1) == string_type # doubling means escape - string_content << string_type << getch - next - end - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' + encoder.text_token match + getch, :content + else + encoder.text_token match, :delimiter + encoder.end_group :string + state = :initial + string_type = nil end - encoder.text_token match, :delimiter - encoder.end_group :string - state = :initial - string_type = nil else - string_content << match + encoder.text_token match, :content end elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :char elsif match = scan(/ \\ . /mox) - string_content << match - next + encoder.text_token match, :content elsif match = scan(/ \\ | $ /x) - unless string_content.empty? - encoder.text_token string_content, :content - string_content = '' - end encoder.text_token match, :error unless match.empty? + encoder.end_group :string state = :initial else raise "else case \" reached; %p not handled." % peek(1), encoder @@ -171,4 +165,5 @@ def scan_tokens encoder, options end -end end \ No newline at end of file +end +end diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb index 96f4e93f..32c8e2cb 100644 --- a/lib/coderay/scanners/yaml.rb +++ b/lib/coderay/scanners/yaml.rb @@ -47,7 +47,7 @@ def scan_tokens encoder, options when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/) encoder.begin_group :string encoder.text_token match, :delimiter - encoder.text_token match, :content if match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx) + encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty? encoder.text_token match, :delimiter if match = scan(/"/) encoder.end_group :string next @@ -84,7 +84,7 @@ def scan_tokens encoder, options when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/) encoder.begin_group :key encoder.text_token match[0,1], :delimiter - encoder.text_token match[1..-2], :content + encoder.text_token match[1..-2], :content if match.size > 2 encoder.text_token match[-1,1], :delimiter encoder.end_group :key key_indent = column(pos - match.size) - 1 diff --git a/lib/coderay/styles.rb b/lib/coderay/styles.rb new file mode 100644 index 00000000..d8fa8aa7 --- /dev/null +++ b/lib/coderay/styles.rb @@ -0,0 +1,15 @@ +module CodeRay + + # This module holds the Style class and its subclasses. + # + # See Plugin. + module Styles + + extend PluginHost + plugin_path File.dirname(__FILE__), 'styles' + + autoload :Style, CodeRay.coderay_path('styles', 'style') + + end + +end diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index f4d07f11..f21cefed 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -3,14 +3,14 @@ module Styles # A colorful theme using CSS 3 colors (with alpha channel). class Alpha < Style - + register_for :alpha - + code_background = 'hsl(0,0%,95%)' numbers_background = 'hsl(180,65%,90%)' border_color = 'silver' normal_color = 'black' - + CSS_MAIN_STYLES = <<-MAIN # :nodoc: .CodeRay { background-color: #{code_background}; @@ -82,7 +82,8 @@ class Alpha < Style .exception { color:#C00; font-weight:bold } .float { color:#60E } .function { color:#06B; font-weight:bold } -.function .delimiter { color:#024; font-weight:bold } +.function .delimiter { color:#059 } +.function .content { color:#037 } .global-variable { color:#d70 } .hex { color:#02b } .id { color:#33D; font-weight:bold } @@ -98,13 +99,16 @@ class Alpha < Style .key .delimiter { color: #404 } .keyword { color:#080; font-weight:bold } .label { color:#970; font-weight:bold } -.local-variable { color:#963 } +.local-variable { color:#950 } +.map .content { color:#808 } +.map .delimiter { color:#40A} +.map { background-color:hsla(200,100%,50%,0.06); } .namespace { color:#707; font-weight:bold } .octal { color:#40E } .operator { } .predefined { color:#369; font-weight:bold } .predefined-constant { color:#069 } -.predefined-type { color:#0a5; font-weight:bold } +.predefined-type { color:#0a8; font-weight:bold } .preprocessor { color:#579 } .pseudo-class { color:#00C; font-weight:bold } .regexp { background-color:hsla(300,100%,50%,0.06); } @@ -122,7 +126,7 @@ class Alpha < Style .string .modifier { color: #E40 } .symbol { color:#A60 } .symbol .content { color:#A60 } -.symbol .delimiter { color:#630 } +.symbol .delimiter { color:#740 } .tag { color:#070; font-weight:bold } .type { color:#339; font-weight:bold } .value { color: #088 } @@ -142,8 +146,8 @@ class Alpha < Style .change .change { color: #88f } .head .head { color: #f4f } TOKENS - + end - + end end diff --git a/lib/coderay/style.rb b/lib/coderay/styles/style.rb similarity index 64% rename from lib/coderay/style.rb rename to lib/coderay/styles/style.rb index df4704f4..a3353861 100644 --- a/lib/coderay/style.rb +++ b/lib/coderay/styles/style.rb @@ -1,11 +1,6 @@ module CodeRay - - # This module holds the Style class and its subclasses. - # - # See Plugin. + module Styles - extend PluginHost - plugin_path File.dirname(__FILE__), 'styles' # Base class for styles. # diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb old mode 100755 new mode 100644 index 91546580..f9118622 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -1,10 +1,7 @@ module CodeRay # A Hash of all known token kinds and their associated CSS classes. - TokenKinds = Hash.new do |h, k| - warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG - false - end + TokenKinds = Hash.new(false) # speedup TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity @@ -51,6 +48,7 @@ module CodeRay :keyword => 'keyword', # reserved word that's actually implemented; most scanners :label => 'label', # C, PHP :local_variable => 'local-variable', # local and magic variables; some scanners + :map => 'map', # Lua tables :modifier => 'modifier', # used inside on strings; lots of scanners :namespace => 'namespace', # Clojure, Java, Taskpaper :octal => 'octal', # lots of scanners @@ -82,5 +80,6 @@ module CodeRay :plain => false # almost all scanners ) - TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:method] = TokenKinds[:function] + TokenKinds[:unknown] = TokenKinds[:plain] end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index 54358d4e..b5f78e71 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -3,17 +3,16 @@ module CodeRay # The Tokens class represents a list of tokens returned from # a Scanner. It's actually just an Array with a few helper methods. # - # A token itself is not a special object, just a two-element Array - # consisting of + # A token itself is not a special object, just two elements in an Array: # * the _token_ _text_ (the original source of the token in a String) or # a _token_ _action_ (begin_group, end_group, begin_line, end_line) # * the _token_ _kind_ (a Symbol representing the type of the token) # # It looks like this: # - # ['# It looks like this', :comment] - # ['3.1415926', :float] - # ['$^', :error] + # ..., '# It looks like this', :comment, ... + # ..., '3.1415926', :float, ... + # ..., '$^', :error, ... # # Some scanners also yield sub-tokens, represented by special # token actions, for example :begin_group and :end_group. @@ -21,11 +20,11 @@ module CodeRay # The Ruby scanner, for example, splits "a string" into: # # [ - # [:begin_group, :string], - # ['"', :delimiter], - # ['a string', :content], - # ['"', :delimiter], - # [:end_group, :string] + # :begin_group, :string, + # '"', :delimiter, + # 'a string', :content, + # '"', :delimiter, + # :end_group, :string # ] # # Tokens can be used to save the output of a Scanners in a simple @@ -40,6 +39,9 @@ module CodeRay # You can serialize it to a JSON string and store it in a database, pass it # around to encode it more than once, send it to other algorithms... class Tokens < Array + # Remove Array#filter that is a new alias for Array#select on Ruby 2.6, + # for method_missing called with filter method. + undef_method :filter if instance_methods.include?(:filter) # The Scanner instance that created the tokens. attr_accessor :scanner diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 4b4f0850..3c68bd83 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.0' + VERSION = '1.1.3' end diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake index 040951b5..8edeffb0 100644 --- a/rake_tasks/benchmark.rake +++ b/rake_tasks/benchmark.rake @@ -1,7 +1,6 @@ desc 'Do a benchmark' task :benchmark do - ruby "-v" - ruby "-wIlib bench/bench.rb ruby div 3000 N5" + ruby 'bench/bench.rb ruby html' end task :bench => :benchmark diff --git a/rake_tasks/code_statistics.rb b/rake_tasks/code_statistics.rb index 0a2016bd..32eb3f06 100644 --- a/rake_tasks/code_statistics.rb +++ b/rake_tasks/code_statistics.rb @@ -156,7 +156,7 @@ def print_code_test_stats code = calculate_code tests = calculate_tests - puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f/code)}]" + puts " Code LOC = #{code} Test LOC = #{tests} Code:Test Ratio = [1 : #{sprintf("%.2f", tests.to_f / code)}]" puts "" end diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index 371214a2..e72c96b2 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -1,13 +1,8 @@ namespace :test do - desc 'run all sample tests' - task :samples do - ruby './test/samples/suite.rb' - end - desc 'run functional tests' task :functional do ruby './test/functional/suite.rb' - ruby './test/functional/for_redcloth.rb' + ruby './test/functional/for_redcloth.rb' unless (''.chop! rescue true) end desc 'run unit tests' @@ -42,7 +37,7 @@ Please rename or remove it and run again to use the GitHub repository: else puts 'Downloading scanner test suite...' sh 'git clone https://github.com/rubychan/coderay-scanner-tests.git test/scanners/' - end + end unless ENV['SKIP_UPDATE_SCANNER_SUITE'] end namespace :scanner do @@ -84,5 +79,9 @@ Please rename or remove it and run again to use the GitHub repository: end end -task :test => %w(test:functional test:units test:exe) -task :samples => 'test:samples' \ No newline at end of file +if RUBY_VERSION >= '1.9' + require 'rspec/core/rake_task' + RSpec::Core::RakeTask.new(:spec) +end + +task :test => %w(test:functional test:units test:exe spec) diff --git a/spec/coderay_spec.rb b/spec/coderay_spec.rb new file mode 100644 index 00000000..88c9aece --- /dev/null +++ b/spec/coderay_spec.rb @@ -0,0 +1,35 @@ +require File.expand_path('../spec_helper', __FILE__) + +RSpec.describe CodeRay do + describe '::VERSION' do + it "returns the Gem's version" do + expect(CodeRay::VERSION).to match(/\A\d\.\d\.\d?\z/) + end + end + + describe '.coderay_path' do + it 'returns an absolute file path to the given code file' do + base = File.expand_path('../..', __FILE__) + expect(CodeRay.coderay_path('file')).to eq("#{base}/lib/coderay/file") + end + end + + describe '.scan' do + let(:code) { 'puts "Hello, World!"' } + let(:tokens) do + [ + ['puts', :ident], + [' ', :space], + [:begin_group, :string], + ['"', :delimiter], + ['Hello, World!', :content], + ['"', :delimiter], + [:end_group, :string] + ].flatten + end + + it 'returns tokens' do + expect(CodeRay.scan(code, :ruby).tokens).to eq(tokens) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..4e2dac6e --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,105 @@ +require 'simplecov' + +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end if RUBY_VERSION >= '1.9' + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # This setting enables warnings. It's recommended, but in some cases may + # be too noisy due to issues in dependencies. + config.warnings = true + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end + +$:.unshift File.expand_path('../lib', __FILE__) +require 'coderay' diff --git a/test/executable/suite.rb b/test/executable/suite.rb index 997405ca..a6f40972 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -1,3 +1,4 @@ +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' require 'rubygems' unless defined? Gem require 'shoulda-context' diff --git a/test/functional/basic.rb b/test/functional/basic.rb old mode 100755 new mode 100644 index 3053b543..059d56c3 --- a/test/functional/basic.rb +++ b/test/functional/basic.rb @@ -97,7 +97,7 @@ def test_comment_filter code more code - EXPECTED + EXPECTED #!/usr/bin/env ruby =begin A multi-line comment. @@ -105,7 +105,7 @@ def test_comment_filter code # A single-line comment. more code # and another comment, in-line. - INPUT + INPUT end def test_lines_of_code @@ -117,7 +117,7 @@ def test_lines_of_code code # A single-line comment. more code # and another comment, in-line. - INPUT + INPUT rHTML = <<-RHTML @@ -138,7 +138,7 @@ def test_lines_of_code - RHTML + RHTML assert_equal 0, CodeRay.scan(rHTML, :html).lines_of_code assert_equal 0, CodeRay.scan(rHTML, :php).lines_of_code assert_equal 0, CodeRay.scan(rHTML, :yaml).lines_of_code @@ -164,9 +164,7 @@ def test_token_kinds end end assert_equal 'reserved', CodeRay::TokenKinds[:reserved] - assert_warning 'Undefined Token kind: :shibboleet' do - assert_equal false, CodeRay::TokenKinds[:shibboleet] - end + assert_equal false, CodeRay::TokenKinds[:shibboleet] end class Milk < CodeRay::Encoders::Encoder diff --git a/test/functional/examples.rb b/test/functional/examples.rb old mode 100755 new mode 100644 diff --git a/test/functional/for_redcloth.rb b/test/functional/for_redcloth.rb index 9fd244ed..32a1a1b3 100644 --- a/test/functional/for_redcloth.rb +++ b/test/functional/for_redcloth.rb @@ -1,3 +1,4 @@ +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' $:.unshift File.expand_path('../../../lib', __FILE__) @@ -22,7 +23,7 @@ def test_for_redcloth 
puts "Hello, World!"
- BLOCKCODE + BLOCKCODE RedCloth.new('bc[ruby]. puts "Hello, World!"').to_html end @@ -32,7 +33,7 @@ def test_for_redcloth_no_lang RedCloth.new('@puts "Hello, World!"@').to_html assert_equal <<-BLOCKCODE.chomp, 
puts \"Hello, World!\"
- BLOCKCODE + BLOCKCODE RedCloth.new('bc. puts "Hello, World!"').to_html end @@ -40,7 +41,7 @@ def test_for_redcloth_style require 'coderay/for_redcloth' assert_equal <<-BLOCKCODE.chomp, 
puts \"Hello, World!\"
- BLOCKCODE + BLOCKCODE RedCloth.new('bc{color: red}. puts "Hello, World!"').to_html end @@ -52,7 +53,7 @@ def test_for_redcloth_escapes 
&
- BLOCKCODE + BLOCKCODE RedCloth.new('bc[ruby]. &').to_html end diff --git a/test/functional/suite.rb b/test/functional/suite.rb old mode 100755 new mode 100644 index ec23eec0..2bbc29c5 --- a/test/functional/suite.rb +++ b/test/functional/suite.rb @@ -1,3 +1,4 @@ +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' $VERBOSE = $CODERAY_DEBUG = true diff --git a/test/samples/README b/test/samples/README deleted file mode 100644 index b0ab6e44..00000000 --- a/test/samples/README +++ /dev/null @@ -1 +0,0 @@ -These demos rely on Ruby 1.8.5, so the tests might fail on other versions. \ No newline at end of file diff --git a/test/samples/cache.actual b/test/samples/cache.actual deleted file mode 100644 index c131857f..00000000 --- a/test/samples/cache.actual +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/test/samples/cache.expected b/test/samples/cache.expected deleted file mode 100644 index c131857f..00000000 --- a/test/samples/cache.expected +++ /dev/null @@ -1,2 +0,0 @@ -test <test> -test <test> diff --git a/test/samples/cache.rb b/test/samples/cache.rb deleted file mode 100644 index 0c0b8478..00000000 --- a/test/samples/cache.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'coderay' - -html_encoder = CodeRay.encoder :html - -scanner = Hash.new do |h, lang| - h[lang] = CodeRay.scanner lang -end - -for lang in [:ruby, :html] - tokens = scanner[lang].tokenize 'test ' - puts html_encoder.encode_tokens(tokens) -end diff --git a/test/samples/count.expected b/test/samples/count.expected deleted file mode 100644 index 7f493b6c..00000000 --- a/test/samples/count.expected +++ /dev/null @@ -1 +0,0 @@ -2 out of 4 tokens have the kind :integer. diff --git a/test/samples/count.rb b/test/samples/count.rb deleted file mode 100644 index bcb7c2dc..00000000 --- a/test/samples/count.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'coderay' - -stats = CodeRay.encoder(:statistic) -stats.encode("puts 17 + 4\n", :ruby) - -puts '%d out of %d tokens have the kind :integer.' % [ - stats.type_stats[:integer].count, - stats.real_token_count -] -#-> 2 out of 4 tokens have the kind :integer. diff --git a/test/samples/css.actual b/test/samples/css.actual deleted file mode 100644 index be73a7f9..00000000 --- a/test/samples/css.actual +++ /dev/null @@ -1,127 +0,0 @@ -.CodeRay { - background-color: hsl(0,0%,95%); - border: 1px solid silver; - color: black; -} -.CodeRay pre { - margin: 0px; -} - -span.CodeRay { white-space: pre; border: 0px; padding: 2px; } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } -table.CodeRay td { padding: 2px 4px; vertical-align: top; } - -.CodeRay .line-numbers { - background-color: hsl(180,65%,90%); - color: gray; - text-align: right; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; -} -.CodeRay .line-numbers a { - background-color: hsl(180,65%,90%) !important; - color: gray !important; - text-decoration: none !important; -} -.CodeRay .line-numbers pre { - word-break: normal; -} -.CodeRay .line-numbers a:target { color: blue !important; } -.CodeRay .line-numbers .highlighted { color: red !important; } -.CodeRay .line-numbers .highlighted a { color: red !important; } -.CodeRay span.line-numbers { padding: 0px 4px; } -.CodeRay .line { display: block; float: left; width: 100%; } -.CodeRay .code { width: 100%; } - -.CodeRay .debug { color: white !important; background: blue !important; } - -.CodeRay .annotation { color:#007 } -.CodeRay .attribute-name { color:#b48 } -.CodeRay .attribute-value { color:#700 } -.CodeRay .binary { color:#549 } -.CodeRay .binary .char { color:#325 } -.CodeRay .binary .delimiter { color:#325 } -.CodeRay .char { color:#D20 } -.CodeRay .char .content { color:#D20 } -.CodeRay .char .delimiter { color:#710 } -.CodeRay .class { color:#B06; font-weight:bold } -.CodeRay .class-variable { color:#369 } -.CodeRay .color { color:#0A0 } -.CodeRay .comment { color:#777 } -.CodeRay .comment .char { color:#444 } -.CodeRay .comment .delimiter { color:#444 } -.CodeRay .constant { color:#036; font-weight:bold } -.CodeRay .decorator { color:#B0B } -.CodeRay .definition { color:#099; font-weight:bold } -.CodeRay .delimiter { color:black } -.CodeRay .directive { color:#088; font-weight:bold } -.CodeRay .docstring { color:#D42; } -.CodeRay .doctype { color:#34b } -.CodeRay .done { text-decoration: line-through; color: gray } -.CodeRay .entity { color:#800; font-weight:bold } -.CodeRay .error { color:#F00; background-color:#FAA } -.CodeRay .escape { color:#666 } -.CodeRay .exception { color:#C00; font-weight:bold } -.CodeRay .float { color:#60E } -.CodeRay .function { color:#06B; font-weight:bold } -.CodeRay .function .delimiter { color:#024; font-weight:bold } -.CodeRay .global-variable { color:#d70 } -.CodeRay .hex { color:#02b } -.CodeRay .id { color:#33D; font-weight:bold } -.CodeRay .include { color:#B44; font-weight:bold } -.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.CodeRay .inline-delimiter { font-weight: bold; color: #666 } -.CodeRay .instance-variable { color:#33B } -.CodeRay .integer { color:#00D } -.CodeRay .imaginary { color:#f00 } -.CodeRay .important { color:#D00 } -.CodeRay .key { color: #606 } -.CodeRay .key .char { color: #60f } -.CodeRay .key .delimiter { color: #404 } -.CodeRay .keyword { color:#080; font-weight:bold } -.CodeRay .label { color:#970; font-weight:bold } -.CodeRay .local-variable { color:#963 } -.CodeRay .namespace { color:#707; font-weight:bold } -.CodeRay .octal { color:#40E } -.CodeRay .operator { } -.CodeRay .predefined { color:#369; font-weight:bold } -.CodeRay .predefined-constant { color:#069 } -.CodeRay .predefined-type { color:#0a5; font-weight:bold } -.CodeRay .preprocessor { color:#579 } -.CodeRay .pseudo-class { color:#00C; font-weight:bold } -.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } -.CodeRay .regexp .content { color:#808 } -.CodeRay .regexp .delimiter { color:#404 } -.CodeRay .regexp .modifier { color:#C2C } -.CodeRay .reserved { color:#080; font-weight:bold } -.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } -.CodeRay .shell .content { color:#2B2 } -.CodeRay .shell .delimiter { color:#161 } -.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } -.CodeRay .string .char { color: #b0b } -.CodeRay .string .content { color: #D20 } -.CodeRay .string .delimiter { color: #710 } -.CodeRay .string .modifier { color: #E40 } -.CodeRay .symbol { color:#A60 } -.CodeRay .symbol .content { color:#A60 } -.CodeRay .symbol .delimiter { color:#630 } -.CodeRay .tag { color:#070 } -.CodeRay .type { color:#339; font-weight:bold } -.CodeRay .value { color: #088 } -.CodeRay .variable { color:#037 } - -.CodeRay .insert { background: hsla(120,100%,50%,0.12) } -.CodeRay .delete { background: hsla(0,100%,50%,0.12) } -.CodeRay .change { color: #bbf; background: #007 } -.CodeRay .head { color: #f8f; background: #505 } -.CodeRay .head .filename { color: white; } - -.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } -.CodeRay .change .change { color: #88f } -.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.expected b/test/samples/css.expected deleted file mode 100644 index be73a7f9..00000000 --- a/test/samples/css.expected +++ /dev/null @@ -1,127 +0,0 @@ -.CodeRay { - background-color: hsl(0,0%,95%); - border: 1px solid silver; - color: black; -} -.CodeRay pre { - margin: 0px; -} - -span.CodeRay { white-space: pre; border: 0px; padding: 2px; } - -table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; } -table.CodeRay td { padding: 2px 4px; vertical-align: top; } - -.CodeRay .line-numbers { - background-color: hsl(180,65%,90%); - color: gray; - text-align: right; - -webkit-user-select: none; - -moz-user-select: none; - user-select: none; -} -.CodeRay .line-numbers a { - background-color: hsl(180,65%,90%) !important; - color: gray !important; - text-decoration: none !important; -} -.CodeRay .line-numbers pre { - word-break: normal; -} -.CodeRay .line-numbers a:target { color: blue !important; } -.CodeRay .line-numbers .highlighted { color: red !important; } -.CodeRay .line-numbers .highlighted a { color: red !important; } -.CodeRay span.line-numbers { padding: 0px 4px; } -.CodeRay .line { display: block; float: left; width: 100%; } -.CodeRay .code { width: 100%; } - -.CodeRay .debug { color: white !important; background: blue !important; } - -.CodeRay .annotation { color:#007 } -.CodeRay .attribute-name { color:#b48 } -.CodeRay .attribute-value { color:#700 } -.CodeRay .binary { color:#549 } -.CodeRay .binary .char { color:#325 } -.CodeRay .binary .delimiter { color:#325 } -.CodeRay .char { color:#D20 } -.CodeRay .char .content { color:#D20 } -.CodeRay .char .delimiter { color:#710 } -.CodeRay .class { color:#B06; font-weight:bold } -.CodeRay .class-variable { color:#369 } -.CodeRay .color { color:#0A0 } -.CodeRay .comment { color:#777 } -.CodeRay .comment .char { color:#444 } -.CodeRay .comment .delimiter { color:#444 } -.CodeRay .constant { color:#036; font-weight:bold } -.CodeRay .decorator { color:#B0B } -.CodeRay .definition { color:#099; font-weight:bold } -.CodeRay .delimiter { color:black } -.CodeRay .directive { color:#088; font-weight:bold } -.CodeRay .docstring { color:#D42; } -.CodeRay .doctype { color:#34b } -.CodeRay .done { text-decoration: line-through; color: gray } -.CodeRay .entity { color:#800; font-weight:bold } -.CodeRay .error { color:#F00; background-color:#FAA } -.CodeRay .escape { color:#666 } -.CodeRay .exception { color:#C00; font-weight:bold } -.CodeRay .float { color:#60E } -.CodeRay .function { color:#06B; font-weight:bold } -.CodeRay .function .delimiter { color:#024; font-weight:bold } -.CodeRay .global-variable { color:#d70 } -.CodeRay .hex { color:#02b } -.CodeRay .id { color:#33D; font-weight:bold } -.CodeRay .include { color:#B44; font-weight:bold } -.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.CodeRay .inline-delimiter { font-weight: bold; color: #666 } -.CodeRay .instance-variable { color:#33B } -.CodeRay .integer { color:#00D } -.CodeRay .imaginary { color:#f00 } -.CodeRay .important { color:#D00 } -.CodeRay .key { color: #606 } -.CodeRay .key .char { color: #60f } -.CodeRay .key .delimiter { color: #404 } -.CodeRay .keyword { color:#080; font-weight:bold } -.CodeRay .label { color:#970; font-weight:bold } -.CodeRay .local-variable { color:#963 } -.CodeRay .namespace { color:#707; font-weight:bold } -.CodeRay .octal { color:#40E } -.CodeRay .operator { } -.CodeRay .predefined { color:#369; font-weight:bold } -.CodeRay .predefined-constant { color:#069 } -.CodeRay .predefined-type { color:#0a5; font-weight:bold } -.CodeRay .preprocessor { color:#579 } -.CodeRay .pseudo-class { color:#00C; font-weight:bold } -.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); } -.CodeRay .regexp .content { color:#808 } -.CodeRay .regexp .delimiter { color:#404 } -.CodeRay .regexp .modifier { color:#C2C } -.CodeRay .reserved { color:#080; font-weight:bold } -.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); } -.CodeRay .shell .content { color:#2B2 } -.CodeRay .shell .delimiter { color:#161 } -.CodeRay .string { background-color:hsla(0,100%,50%,0.05); } -.CodeRay .string .char { color: #b0b } -.CodeRay .string .content { color: #D20 } -.CodeRay .string .delimiter { color: #710 } -.CodeRay .string .modifier { color: #E40 } -.CodeRay .symbol { color:#A60 } -.CodeRay .symbol .content { color:#A60 } -.CodeRay .symbol .delimiter { color:#630 } -.CodeRay .tag { color:#070 } -.CodeRay .type { color:#339; font-weight:bold } -.CodeRay .value { color: #088 } -.CodeRay .variable { color:#037 } - -.CodeRay .insert { background: hsla(120,100%,50%,0.12) } -.CodeRay .delete { background: hsla(0,100%,50%,0.12) } -.CodeRay .change { color: #bbf; background: #007 } -.CodeRay .head { color: #f8f; background: #505 } -.CodeRay .head .filename { color: white; } - -.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold } -.CodeRay .change .change { color: #88f } -.CodeRay .head .head { color: #f4f } diff --git a/test/samples/css.rb b/test/samples/css.rb deleted file mode 100644 index 52e4bcc7..00000000 --- a/test/samples/css.rb +++ /dev/null @@ -1,4 +0,0 @@ -require 'coderay' - -# print the default stylesheet for HTML codes -puts CodeRay::Encoders[:html]::CSS.new.stylesheet diff --git a/test/samples/div.actual b/test/samples/div.actual deleted file mode 100644 index d1e692ab..00000000 --- a/test/samples/div.actual +++ /dev/null @@ -1,17 +0,0 @@ -
- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/test/samples/div.expected b/test/samples/div.expected deleted file mode 100644 index d1e692ab..00000000 --- a/test/samples/div.expected +++ /dev/null @@ -1,17 +0,0 @@ -
- 
for a in 0..255
-        a = a.chr
-        begin
-                x = eval("?\\#{a}")
-                if x == a[0]
-                        next
-                else
-                        print "#{a}: #{x}"
-                end
-        rescue SyntaxError => boom
-                print "#{a}: error"
-        end
-        puts
-end
-
-
diff --git a/test/samples/div.rb b/test/samples/div.rb deleted file mode 100644 index 27b6f328..00000000 --- a/test/samples/div.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'coderay' - -puts CodeRay.scan(DATA.read, :ruby).div - -__END__ -for a in 0..255 - a = a.chr - begin - x = eval("?\\#{a}") - if x == a[0] - next - else - print "#{a}: #{x}" - end - rescue SyntaxError => boom - print "#{a}: error" - end - puts -end diff --git a/test/samples/encoder.actual b/test/samples/encoder.actual deleted file mode 100644 index 8bd83a92..00000000 --- a/test/samples/encoder.actual +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" -compressed: 79 byte < 1200 byte - -Undump: - -Code Statistics - -Tokens 800 - Non-Whitespace 400 -Bytes Total 1200 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 800 100.00 % 1.5 - space 400 50.00 % 1.0 - integer 200 25.00 % 1.5 - ident 100 12.50 % 4.0 - operator 100 12.50 % 1.0 - diff --git a/test/samples/encoder.expected b/test/samples/encoder.expected deleted file mode 100644 index 8bd83a92..00000000 --- a/test/samples/encoder.expected +++ /dev/null @@ -1,65 +0,0 @@ -Encoders Demo: puts 17 + 4 - -Statistic: - -Code Statistics - -Tokens 8 - Non-Whitespace 4 -Bytes Total 12 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 8 100.00 % 1.5 - space 4 50.00 % 1.0 - integer 2 25.00 % 1.5 - ident 1 12.50 % 4.0 - operator 1 12.50 % 1.0 - - -Original text: -[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}] - -YAML: ---- -- - puts - - :ident -- - " " - - :space -- - "17" - - :integer -- - " " - - :space -- - + - - :operator -- - " " - - :space -- - "4" - - :integer -- - | - - - - :space - -Dump: -"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257" -compressed: 79 byte < 1200 byte - -Undump: - -Code Statistics - -Tokens 800 - Non-Whitespace 400 -Bytes Total 1200 - -Token Types (4): - type count ratio size (average) -------------------------------------------------------------- - TOTAL 800 100.00 % 1.5 - space 400 50.00 % 1.0 - integer 200 25.00 % 1.5 - ident 100 12.50 % 4.0 - operator 100 12.50 % 1.0 - diff --git a/test/samples/encoder.rb b/test/samples/encoder.rb deleted file mode 100644 index bfcfbfa4..00000000 --- a/test/samples/encoder.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'coderay' - -SAMPLE = "puts 17 + 4\n" -puts 'Encoders Demo: ' + SAMPLE -scanner = CodeRay::Scanners[:ruby].new SAMPLE -encoder = CodeRay::Encoders[:statistic].new - -tokens = scanner.tokenize -stats = encoder.encode_tokens tokens - -puts -puts 'Statistic:' -puts stats - -# alternative 1 -tokens = CodeRay.scan SAMPLE, :ruby -encoder = CodeRay.encoder(:json) -textual = encoder.encode_tokens tokens -puts -puts 'Original text:' -puts textual - -# alternative 2 -yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby -puts -puts 'YAML:' -puts yaml - -# alternative 3 -require 'zlib' -BIGSAMPLE = SAMPLE * 100 -dump = Zlib::Deflate.deflate(CodeRay.scan(BIGSAMPLE, :ruby).debug) -puts -puts 'Dump:' -p dump -puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size] - -puts -puts 'Undump:' -puts CodeRay.scan(Zlib::Inflate.inflate(dump), :debug).statistic diff --git a/test/samples/global_vars.actual b/test/samples/global_vars.actual deleted file mode 100644 index e69de29b..00000000 diff --git a/test/samples/global_vars.diff b/test/samples/global_vars.diff deleted file mode 100644 index 2ed33821..00000000 --- a/test/samples/global_vars.diff +++ /dev/null @@ -1,5 +0,0 @@ -1,3d0 -< <--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -< ODER -< TEXT = <--$BLA-->.TEST(...) -\ No newline at end of file diff --git a/test/samples/global_vars.expected b/test/samples/global_vars.expected deleted file mode 100644 index 0dc13c8f..00000000 --- a/test/samples/global_vars.expected +++ /dev/null @@ -1,3 +0,0 @@ -<--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL -ODER -TEXT = <--$BLA-->.TEST(...) \ No newline at end of file diff --git a/test/samples/global_vars.rb b/test/samples/global_vars.rb deleted file mode 100644 index 8066d67d..00000000 --- a/test/samples/global_vars.rb +++ /dev/null @@ -1,13 +0,0 @@ -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE - -require 'coderay' - -tokens = CodeRay.scan code, :ruby -tokens.each_text_token { |text, kind| text.upcase! } -tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text } - -print tokens diff --git a/test/samples/global_vars2.expected b/test/samples/global_vars2.expected deleted file mode 100644 index 964cf504..00000000 --- a/test/samples/global_vars2.expected +++ /dev/null @@ -1,10 +0,0 @@ - - - - - -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) - - diff --git a/test/samples/global_vars2.rb b/test/samples/global_vars2.rb deleted file mode 100644 index 76468906..00000000 --- a/test/samples/global_vars2.rb +++ /dev/null @@ -1,28 +0,0 @@ -require 'coderay' -require 'erb' -include ERB::Util - -code = <<'CODE' -$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel -oder -text = $bla.test(...) -CODE -puts < - - - - -HTML - -CodeRay.scan_stream code, :ruby do |text, kind| - next if text.is_a? Symbol - text = h(text) - text = '%s' % text if kind == :global_variable - print text -end - -puts < - -HTML diff --git a/test/samples/highlight.expected b/test/samples/highlight.expected deleted file mode 100644 index 6a9b2784..00000000 --- a/test/samples/highlight.expected +++ /dev/null @@ -1,175 +0,0 @@ -
- 
puts "Hello, World!"
-
- - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-
require 'coderay'
-
-puts CodeRay.highlight('puts "Hello, World!"', :ruby)
-
-output = CodeRay.highlight_file($0, :line_numbers => :table)
-puts <<HTML
-<html>
-<head>
-#{output.stylesheet true}
-<body>
-#{output}
-</body>
-</html>
-HTML
-
- - - diff --git a/test/samples/highlight.rb b/test/samples/highlight.rb deleted file mode 100644 index 846efa45..00000000 --- a/test/samples/highlight.rb +++ /dev/null @@ -1,14 +0,0 @@ -require 'coderay' - -puts CodeRay.highlight('puts "Hello, World!"', :ruby) - -output = CodeRay.highlight_file($0, :line_numbers => :table) -puts < - -#{output.stylesheet true} - -#{output} - - -HTML diff --git a/test/samples/html.expected b/test/samples/html.expected deleted file mode 100644 index e98d5897..00000000 --- a/test/samples/html.expected +++ /dev/null @@ -1,919 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-59
-60
-61
-62
-63
-64
-65
-66
-67
-68
-69
-70
-71
-72
-73
-74
-75
-76
-77
-78
-79
-80
-81
-82
-83
-84
-85
-86
-87
-88
-89
-90
-91
-92
-93
-94
-95
-96
-97
-98
-99
-100
-101
-102
-103
-104
-105
-106
-107
-108
-109
-110
-111
-112
-113
-114
-115
-116
-117
-118
-119
-120
-121
-122
-123
-124
-125
-126
-127
-128
-129
-130
-131
-132
-133
-134
-135
-136
-137
-138
-139
-140
-141
-142
-143
-144
-145
-146
-147
-148
-149
-150
-151
-152
-153
-154
-155
-156
-157
-158
-159
-160
-161
-162
-163
-164
-165
-166
-167
-168
-169
-170
-171
-172
-173
-174
-175
-176
-177
-178
-179
-180
-181
-182
-183
-184
-185
-186
-187
-188
-189
-190
-191
-192
-193
-194
-195
-196
-197
-198
-199
-200
-201
-202
-203
-204
-205
-206
-207
-208
-209
-210
-211
-212
-213
-214
-215
-216
-217
-218
-219
-220
-221
-222
-223
-224
-225
-226
-227
-228
-229
-230
-231
-232
-233
-234
-235
-236
-237
-238
-239
-240
-241
-242
-243
-244
-245
-246
-247
-248
-249
-250
-251
-252
-253
-254
-255
-256
-257
-258
-259
-260
-261
-262
-263
-264
-265
-266
-267
-268
-269
-270
-271
-272
-273
-274
-275
-276
-277
-278
-279
-280
-281
-282
-283
-284
-285
-286
-287
-288
-289
-290
-291
-292
-293
-294
-295
-296
-297
-298
-299
-300
-301
-302
-303
-304
-305
-306
-307
-308
-309
-310
-311
-312
-313
-314
-315
-316
-317
-318
-319
-320
-321
-322
-323
-324
-325
-326
-327
-328
-329
-330
-331
-332
-333
-334
-335
-336
-337
-338
-339
-340
-341
-342
-343
-344
-345
-346
-347
-348
-349
-350
-351
-352
-353
-354
-355
-356
-357
-358
-359
-360
-361
-362
-363
-364
-365
-366
-367
-368
-369
-370
-371
-372
-373
-374
-375
-376
-377
-378
-379
-380
-381
-382
-383
-384
-385
-
require 'scanner'
-
-module CodeRay
-  
-  class RubyScanner < Scanner
-    
-    RESERVED_WORDS = [
-      'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
-      'defined?', 'ensure', 'module', 'redo', 'super', 'until',
-      'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
-      'when', 'END', 'case', 'else', 'for', 'retry',
-      'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
-      'undef', 'yield',
-    ]
-
-    DEF_KEYWORDS = ['def']
-    MODULE_KEYWORDS = ['class', 'module']
-    DEF_NEW_STATE = WordList.new(:initial).
-      add(DEF_KEYWORDS, :def_expected).
-      add(MODULE_KEYWORDS, :module_expected)
-
-    WORDS_ALLOWING_REGEXP = [
-      'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
-    ]
-    REGEXP_ALLOWED = WordList.new(false).
-      add(WORDS_ALLOWING_REGEXP, :set)
-    
-    PREDEFINED_CONSTANTS = [
-      'nil', 'true', 'false', 'self',
-      'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
-    ]
-
-    IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant)
-
-    METHOD_NAME = / #{IDENT} [?!]? /xo
-    METHOD_NAME_EX = /
-     #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
-     | \*\*?         # multiplication and power
-     | [-+~]@?       # plus, minus
-     | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
-     | \[\]=?        # array getter and setter
-     | <=?>? | >=?   # comparison, rocket operator
-     | << | >>       # append or shift left, shift right
-     | ===?          # simple equality and case equality
-    /ox
-    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
-
-    DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
-    SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
-    STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
-    SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
-    REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
-    
-    DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
-    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
-    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
-    BINARY = /0b[01]+(?:_[01]+)*/
-
-    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
-    FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
-    INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
-    
-    def reset
-      super
-      @regexp_allowed = false
-    end
-    
-    def next_token
-      return if @scanner.eos?
-
-      kind = :error
-      if @scanner.scan(/\s+/)  # in every state
-        kind = :space
-        @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
-
-      elsif @state == :def_expected
-        if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
-          kind = :method
-          @state = :initial
-        else
-          @scanner.scan(/./)
-          kind = :error
-        end
-        @state = :initial
-        
-      elsif @state == :module_expected
-        if @scanner.scan(/<</)
-          kind = :operator
-        else
-          if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
-            kind = :method
-          else
-            @scanner.scan(/./)
-            kind = :error
-          end
-          @state = :initial
-        end
-        
-      elsif # state == :initial
-        # IDENTIFIERS, KEYWORDS
-        if @scanner.scan(GLOBAL_VARIABLE)
-          kind = :global_variable
-        elsif @scanner.scan(/ @@ #{IDENT} /ox)
-          kind = :class_variable
-        elsif @scanner.scan(/ @ #{IDENT} /ox)
-          kind = :instance_variable
-        elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
-          kind = :comment
-        elsif @scanner.scan(METHOD_NAME)
-          if @last_token_dot
-            kind = :ident
-          else
-            matched = @scanner.matched
-            kind = IDENT_KIND[matched]
-            if kind == :ident and matched =~ /^[A-Z]/
-              kind = :constant
-            elsif kind == :reserved
-              @state = DEF_NEW_STATE[matched]
-              @regexp_allowed = REGEXP_ALLOWED[matched]
-            end
-          end
-          
-        elsif @scanner.scan(STRING)
-          kind = :string
-        elsif @scanner.scan(SHELL)
-          kind = :shell
-        ## HEREDOCS
-        elsif @scanner.scan(/\//) and @regexp_allowed
-           @scanner.unscan
-           @scanner.scan(REGEXP)
-          kind = :regexp
-        ## %strings
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        elsif @scanner.scan(/
-          \? (?:
-            [^\s\\]
-          | 
-            \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
-          )
-        /ox)
-          kind = :integer
-          
-        elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
-          kind = :operator
-          @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
-        elsif @scanner.scan(FLOAT)
-          kind = :float
-        elsif @scanner.scan(INTEGER)
-          kind = :integer
-        elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
-          kind = :global_variable
-        else
-          @scanner.scan(/./m)
-        end
-      end
-      
-      token = Token.new @scanner.matched, kind
-
-      if kind == :regexp
-        token.text << @scanner.scan(/[eimnosux]*/)
-      end
-      
-      @regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
-
-      token
-    end
-  end
-  
-  ScannerList.register RubyScanner, 'ruby'
-
-end
-
-module CodeRay
-  require 'scanner'
-
-  class Highlighter
-
-    def initialize lang
-      @scanner = Scanner[lang].new
-    end
-
-    def highlight code
-      @scanner.feed code
-      @scanner.all_tokens.map { |t| t.inspect }.join "\n"
-    end
-
-  end
-
-  class HTMLHighlighter < Highlighter
-    
-    ClassOfKind = {
-      :attribute_name => 'an',
-      :attribute_name_fat => 'af',
-      :attribute_value => 'av',
-      :attribute_value_fat => 'aw',
-      :bin => 'bi',
-       :char => 'ch',
-      :class => 'cl',
-      :class_variable => 'cv',
-      :color => 'cr',
-      :comment => 'c',
-      :constant => 'co',
-      :definition => 'df',
-      :directive => 'di',
-      :doc => 'do',
-      :doc_string => 'ds',
-      :exception => 'ex',
-      :error => 'er',
-      :float => 'fl',
-      :function => 'fu',
-      :global_variable => 'gv',
-      :hex => 'hx',
-      :include => 'ic',
-      :instance_variable => 'iv',
-      :integer => 'i',
-      :interpreted => 'in',
-      :label => 'la',
-      :local_variable => 'lv',
-      :oct => 'oc',
-      :operator_name => 'on',
-      :pre_constant => 'pc',
-      :pre_type => 'pt',
-      :predefined => 'pd',
-      :preprocessor => 'pp',
-      :regexp => 'rx',
-      :reserved => 'r',
-      :shell => 'sh',
-      :string => 's',
-      :symbol => 'sy',
-      :tag => 'ta',
-      :tag_fat => 'tf',
-      :tag_special => 'ts',
-      :type => 'ty',
-      :variable => 'v',
-      :xml_text => 'xt',
-
-      :ident => :NO_HIGHLIGHT,
-      :operator => :NO_HIGHLIGHT,
-      :space => :NO_HIGHLIGHT,
-    }
-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
-    ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
-    
-    def initialize lang, options = {}
-      super lang
-      
-      @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
-      case level = options.fetch(:level, 'xhtml')
-        when 'html'
-          @HTML_BR = "<BR>\n"
-        when 'xhtml'
-          @HTML_BR = "<br />\n"
-      else
-        raise "Unknown HTML level: #{level}"
-      end
-    end
-
-    def highlight code
-      @scanner.feed code
-      
-      out = ''
-      while t = @scanner.next_token
-        warn t.inspect if t.text.nil?
-        out << to_html(t)
-      end
-      TEMPLATE =~ /<%CONTENT%>/
-      $` + out + $'
-    end
-    
-  private
-    def to_html token
-      css_class = ClassOfKind[token.kind]
-      if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
-        warn "no token class found for :#{token.kind}"
-      end
-        
-      text = text_to_html token.text
-      if css_class == :NO_HIGHLIGHT
-        text
-      else
-        "<span class=\"#{css_class}\">#{text}</span>"
-      end
-    end
-    
-    def text_to_html text
-      return '' if text.empty?
-      text = text.dup  # important
-      if text.index(/["><&]/)
-        text.gsub!('&', '&amp;')
-        text.gsub!('"', '&quot;')
-        text.gsub!('>', '&gt;')
-        text.gsub!('<', '&lt;')
-      end
-      if text.index(/\s/)
-        text.gsub!("\n", @HTML_BR)
-        text.gsub!("\t", @HTML_TAB)
-        text.gsub!(/^ /, '&nbsp;')
-        text.gsub!('  ', ' &nbsp;')
-      end
-      text
-    end
-    
-    TEMPLATE = <<-'TEMPLATE'
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
-<html dir="ltr">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-<meta http-equiv="Content-Style-Type" content="text/css">
-
-<title>RubyBB BBCode</title>
-<style type="text/css">
-.code {
-  width: 100%;
-  background-color: #FAFAFA;
-  border: 1px solid #D1D7DC;
-  font-family: 'Courier New', 'Terminal', monospace;
-  font-size: 10pt;
-  color: black;
-  vertical-align: top;
-  text-align: left;
-}
-.code .af { color:#00C; }
-.code .an { color:#007; }
-.code .av { color:#700; }
-.code .aw { color:#C00; }
-.code .bi { color:#509; font-weight:bold; }
-.code .c  { color:#888; }
-.code .ch { color:#C28; font-weight:bold; }
-.code .cl { color:#B06; font-weight:bold; }
-.code .co { color:#036; font-weight:bold; }
-.code .cr { color:#0A0; }
-.code .cv { color:#369; }
-.code .df { color:#099; font-weight:bold; }
-.code .di { color:#088; font-weight:bold; }
-.code .do { color:#970; }
-.code .ds { color:#D42; font-weight:bold; }
-.code .er { color:#F00; background-color:#FAA; }
-.code .ex { color:#F00; font-weight:bold; }
-.code .fl { color:#60E; font-weight:bold; }
-.code .fu { color:#06B; font-weight:bold; }
-.code .gv { color:#800; font-weight:bold; }
-.code .hx { color:#058; font-weight:bold; }
-.code .i  { color:#00D; font-weight:bold; }
-.code .ic { color:#B44; font-weight:bold; }
-.code .in { color:#B2B; font-weight:bold; }
-.code .iv { color:#33B; }
-.code .la { color:#970; font-weight:bold; }
-.code .lv { color:#963; }
-.code .oc { color:#40E; font-weight:bold; }
-.code .on { color:#000; font-weight:bold; }
-.code .pc { color:#038; font-weight:bold; }
-.code .pd { color:#369; font-weight:bold; }
-.code .pp { color:#579; }
-.code .pt { color:#339; font-weight:bold; }
-.code .r  { color:#080; font-weight:bold; }
-.code .rx { color:#927; font-weight:bold; }
-.code .s  { color:#D42; font-weight:bold; }
-.code .sh { color:#B2B; font-weight:bold; }
-.code .sy { color:#A60; }
-.code .ta { color:#070; }
-.code .tf { color:#070; font-weight:bold; }
-.code .ts { color:#D70; font-weight:bold; }
-.code .ty { color:#339; font-weight:bold; }
-.code .v  { color:#036; }
-.code .xt { color:#444; }
-</style>
-</head>
-<body>
-<div class="code">
-<%CONTENT%>
-</div>
-<div class="validators">
-<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
-<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
-</div>    
-</body>
-</html>
-    TEMPLATE
-
-  end
-
-end
-
- - - diff --git a/test/samples/html.rb b/test/samples/html.rb deleted file mode 100644 index c18284a1..00000000 --- a/test/samples/html.rb +++ /dev/null @@ -1,394 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan DATA.read, :ruby -html = tokens.page(:tab_width => 2, :line_numbers => :table, :title => 'CodeRay HTML Encoder Example') - -puts html - -__END__ -require 'scanner' - -module CodeRay - - class RubyScanner < Scanner - - RESERVED_WORDS = [ - 'and', 'def', 'end', 'in', 'or', 'unless', 'begin', - 'defined?', 'ensure', 'module', 'redo', 'super', 'until', - 'BEGIN', 'break', 'do', 'next', 'rescue', 'then', - 'when', 'END', 'case', 'else', 'for', 'retry', - 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return', - 'undef', 'yield', - ] - - DEF_KEYWORDS = ['def'] - MODULE_KEYWORDS = ['class', 'module'] - DEF_NEW_STATE = WordList.new(:initial). - add(DEF_KEYWORDS, :def_expected). - add(MODULE_KEYWORDS, :module_expected) - - WORDS_ALLOWING_REGEXP = [ - 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when' - ] - REGEXP_ALLOWED = WordList.new(false). - add(WORDS_ALLOWING_REGEXP, :set) - - PREDEFINED_CONSTANTS = [ - 'nil', 'true', 'false', 'self', - 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__', - ] - - IDENT_KIND = WordList.new(:ident). - add(RESERVED_WORDS, :reserved). - add(PREDEFINED_CONSTANTS, :pre_constant) - - METHOD_NAME = / #{IDENT} [?!]? /xo - METHOD_NAME_EX = / - #{METHOD_NAME} # common methods: split, foo=, empty?, gsub! - | \*\*? # multiplication and power - | [-+~]@? # plus, minus - | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system` - | \[\]=? # array getter and setter - | <=?>? | >=? # comparison, rocket operator - | << | >> # append or shift left, shift right - | ===? # simple equality and case equality - /ox - GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox - - DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox - SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox - STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox - SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox - REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox - - DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error - OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ - HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ - BINARY = /0b[01]+(?:_[01]+)*/ - - EXPONENT = / [eE] [+-]? #{DECIMAL} /ox - FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) / - INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/ - - def reset - super - @regexp_allowed = false - end - - def next_token - return if @scanner.eos? - - kind = :error - if @scanner.scan(/\s+/) # in every state - kind = :space - @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting - - elsif @state == :def_expected - if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox) - kind = :method - @state = :initial - else - @scanner.scan(/./) - kind = :error - end - @state = :initial - - elsif @state == :module_expected - if @scanner.scan(/<;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x) - kind = :operator - @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/ - elsif @scanner.scan(FLOAT) - kind = :float - elsif @scanner.scan(INTEGER) - kind = :integer - elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox) - kind = :global_variable - else - @scanner.scan(/./m) - end - end - - token = Token.new @scanner.matched, kind - - if kind == :regexp - token.text << @scanner.scan(/[eimnosux]*/) - end - - @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting - - token - end - end - - ScannerList.register RubyScanner, 'ruby' - -end - -module CodeRay - require 'scanner' - - class Highlighter - - def initialize lang - @scanner = Scanner[lang].new - end - - def highlight code - @scanner.feed code - @scanner.all_tokens.map { |t| t.inspect }.join "\n" - end - - end - - class HTMLHighlighter < Highlighter - - ClassOfKind = { - :attribute_name => 'an', - :attribute_name_fat => 'af', - :attribute_value => 'av', - :attribute_value_fat => 'aw', - :bin => 'bi', - :char => 'ch', - :class => 'cl', - :class_variable => 'cv', - :color => 'cr', - :comment => 'c', - :constant => 'co', - :definition => 'df', - :directive => 'di', - :doc => 'do', - :doc_string => 'ds', - :exception => 'ex', - :error => 'er', - :float => 'fl', - :function => 'fu', - :global_variable => 'gv', - :hex => 'hx', - :include => 'ic', - :instance_variable => 'iv', - :integer => 'i', - :interpreted => 'in', - :label => 'la', - :local_variable => 'lv', - :oct => 'oc', - :operator_name => 'on', - :pre_constant => 'pc', - :pre_type => 'pt', - :predefined => 'pd', - :preprocessor => 'pp', - :regexp => 'rx', - :reserved => 'r', - :shell => 'sh', - :string => 's', - :symbol => 'sy', - :tag => 'ta', - :tag_fat => 'tf', - :tag_special => 'ts', - :type => 'ty', - :variable => 'v', - :xml_text => 'xt', - - :ident => :NO_HIGHLIGHT, - :operator => :NO_HIGHLIGHT, - :space => :NO_HIGHLIGHT, - } - ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function] - ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!' - - def initialize lang, options = {} - super lang - - @HTML_TAB = ' ' * options.fetch(:tabs2space, 8) - case level = options.fetch(:level, 'xhtml') - when 'html' - @HTML_BR = "
\n" - when 'xhtml' - @HTML_BR = "
\n" - else - raise "Unknown HTML level: #{level}" - end - end - - def highlight code - @scanner.feed code - - out = '' - while t = @scanner.next_token - warn t.inspect if t.text.nil? - out << to_html(t) - end - TEMPLATE =~ /<%CONTENT%>/ - $` + out + $' - end - - private - def to_html token - css_class = ClassOfKind[token.kind] - if defined? ::DEBUG and not ClassOfKind.has_key? token.kind - warn "no token class found for :#{token.kind}" - end - - text = text_to_html token.text - if css_class == :NO_HIGHLIGHT - text - else - "#{text}" - end - end - - def text_to_html text - return '' if text.empty? - text = text.dup # important - if text.index(/["><&]/) - text.gsub!('&', '&') - text.gsub!('"', '"') - text.gsub!('>', '>') - text.gsub!('<', '<') - end - if text.index(/\s/) - text.gsub!("\n", @HTML_BR) - text.gsub!("\t", @HTML_TAB) - text.gsub!(/^ /, ' ') - text.gsub!(' ', '  ') - end - text - end - - TEMPLATE = <<-'TEMPLATE' - - - - - - -RubyBB BBCode - - - -
-<%CONTENT%> -
-
-Valid HTML 4.01! -Valid CSS! -
- - - TEMPLATE - - end - -end diff --git a/test/samples/html2.expected b/test/samples/html2.expected deleted file mode 100644 index c8ae56a7..00000000 --- a/test/samples/html2.expected +++ /dev/null @@ -1,185 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - - - - -
1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-
require 'coderay'
-
-# scan this file
-tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-
-# output it with two styles of line numbers
-out = tokens.div(:line_numbers => :table)
-out << '<hr />'
-out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-
-puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- 
 8 require 'coderay'
- 9 
-10 # scan this file
-11 tokens = CodeRay.scan(File.read($0) * 1, :ruby)
-12 
-13 # output it with two styles of line numbers
-14 out = tokens.div(:line_numbers => :table)
-15 out << '<hr />'
-16 out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
-17 
-18 puts out.page(:title => 'CodeRay HTML Encoder Example')
-
-
- - - diff --git a/test/samples/html2.rb b/test/samples/html2.rb deleted file mode 100644 index 618d168d..00000000 --- a/test/samples/html2.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'coderay' - -# scan this file -tokens = CodeRay.scan(File.read($0) * 1, :ruby) - -# output it with two styles of line numbers -out = tokens.div(:line_numbers => :table) -out << '
' -out << tokens.div(:line_numbers => :inline, :line_number_start => 8) - -puts out.page(:title => 'CodeRay HTML Encoder Example') diff --git a/test/samples/html_list.expected b/test/samples/html_list.expected deleted file mode 100644 index a4092c8d..00000000 --- a/test/samples/html_list.expected +++ /dev/null @@ -1,160 +0,0 @@ - - - - - CodeRay HTML Encoder Example - - - - -
- 
-1 $: << '..'
- 0 require 'coderay'
- 1 
- 2 tokens = CodeRay.scan File.read(__FILE__), :ruby
- 3 html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)
- 4 
- 5 puts html.page(:title => 'CodeRay HTML Encoder Example')
- 6 
- 7 commment = <<_
- 8 This code must be > 10 lines
- 9 because I want to test the correct adjustment of the line numbers.
-10 _
-
-
- - - diff --git a/test/samples/html_list.rb b/test/samples/html_list.rb deleted file mode 100644 index fdfa5123..00000000 --- a/test/samples/html_list.rb +++ /dev/null @@ -1,12 +0,0 @@ -$: << '..' -require 'coderay' - -tokens = CodeRay.scan File.read(__FILE__), :ruby -html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1) - -puts html.page(:title => 'CodeRay HTML Encoder Example') - -commment = <<_ -This code must be > 10 lines -because I want to test the correct adjustment of the line numbers. -_ diff --git a/test/samples/load_encoder.expected b/test/samples/load_encoder.expected deleted file mode 100644 index 1cff356d..00000000 --- a/test/samples/load_encoder.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::YAML is not defined; you must load it first. -Now it is loaded: CodeRay::Encoders::YAML -See? -Require is also possible: CodeRay::Encoders::Tokens -See? -Now load some mapped encoders: stats and plain. -Require all Encoders: -[[:count, CodeRay::Encoders::Count], [:debug, CodeRay::Encoders::Debug], [:div, CodeRay::Encoders::Div], [:html, CodeRay::Encoders::HTML], [:null, CodeRay::Encoders::Null], [:page, CodeRay::Encoders::Page], [:plain, :text], [:span, CodeRay::Encoders::Span], [:statistic, CodeRay::Encoders::Statistic], [:stats, CodeRay::Encoders::Statistic], [:text, CodeRay::Encoders::Text], [:tokens, CodeRay::Encoders::Tokens], [:xml, CodeRay::Encoders::XML], [:yaml, CodeRay::Encoders::YAML]] diff --git a/test/samples/load_encoder.rb b/test/samples/load_encoder.rb deleted file mode 100644 index 9594bfa1..00000000 --- a/test/samples/load_encoder.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Encoders::YAML -rescue - puts 'CodeRay::Encoders::YAML is not defined; you must load it first.' -end - -yaml_encoder = CodeRay::Encoders[:yaml] -print 'Now it is loaded: ' -p yaml_encoder -puts 'See?' - -tokens_encoder = CodeRay.require_plugin 'CodeRay::Encoders/tokens' -print 'Require is also possible: ' -p tokens_encoder -puts 'See?' - -puts 'Now load some mapped encoders: stats and plain.' -CodeRay.require_plugin 'CodeRay::Encoders/stats' -CodeRay.require_plugin 'CodeRay::Encoders/plain' - -puts 'Require all Encoders:' -CodeRay::Encoders.load_all -p CodeRay::Encoders.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/load_scanner.expected b/test/samples/load_scanner.expected deleted file mode 100644 index a2d200d7..00000000 --- a/test/samples/load_scanner.expected +++ /dev/null @@ -1,8 +0,0 @@ -CodeRay::Encoders::Ruby is not defined; you must load it first. -Now it is loaded: CodeRay::Scanners::Ruby -See? -Require is also possible: CodeRay::Scanners::C -See? -Now load some mapped scanners: cpp and plain. -Require all Scanners: -[[nil, :plain], [:c, CodeRay::Scanners::C], [:cpp, :c], [:delphi, CodeRay::Scanners::Delphi], [:html, CodeRay::Scanners::HTML], [:irb, :ruby], [:nitro, :nitro_xhtml], [:nitro_xhtml, CodeRay::Scanners::NitroXHTML], [:pascal, :delphi], [:plain, CodeRay::Scanners::Plaintext], [:plaintext, CodeRay::Scanners::Plaintext], [:rhtml, CodeRay::Scanners::RHTML], [:ruby, CodeRay::Scanners::Ruby], [:xhtml, :nitro_xhtml], [:xml, :html]] diff --git a/test/samples/load_scanner.rb b/test/samples/load_scanner.rb deleted file mode 100644 index 23be8a29..00000000 --- a/test/samples/load_scanner.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -begin - CodeRay::Scanners::Ruby -rescue - puts 'CodeRay::Encoders::Ruby is not defined; you must load it first.' -end - -ruby_scanner = CodeRay::Scanners[:ruby] -print 'Now it is loaded: ' -p ruby_scanner -puts 'See?' - -c_scanner = CodeRay.require_plugin 'CodeRay::Scanners/c' -print 'Require is also possible: ' -p c_scanner -puts 'See?' - -puts 'Now load some mapped scanners: cpp and plain.' -CodeRay.require_plugin 'CodeRay::Scanners/cpp' -CodeRay.require_plugin 'CodeRay::Scanners/plain' - -puts 'Require all Scanners:' -CodeRay::Scanners.load_all -p CodeRay::Scanners.plugin_hash.sort_by { |k,v| k.to_s } diff --git a/test/samples/more.expected b/test/samples/more.expected deleted file mode 100644 index 196904d8..00000000 --- a/test/samples/more.expected +++ /dev/null @@ -1,2 +0,0 @@ -Input: 4983B, Output: 23484B -Take a look with your browser. diff --git a/test/samples/more.rb b/test/samples/more.rb deleted file mode 100644 index 0db7ba47..00000000 --- a/test/samples/more.rb +++ /dev/null @@ -1,205 +0,0 @@ -require 'coderay' - -c, ruby = DATA.read.split(/^---$/) -DATA.rewind -me = DATA.read[/.*^__END__$/m] -$input = c + ruby + me - -require 'benchmark' -time = Benchmark.realtime do - - # here CodeRay comes to play - hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div) - c = hl.highlight c, :c - ruby = hl.highlight ruby, :ruby - me = hl.highlight me, :ruby - - body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code| - "

#{title}

\n#{code}" - end.join - body = hl.class::Output.new(body, hl.css, :div).page! - - # CodeRay also provides a simple page generator - $output = body #hl.class.wrap_in_page body -end - -File.open('test.html', 'w') do |f| - f.write $output -end -puts 'Input: %dB, Output: %dB' % [$input.size, $output.size] -#puts 'Created "test.html" in %0.3f seconds (%d KB/s).' % [time, $input.size / 1024.0 / time] -puts 'Take a look with your browser.' - -__END__ -/********************************************************************** - - version.c - - - $Author: nobu $ - $Date: 2004/03/25 12:01:40 $ - created at: Thu Sep 30 20:08:01 JST 1993 - - Copyright (C) 1993-2003 Yukihiro Matsumoto - -**********************************************************************/ - -#include "ruby.h" -#include "version.h" -#include - -const char ruby_version[] = RUBY_VERSION; -const char ruby_release_date[] = RUBY_RELEASE_DATE; -const char ruby_platform[] = RUBY_PLATFORM; - -void -Init_version() -{ - VALUE v = rb_obj_freeze(rb_str_new2(ruby_version)); - VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date)); - VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform)); - - rb_define_global_const("RUBY_VERSION", v); - rb_define_global_const("RUBY_RELEASE_DATE", d); - rb_define_global_const("RUBY_PLATFORM", p); -} - -void -ruby_show_version() -{ - printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM); -} - -void -ruby_show_copyright() -{ - printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR); - exit(0); -} ---- -# -# = ostruct.rb: OpenStruct implementation -# -# Author:: Yukihiro Matsumoto -# Documentation:: Gavin Sinclair -# -# OpenStruct allows the creation of data objects with arbitrary attributes. -# See OpenStruct for an example. -# - -# -# OpenStruct allows you to create data objects and set arbitrary attributes. -# For example: -# -# require 'ostruct' -# -# record = OpenStruct.new -# record.name = "John Smith" -# record.age = 70 -# record.pension = 300 -# -# puts record.name # -> "John Smith" -# puts record.address # -> nil -# -# It is like a hash with a different way to access the data. In fact, it is -# implemented with a hash, and you can initialize it with one. -# -# hash = { "country" => "Australia", :population => 20_000_000 } -# data = OpenStruct.new(hash) -# -# p data # -> -# -class OpenStruct - # - # Create a new OpenStruct object. The optional +hash+, if given, will - # generate attributes and values. For example. - # - # require 'ostruct' - # hash = { "country" => "Australia", :population => 20_000_000 } - # data = OpenStruct.new(hash) - # - # p data # -> - # - # By default, the resulting OpenStruct object will have no attributes. - # - def initialize(hash=nil) - @table = {} - if hash - for k,v in hash - @table[k.to_sym] = v - new_ostruct_member(k) - end - end - end - - # Duplicate an OpenStruct object members. - def initialize_copy(orig) - super - @table = @table.dup - end - - def marshal_dump - @table - end - def marshal_load(x) - @table = x - @table.each_key{|key| new_ostruct_member(key)} - end - - def new_ostruct_member(name) - unless self.respond_to?(name) - self.instance_eval %{ - def #{name}; @table[:#{name}]; end - def #{name}=(x); @table[:#{name}] = x; end - } - end - end - - def method_missing(mid, *args) # :nodoc: - mname = mid.id2name - len = args.length - if mname =~ /=$/ - if len != 1 - raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1) - end - if self.frozen? - raise TypeError, "can't modify frozen #{self.class}", caller(1) - end - mname.chop! - @table[mname.intern] = args[0] - self.new_ostruct_member(mname) - elsif len == 0 - @table[mid] - else - raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1) - end - end - - # - # Remove the named field from the object. - # - def delete_field(name) - @table.delete name.to_sym - end - - # - # Returns a string containing a detailed summary of the keys and values. - # - def inspect - str = "<#{self.class}" - for k,v in @table - str << " #{k}=#{v.inspect}" - end - str << ">" - end - - attr_reader :table # :nodoc: - protected :table - - # - # Compare this object and +other+ for equality. - # - def ==(other) - return false unless(other.kind_of?(OpenStruct)) - return @table == other.table - end -end diff --git a/test/samples/scanner.expected b/test/samples/scanner.expected deleted file mode 100644 index 5015168f..00000000 --- a/test/samples/scanner.expected +++ /dev/null @@ -1,16 +0,0 @@ -C Code: if (*p == '{') nest++; - -> print only operators: -(*==)++; ------------------------------- - -Ruby Code: ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee) - -> has a string? -false - -> number of regexps? -2 - -> has a string? -"ruby_code" (ident), "(" (operator), ":can" (symbol), "," (operator), " " (space), "BE" (constant), "," (operator), " " (space), "%r[" (delimiter), "q" (content), "[" (nesting_delimiter), "ui" (content), "]" (nesting_delimiter), "te " (content), "#{" (inline_delimiter), " " (space), "/" (delimiter), "comple" (content), "/" (delimiter), "x" (modifier), " " (space), "}" (inline_delimiter), "," (content), "]" (delimiter), " " (space), "=" (operator), ">" (operator), " " (space), "$-s" (global_variable), "," (operator), " " (space), "&" (operator), "?\xee" (integer), ")" (operator) diff --git a/test/samples/scanner.rb b/test/samples/scanner.rb deleted file mode 100644 index 6a0245ea..00000000 --- a/test/samples/scanner.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'coderay' - -c_code = "if (*p == '{') nest++;" -puts 'C Code: ' + c_code -puts - -c_scanner = CodeRay::Scanners[:c].new c_code - -puts '> print only operators:' -for text, kind in c_scanner - print text if kind == :operator -end -puts -puts '-' * 30 -puts - -ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)! -puts 'Ruby Code: ' + ruby_code -puts - -ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code - -puts '> has a string?' -puts ruby_scanner. - any? { |text, kind| kind == :string } -puts - -puts '> number of regexps?' -puts ruby_scanner. - select { |token| token == [:open, :regexp] }.size -puts - -puts '> has a string?' -puts ruby_scanner. - reject { |text, kind| not text.is_a? String }. - map { |text, kind| %("#{text}" (#{kind})) }.join(', ') diff --git a/test/samples/server.rb b/test/samples/server.rb deleted file mode 100644 index ccdff324..00000000 --- a/test/samples/server.rb +++ /dev/null @@ -1,110 +0,0 @@ -# CodeRay dynamic highlighter - -unless ARGV.grep(/-[hv]|--(help|version)/).empty? - puts <<-USAGE -CodeRay Server 0.5 -$Id: demo_server.rb 113 2006-03-15 23:24:37Z murphy $ - -Usage: - 1) Start this and your browser. - 2) Go to http://localhost:2468/? - and you should get the highlighted version. - -Parameters: - -d Debug mode; reload CodeRay engine for every file. - (prepare for MANY "already initialized" and "method redefined" - messages - ingore it.) - - ... More to come. - USAGE - exit -end - -require 'webrick' -require 'pathname' - -class << File - alias dir? directory? -end - -require 'erb' -include ERB::Util -def url_decode s - s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' } -end - -class String - def to_link name = File.basename(self) - "#{name}" - end -end - -require 'coderay' -class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet - - STYLE = 'style="font-family: sans-serif; color: navy;"' - BANNER = '

Highlighted by CodeRay

' - - def do_GET req, res - q = req.query_string || '' - args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) } - path = args.fetch 'path', '.' - - backlinks = '

current path: %s
' % html_escape(path) + - (Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' + - '.'.to_link('current') + '

' - - res.body = - if File.dir? path - path = Pathname.new(path).cleanpath.to_s - dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p } - - page = "" - page << backlinks - - page << '
' - page << "
Directories
\n" + dirs.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
Files
\n" + files.map do |p| - "
#{p.to_link}
\n" - end.join << "\n" - page << "
\n" - page << "#{BANNER}" - - elsif File.exist? path - if $DEBUG - $".delete_if { |f| f =~ /coderay/ } - require 'coderay' - end - div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div, :hint => :info - div.replace <<-DIV -
- #{backlinks} -#{div} -
- #{BANNER} - DIV - div.page - end - - res['Content-Type'] = 'text/html' - end -end - -# This port is taken by "qip_msgd" - I don't know that. Do you? -module CodeRay - PORT = 0xC0DE / 20 -end - -server = WEBrick::HTTPServer.new :Port => CodeRay::PORT - -server.mount '/', CodeRayServlet - -server.mount_proc '/version' do |req, res| - res.body = 'CodeRay::Version = ' + CodeRay::Version - res['Content-Type'] = "text/plain" -end - -trap("INT") { server.shutdown } -server.start diff --git a/test/samples/simple.expected b/test/samples/simple.expected deleted file mode 100644 index b3d78753..00000000 --- a/test/samples/simple.expected +++ /dev/null @@ -1 +0,0 @@ -puts 'Hello, world!' diff --git a/test/samples/simple.rb b/test/samples/simple.rb deleted file mode 100644 index a3129b01..00000000 --- a/test/samples/simple.rb +++ /dev/null @@ -1,10 +0,0 @@ - -# Load CodeRay -# If this doesn't work, try ruby -rubygems. -require 'coderay' - -# Generate HTML page for Ruby code. -page = CodeRay.scan("puts 'Hello, world!'", :ruby).span - -# Print it -puts page diff --git a/test/samples/stream.rb b/test/samples/stream.rb deleted file mode 100644 index 7ed8a22b..00000000 --- a/test/samples/stream.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'coderay' - -code = File.read($0) * 500 -puts "Size of code: %d KB" % [code.size / 1024] - -puts "Use your system's memory tracker to see how much RAM this takes." -print 'Press some key to continue...'; gets - -require 'benchmark' -e = CodeRay.encoder(:div) -for do_stream in [true, false] - puts "Scanning and encoding in %s mode, please wait..." % - [do_stream ? 'streaming' : 'normal'] - output = '' - time = Benchmark.realtime do - if do_stream - output = e.encode_stream(code, :ruby) - else - output = e.encode_tokens(t = CodeRay.scan(code, :ruby)) - end - end - puts 'Finished after %4.2f seconds.' % time - puts "Size of output: %d KB" % [output.size / 1024] - print 'Press some key to continue...'; gets -end diff --git a/test/samples/stream2.expected b/test/samples/stream2.expected deleted file mode 100644 index 83aee987..00000000 --- a/test/samples/stream2.expected +++ /dev/null @@ -1,2 +0,0 @@ -kind: regexp, text size: 5. -kind: space, text size: 1. diff --git a/test/samples/stream2.rb b/test/samples/stream2.rb deleted file mode 100644 index d43cc9ad..00000000 --- a/test/samples/stream2.rb +++ /dev/null @@ -1,8 +0,0 @@ -require 'coderay' - -token_stream = CodeRay::TokenStream.new do |kind, text| - puts 'kind: %s, text size: %d.' % [kind, text.size] -end - -token_stream << [:regexp, '/\d+/'] << [:space, "\n"] -#-> kind: rexpexp, text size: 5. diff --git a/test/samples/suite.rb b/test/samples/suite.rb deleted file mode 100644 index cfe53c0f..00000000 --- a/test/samples/suite.rb +++ /dev/null @@ -1,86 +0,0 @@ -mydir = File.dirname(__FILE__) -$:.unshift mydir + '/../../lib/' - -$VERBOSE = true - -require 'test/unit' -include Test::Unit - -class CodeRaySuite < TestCase - - def self.dir &block - @dir ||= File.dirname(__FILE__) - if block - Dir.chdir @dir, &block - end - @dir - end - - def dir &block - self.class.dir(&block) - end - - def test_ALL - dir do - for input in Dir["*.rb"] - %w(server.rb stream.rb suite.rb) - next if input[/^load_/] - puts "[ testing #{input}... ]" - name = File.basename(input, ".rb") - output = name + '.expected' - code = File.open(input, 'rb') { |f| break f.read } - - result = `ruby -wI../../lib #{input}` - - diff = output.sub '.expected', '.diff' - File.delete diff if File.exist? diff - computed = output.sub '.expected', '.actual' - if File.exist? output - expected = File.read output - ok = expected == result - unless ok - File.open(computed, 'w') { |f| f.write result } - `diff #{output} #{computed} > #{diff}` - puts "Test failed; output written to #{diff}." - end - assert(ok, "Output error: #{computed} != #{output}") - else - File.open(output, 'w') do |f| f.write result end - puts "New test: #{output}" - end - - end - end - end - -end - -require 'test/unit/testsuite' -$suite = TestSuite.new 'CodeRay Demos Test' -$suite << CodeRaySuite.suite - -def load_suite name - begin - require name + '/suite.rb' - rescue LoadError - $stderr.puts <<-ERR - -!! Folder #{File.split(__FILE__).first + '/' + name} not found - - ERR - false - end -end - -if subsuite = ARGV.find { |a| break $1 if a[/^([^-].*)/] } - load_suite(subsuite) or exit -else - Dir[mydir + '/*/'].each { |suite| load_suite suite } -end - -if ARGV.include? '-f' - require 'test/unit/ui/fox/testrunner' - UI::Fox::TestRunner.run $suite -else - require 'test/unit/ui/console/testrunner' - UI::Console::TestRunner.run $suite -end diff --git a/test/samples/tokens.expected b/test/samples/tokens.expected deleted file mode 100644 index 747904e5..00000000 --- a/test/samples/tokens.expected +++ /dev/null @@ -1 +0,0 @@ -[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]] diff --git a/test/samples/tokens.rb b/test/samples/tokens.rb deleted file mode 100644 index 91b8abbf..00000000 --- a/test/samples/tokens.rb +++ /dev/null @@ -1,3 +0,0 @@ -require 'coderay' - -p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby) diff --git a/test/unit/comment_filter.rb b/test/unit/comment_filter.rb index e255d07f..c8147e93 100644 --- a/test/unit/comment_filter.rb +++ b/test/unit/comment_filter.rb @@ -47,7 +47,7 @@ def mymethod(self): def myfunction(): -PYTHON_FILTERED + PYTHON_FILTERED end end \ No newline at end of file diff --git a/test/unit/debug.rb b/test/unit/debug.rb index f2b80bd4..b694f21e 100644 --- a/test/unit/debug.rb +++ b/test/unit/debug.rb @@ -18,15 +18,16 @@ def test_creation [:begin_group, :string], ['test', :content], [:end_group, :string], - [:begin_line, :test], + [:begin_line, :head], ["\n", :space], ["\n \t", :space], [" \n", :space], ["[]", :method], - [:end_line, :test], - ].flatten + [:end_line, :head], + ] + TEST_INPUT.flatten! TEST_OUTPUT = <<-'DEBUG'.chomp -integer(10)operator((\\\))stringtest[ +integer(10)operator((\\\))stringhead[ method([])] @@ -62,10 +63,10 @@ def test_creation [:begin_group, :string], ['test', :content], [:end_group, :string], - [:begin_line, :test], + [:begin_line, :unknown], ["\n\n \t \n", :space], ["[]", :method], - [:end_line, :test], + [:end_line, :unknown], ].flatten def test_filtering_text_tokens diff --git a/test/unit/filter.rb b/test/unit/filter.rb index 25dff77c..6e939f32 100644 --- a/test/unit/filter.rb +++ b/test/unit/filter.rb @@ -18,6 +18,7 @@ def test_filtering_text_tokens tokens.text_token i.to_s, :index end assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) + assert_equal CodeRay::Tokens, tokens.filter.class assert_equal tokens, tokens.filter end @@ -32,6 +33,7 @@ def test_filtering_block_tokens tokens.end_line :index end assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens) + assert_equal CodeRay::Tokens, tokens.filter.class assert_equal tokens, tokens.filter end diff --git a/test/unit/json_encoder.rb b/test/unit/json_encoder.rb index 4e44a646..a3a8152b 100644 --- a/test/unit/json_encoder.rb +++ b/test/unit/json_encoder.rb @@ -10,13 +10,13 @@ def test_json_output $:.delete File.dirname(__FILE__) json = CodeRay.scan('puts "Hello world!"', :ruby).json assert_equal [ - {"type"=>"text", "text"=>"puts", "kind"=>"ident"}, - {"type"=>"text", "text"=>" ", "kind"=>"space"}, - {"type"=>"block", "action"=>"open", "kind"=>"string"}, - {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"}, - {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"}, - {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"}, - {"type"=>"block", "action"=>"close", "kind"=>"string"}, + { "type" => "text", "text" => "puts", "kind" => "ident" }, + { "type" => "text", "text" => " ", "kind" => "space" }, + { "type" => "block", "action" => "open", "kind" => "string" }, + { "type" => "text", "text" => "\"", "kind" => "delimiter" }, + { "type" => "text", "text" => "Hello world!", "kind" => "content" }, + { "type" => "text", "text" => "\"", "kind" => "delimiter" }, + { "type" => "block", "action" => "close", "kind" => "string" }, ], JSON.load(json) ensure for path in old_load_paths - $: diff --git a/test/unit/statistic.rb b/test/unit/statistic.rb index 1326dca6..776774d4 100644 --- a/test/unit/statistic.rb +++ b/test/unit/statistic.rb @@ -24,7 +24,8 @@ def test_creation [" \n", :space], ["[]", :method], [:end_line, :test], - ].flatten + ] + TEST_INPUT.flatten! TEST_OUTPUT = <<-'DEBUG' Code Statistics @@ -56,4 +57,4 @@ def test_filtering_text_tokens assert_equal TEST_OUTPUT, TEST_INPUT.statistic end -end \ No newline at end of file +end diff --git a/test/unit/suite.rb b/test/unit/suite.rb index 417dfed8..7d20dc0c 100755 --- a/test/unit/suite.rb +++ b/test/unit/suite.rb @@ -1,3 +1,4 @@ +require 'simplecov' if RUBY_VERSION >= '1.9' require 'test/unit' require 'rubygems'