diff options
43 files changed, 842 insertions, 528 deletions
@@ -1,25 +1,15 @@ .DS_Store -*.gem -*.rbc -.bundle -.config +.*~ coverage -InstalledFiles -lib/bundler/man pkg -rdoc spec/reports -test/tmp -test/version_tmp -tmp doc Gemfile.lock .rvmrc +.ruby-gemset +.ruby-version test/executable/source.rb.html test/executable/source.rb.json test/scanners bench/test.div.html -diff.html -etc/CodeRay.tmproj -*.swp -etc
\ No newline at end of file +old-stuff diff --git a/.travis.yml b/.travis.yml index 63a9b0b..59bb791 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,19 @@ rvm: - 1.8.7 - - 1.9.2 + - ree - 1.9.3 + - 2.0.0 + - ruby-head - jruby-18mode - jruby-19mode + - jruby-head - rbx-18mode - rbx-19mode - # - ruby-head # test again later: RedCloth not compiling - - jruby-head - - ree branches: only: - master +matrix: + allow_failures: + - rvm: rbx-18mode + - rvm: rbx-19mode script: "rake test" # test:scanners" diff --git a/README.textile b/CREDITS.textile index 543dc47..4c58c54 100644 --- a/README.textile +++ b/CREDITS.textile @@ -1,43 +1,4 @@ -h1. CodeRay !https://secure.travis-ci.org/rubychan/coderay.png!:https://secure.travis-ci.org/rubychan/coderay - -h2. About - -CodeRay is a Ruby library for syntax highlighting. - -You put your code in, and you get it back colored; Keywords, strings, floats, comments - all in different colors. And with line numbers. - -*Syntax Highlighting*… - -* makes code easier to read and maintain -* lets you detect syntax errors faster -* helps you to understand the syntax of a language -* looks nice -* is what everybody wants to have on their website -* solves all your problems and makes the girls run after you - - -h2. Installation - -bc. gem install coderay - -h3. Dependencies - -CodeRay needs Ruby 1.8.7+ or 1.9.2+. It also runs on Rubinius and JRuby. - -h2. Example Usage - -bc.. require 'coderay' - -html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table) - -p. - -h2. Documentation - -See "http://coderay.rubychan.de/doc/":http://coderay.rubychan.de/doc/. - - -h2. Credits +h1. Credits h3. Special Thanks to diff --git a/Changes.textile b/Changes.textile index 42af2c8..9570a03 100644 --- a/Changes.textile +++ b/Changes.textile @@ -1,83 +1,103 @@ h1=. CodeRay Version History - + p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ - -{{toc}} - -h2. Next Version - + +h2. Changes in 1.1 + +* New scanner: Sass [#93] +* Diff scanner: Highlight inline changes in multi-line changes [#99] +* JavaScript scanner: Highlight multi-line comments in diff correctly +* Remove double-click toggle handler from HTML table output +* Fixes to CSS scanner (floats, pseudoclasses) +* Plugin does not warn about fallback when default is defined +* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom] +* Add .xaml file type [#121, thanks to Kozman Bálint] +* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner] +* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05] +* Accept keywords as Ruby 1.9 hash keys [#126] +* New token type @:id@ for CSS/Sass [#27] +* CSS scanner uses @:id@ and @:tag@ now [#27] + +h2. Changes in 1.0.9 + +* Fix Ruby scanner: Ruby 1.9 hash syntax @{ key: value }@ is highlighted correctly. [GH #106, thanks to Seth Vargo] +* Fix HTML scanner: Accept DTDs. [GH #83] +* Fix PHP scanner: Accept Unicode. [GH #40, thanks to Lance Li] + +h2. Changes in 1.0.8 + * add @:string/:char@, remove @:regexp/:function@ color from Terminal encoder [GH #29, thanks to Kyrylo Silin] * allow @-@ in line number anchor prefix for HTML encoder [GH #32, thanks to shurizzle] * Fix HTML scanner: Don't crash if HTML in a diff contains a JavaScript tag. h2. Changes in 1.0.7 - + * Changed license from LGPL to MIT. [GH-25, thanks to jessehu] * Fix issue with plugin files not being loaded. [GH-20, thanks to Will Read] * Fix HTML scanner bug: Don't choke on boolean attributes. [GH-26, thanks to jugglinmike] h2. Changes in 1.0.6 - + * New option @:break_lines@ for the HTML encoder (splits tokens at line breaks). [GH-15, thanks to Etienne Massip] * Improved speed of @:line_numbers => :inline@ option for the HTML encoder. * Fixed wrong HTML file type. (was @:page@) [GH-16, thanks to Doug Hammond] * The CSS Scanner now highlights tokens like @url(...)@ as @:function@ instead of @:string@. [GH-13, thanks to Joel Holdbrooks] h2. Changes in 1.0.5 - + Fixes: - + * @autoload@ calls do not depend on @coderay/lib@ being in the load path (GitHub issue #6; thanks to tvon, banister, envygeeks, and ConradIrwin) * avoid dark blue as terminal color (GitHub issue #9; thanks to shevegen) h2. Changes in 1.0.4 - + Fixes in the CSS scanner: - + * understands the unit "s" (seconds) * ignores unexpected curly braces * code inside of diffs is highlighted correctly h2. Changes in 1.0.3 - + New: - + * .tmproj files are recognized as XML. - + Fixes: - + * Removed files are highlighted inside diffs generated by git. h2. Changes in 1.0.2 - + Fixes: - + * .erb files are recognized as ERB. h2. Changes in 1.0.1 - + New: - + * YAML scanner allows "-" and "/" in key names - + Changes: - + * HTML page output has no white border anymore (alpha style) - + Fixes: - + * fixed warning in the output of "coderay stylesheet" * fixed additional scrollbar in code when last line contains an eyecatcher * minor fixes in the tests (issue github-#4) h2. Changes in 1.0 - + CodeRay 1.0 is a major rewrite of the library, and incompatible to earlier versions. - + The command line and programmer interfaces are similar to 0.9, but the internals have completely changed. h3. General changes - + * *NEW*: The new Diff scanner colorizes code inside of the diff, and highlights inline changes. * *NEW*: Extended support and usage of HTML5 and CSS 3 features. * *NEW*: Direct Streaming @@ -89,18 +109,18 @@ h3. General changes * *IMPROVED* Tests: There are more of them now! h3. Direct Streaming - + CodeRay 1.0 introduces _Direct Streaming_ as a faster and simpler alternative to Tokens. It means that all Scanners, Encoders and Filters had to be rewritten, and that older scanners using the Tokens API are no longer compatible with this version. - + The main benefits of this change are: - + * more speed (benchmarks show 10% to 50% more tokens per second compared to CodeRay 0.9) * the ability to stream output into a pipe on the command line * a simpler API * less code - + Changes related to the new tokens handling include: * *CHANGED*: The Scanners now call Encoders directly; tokens are not added to a Tokens array, but are send to the Encoder as a method call. The Tokens representation (which can be seen as a cache now) is still present, but as a @@ -117,16 +137,16 @@ Changes related to the new tokens handling include: and have been removed. h3. Command Line - + The @coderay@ executable was rewritten and has a few new features: - + * *NEW* Ability to stream into a pipe; try @coderay file | more -r@ * *NEW* help * *IMPROVED*: more consistent parameter handling * *REMOVED* @coderay_stylesheet@ executable; use @coderay stylesheet [name]@. h3. @Tokens@ - + * *NEW* methods @count@, @begin_group@, @end_group@, @begin_line@, and @end_line@. * *REMOVED* methods @#stream?@, @#each_text_token@. * *REMOVED* methods @#optimize@, @#fix@, @#split_into_lines@ along with their bang! variants. @@ -134,11 +154,11 @@ h3. @Tokens@ * *REMOVED* special implementation of @#each@ taking a filter parameter. Use @TokenKindFilter@ instead. h3. *RENAMED*: @TokenKinds@ - + Renamed from @Tokens::ClassOfKind@ (was also @Tokens::AbbreviationForKind@ for a while). The term "token class" is no longer used in CodeRay. Instead, tokens have _kinds_. See "#122":http://odd-eyed-code.org/issues/122. - + * *CHANGED* all token CSS classes to readable names. * *ADDED* token kinds @:filename@, @:namespace@, and @:eyecatcher@. * *RENAMED* @:pre_constant@ and @:pre_type@ to @:predefined_constant@ and @predefined_type@. @@ -150,23 +170,23 @@ See "#122":http://odd-eyed-code.org/issues/122. @:NO_HIGHLIGHT@ to @false@. h3. @Duo@ - + * *NEW* method @call@ for allowing code like @CodeRay::Duo[:python => :yaml].(code)@ in Ruby 1.9. h3. @Encoders::CommentFilter@ - + * *NEW* alias @:remove_comments@ h3. @Encoders::Filter@ - + * *NEW* option @tokens@. * *CHANGED*: Now it simply delegates to the output. * *REMOVED* @include_text_token?@ and @include_block_token?@ methods. h3. @Encoders::HTML@ - + The HTML encoder was cleaned up and simplified. - + * *NEW*: HTML5 and CSS 3 compatible. See "#215":http://odd-eyed-code.org/issues/215. * *ADDED* support for @:line_number_anchors@. @@ -180,11 +200,11 @@ The HTML encoder was cleaned up and simplified. * *RENAMED* @Output#numerize@ to @#number@, which is an actual English word. h3. @Encoders::LinesOfCode@ - + * *CHANGED*: @compile@ and @finish@ methods are now protected. h3. *Renamed*: @Encoders::Terminal@ (was @Encoders::Term@) - + * *RENAMED* from @Encoders::Term@, added @:term@ alias. * *CLEANUP*: Use @#setup@'s @super@, don't use @:procedure@ token class. * *CHANGED*: @#token@'s second parameter is no longer optional. @@ -192,21 +212,21 @@ h3. *Renamed*: @Encoders::Terminal@ (was @Encoders::Term@) * *FIXED* handling of line tokens. h3. @Encoders::Text@ - + * *FIXED* default behavior of stripping the trailing newline. h3. *RENAMED*: @Encoders::TokenKindFilter@ (was @Encoders::TokenClassFilter@) - + * *NEW*: Handles token groups. See "#223":http://odd-eyed-code.org/issues/223. * *RENAMED* @include_block_token?@ to @include_group?@. h3. @Encoders::Statistic@ - + * *CHANGED*: Tokens actions are counted separately. h3. @Scanners::Scanner@ - + * *NEW* methods @#file_extension@ and @#encoding@. * *NEW*: The @#tokenize@ method also takes an Array of Strings as source. The code is highlighted as one and split into parts of the input lengths @@ -219,11 +239,11 @@ h3. @Scanners::Scanner@ * *CHANGED*: @#column@ starts counting with 1 instead of 0 h3. *NEW*: @Scanners::Clojure@ - + Thanks to Licenser, CodeRay now supports the Clojure language. h3. @Scanners::CSS@ - + * *NEW*: Rudimentary support for the @attr@, @counter@, and @counters@ functions. See "#224":http://odd-eyed-code.org/issues/224. * *NEW*: Rudimentary support for CSS 3 colors. @@ -231,7 +251,7 @@ h3. @Scanners::CSS@ * *CHANGED*: Comments are scanned as one token instead of three. h3. @Scanners::Debug@ - + * *NEW*: Support for line tokens (@begin_line@ and @end_line@ represented by @[@ and @]@.) * *FIXED*: Don't send @:error@ and @nil@ tokens for buggy input any more. * *FIXED*: Closes unclosed tokens at the end of @scan_tokens@. @@ -239,32 +259,32 @@ h3. @Scanners::Debug@ * *CHANGED*: Raises an error when trying to end an invalid token group. h3. @Scanners::Delphi@ - + * *FIXED*: Closes open string groups. h3. @Scanners::Diff@ - + * *NEW*: Highlighting of code based on file names. See ticket "#52":http://odd-eyed-code.org/issues/52. - + Use the @:highlight_code@ option to turn this feature off. It's enabled by default. - + This is a very original feature. It enables multi-language highlighting for diff files, which is especially helpful for CodeRay development itself. The updated version of the scanner test suite generated .debug.diff.html files using this. - + Note: This is still experimental. Tokens spanning more than one line may get highlighted incorrectly. CodeRay tries to keep scanner states between the lines and changes, but the quality of the results depend on the scanner. * *NEW*: Inline change highlighting, as suggested by Eric Thomas. See ticket "#227":http://odd-eyed-code.org/issues/227 for details. - + Use the @:inline_diff@ option to turn this feature off. It's enabled by default. - + For single-line changes (that is, a single deleted line followed by a single inserted line), this feature surrounds the changed parts with an @:eyecatcher@ group which appears in a more saturated background color. @@ -281,38 +301,38 @@ h3. @Scanners::Diff@ h3. *RENAMED*: @Scanners::ERB@ (was @Scanners::RHTML@) h3. *NEW*: @Scanners::HAML@ - + It uses the new :state options of the HTML and Ruby scanners. - + Some rare cases are not considered (like @#{...}@ snippets inside of :javascript blocks), but it highlights pretty well. h3. @Scanners::HTML@ - + * *FIXED*: Closes open string groups. h3. @Scanners::JavaScript@ - + * *IMPROVED*: Added @NaN@ and @Infinity@ to list of predefined constants. * *IMPROVED* recognition of RegExp literals with leading spaces. h3. @Scanners::Java@ - + * *NEW*: Package names are highlighted as @:namespace@. See "#210":http://odd-eyed-code.org/issues/210. h3. *REMOVED*: @Scanners::NitroXHTML@ - + Nitro is "dead":http://www.nitrohq.com/. h3. *RENAMED*: @Scanners::Text@ (was @Scanners::Plaintext@) - + * *IMPROVED*: Just returns the string without scanning (faster). - + This is much faster than scanning until @/\z/@ in Ruby 1.8. h3. @Scanners::Python@ - + * *CHANGED*: Docstrings are highlighted as @:comment@. See "#190":http://odd-eyed-code.org/issues/190. @@ -322,7 +342,7 @@ Copied from @Scanners::Debug@, highlights the token dump instead of importing it name suffix now. h3. @Scanners::Ruby@ - + * *ADDED* more predefined keywords (see http://murfy.de/ruby-constants). * *IMPROVED* support for singleton method definitions. See "#147":http://odd-eyed-code.org/issues/147. @@ -330,44 +350,44 @@ h3. @Scanners::Ruby@ (eg. @GL.PushMatrix@). * *NEW*: Highlight buggy floats (like .5) as @:error@. * *CLEANUP* of documentation, names of constants and variables, state handling. - + Moved @StringState@ class from @patterns.rb@ into a separate file. * *NEW*: Complicated rule for recognition of @foo=@ style method names. * *NEW*: Handles @:keep_state@ option (a bit; experimental). - + Actually, Ruby checks if there is @[~>=]@, but not @=>@ following the name. - + * *REMOVED* @EncodingError@ h3. *REMOVED* @Scanners::Scheme@ - + * It is too buggy, and nobody was using it. To be added again when it's fixed. See "#59":http://odd-eyed-code.org/issues/59. h3. @Scanners::SQL@ - + * *IMPROVED*: Extended list of keywords and functions (thanks to Joshua Galvez, Etienne Massip, and others). - + See "#221":http://odd-eyed-code.org/issues/221. * *FIXED*: Closes open string groups. * *FIXED*: Words after @.@ are always recognized as @:ident@. h3. @Scanners::YAML@ - + * *FIXED*: Allow spaces before colon in mappings. - + See "#231":http://odd-eyed-code.org/issues/231. h3. *NEW*: @Styles::Alpha@ A style that uses transparent HSLA colors as defined in CSS 3. See "#199":http://odd-eyed-code.org/issues/199. - + It also uses the CSS 3 property @user-select: none@ to keep the user from selecting the line numbers. This is especially nice for @:inline@ line numbers. See "#226":http://odd-eyed-code.org/issues/226. h3. @WordList@ - + Stripped down to 19 LOC. * *RENAMED* @CaseIgnoringWordList@ to @WordList::CaseIgnoring@. @@ -375,14 +395,14 @@ Stripped down to 19 LOC. * *REMOVED* block option. h3. @FileType@ - + * *NEW*: Recognizes @.gemspec@, @.rjs@, @.rpdf@ extensions, @Gemfile@, and @Capfile@ as Ruby. - + Thanks to the authors of the TextMate Ruby bundle! * *REMOVED* @FileType#shebang@ is a protected method now. h3. @Plugin@ - + * *IMPROVED*: @register_for@ sets the @plugin_id@; it can now be a @Symbol@. * *ADDED* @PluginHost#const_missing@ method: Plugins are loaded automatically. Using @Scanners::JavaScript@ in your code loads @scanners/java_script.rb@. @@ -391,19 +411,19 @@ h3. @Plugin@ * *CHANGED* the default plugin key from @nil@ to @:default@. h3. @GZip@ - + * *MOVED* into @CodeRay@ namespace. * *MOVED* file from @gzip_simple.rb@ to @gzip.rb@. * *REMOVED* @String@ extensions. h3. More API changes - + * *FIXED* @Encoders::HTML#token@'s second parameter is no longer optional. * *CHANGED* @Encoders::HTML::Output@'s API. * *REMOVED* lots of unused methods. - + The helper classes were cleaned up; see above for details. - + * *CHANGED* @Plugin@ API was simplified and stripped of all unnecessary features. * *CHANGED* Moved @GZip@ and @FileType@ libraries into @CodeRay@; cleaned them up. @@ -1,4 +1,4 @@ -source "http://rubygems.org" +source 'https://rubygems.org' # Specify your gem's dependencies in coderay.gemspec gemspec @@ -7,10 +7,10 @@ gemspec # Include everything needed to run rake, tests, features, etc. group :development do gem "bundler", ">= 1.0.0" - gem "rake", "~> 0.9.2" + gem "rake" gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor" - gem "shoulda-context", "~> 1.0.0" if RUBY_VERSION >= '1.8.7' - gem "json" unless RUBY_VERSION >= '1.9.1' - gem "rdoc" if RUBY_VERSION >= '1.8.7' + gem "term-ansicolor", '~> 1.2.2' + gem "shoulda-context", "~> 1.1.2" + gem "json" if RUBY_VERSION < '1.9' + gem "rdoc" end diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000..f333655 --- /dev/null +++ b/README.markdown @@ -0,0 +1,31 @@ +# CodeRay + +[](https://travis-ci.org/rubychan/coderay) +[](http://badge.fury.io/rb/coderay) +[](https://gemnasium.com/rubychan/coderay) + +## About + +CodeRay is a Ruby library for syntax highlighting. + +You put your code in, and you get it back colored; Keywords, strings, floats, comments - all in different colors. And with line numbers. + +## Installation + +`gem install coderay` + +### Dependencies + +CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby. + +## Example Usage + +```ruby +require 'coderay' + +html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table) +```` + +## Documentation + +See [http://coderay.rubychan.de/doc/](http://coderay.rubychan.de/doc/). diff --git a/bench/bench.rb b/bench/bench.rb index 45dc5b0..1889eed 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -108,7 +108,7 @@ N.times do $file_created = here('test.' + ($dump_output ? 'dump' : $hl.file_extension)) File.open($file_created, 'wb') do |f| - f.write $o + # f.write $o end Dir.chdir(here) do FileUtils.copy 'test.dump', 'example.dump' if $dump_output diff --git a/bin/coderay b/bin/coderay index d78cd57..9aecb88 100755 --- a/bin/coderay +++ b/bin/coderay @@ -125,7 +125,7 @@ when 'highlight', nil end if output_file - output_format ||= CodeRay::FileType[output_file] + output_format ||= CodeRay::FileType[output_file] || :plain else output_format ||= :terminal end @@ -143,7 +143,6 @@ when 'highlight', nil if output_file File.open output_file, 'w' else - $stdout.sync = true $stdout end CodeRay.encode(input, input_lang, output_format, :out => file) diff --git a/coderay.gemspec b/coderay.gemspec index e686035..328b94c 100644 --- a/coderay.gemspec +++ b/coderay.gemspec @@ -17,12 +17,14 @@ Gem::Specification.new do |s| s.summary = 'Fast syntax highlighting for selected languages.' s.description = 'Fast and easy syntax highlighting for selected languages, written in Ruby. Comes with RedCloth integration and LOC counter.' + s.license = 'MIT' + s.platform = Gem::Platform::RUBY s.required_ruby_version = '>= 1.8.6' readme_file = 'README_INDEX.rdoc' - s.files = `git ls-files -- lib/* test/functional/* Rakefile #{readme_file} LICENSE`.split("\n") + s.files = `git ls-files -- lib/* test/functional/* Rakefile #{readme_file} MIT-LICENSE`.split("\n") s.test_files = `git ls-files -- test/functional/*`.split("\n") s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } s.require_paths = ['lib'] diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb index 95d6138..c03d3fb 100644 --- a/lib/coderay/encoders/debug.rb +++ b/lib/coderay/encoders/debug.rb @@ -24,11 +24,12 @@ module Encoders end def text_token text, kind + raise 'empty token' if $CODERAY_DEBUG && text.empty? if kind == :space @out << text else # TODO: Escape ( - text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \ + text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/) @out << kind.to_s << '(' << text << ')' end end diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 635a4d8..b897f5e 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -126,22 +126,21 @@ module Encoders protected - HTML_ESCAPE = { #:nodoc: - '&' => '&', - '"' => '"', - '>' => '>', - '<' => '<', - } + def self.make_html_escape_hash + { + '&' => '&', + '"' => '"', + '>' => '>', + '<' => '<', + # "\t" => will be set to ' ' * options[:tab_width] during setup + }.tap do |hash| + # Escape ASCII control codes except \x9 == \t and \xA == \n. + (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' } + end + end - # This was to prevent illegal HTML. - # Strange chars should still be avoided in codes. - evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s] - evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' } - #ansi_chars = Array(0x7f..0xff) - #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i } - # \x9 (\t) and \xA (\n) not included - #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/ - HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/ + HTML_ESCAPE = make_html_escape_hash + HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/ TOKEN_KIND_TO_INFO = Hash.new do |h, kind| h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } @@ -172,59 +171,22 @@ module Encoders def setup options super + check_options! options + if options[:wrap] || options[:line_numbers] @real_out = @out @out = '' end - options[:break_lines] = true if options[:line_numbers] == :inline - @break_lines = (options[:break_lines] == true) - @HTML_ESCAPE = HTML_ESCAPE.dup - @HTML_ESCAPE["\t"] = ' ' * options[:tab_width] + @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width]) @opened = [] @last_opened = nil @css = CSS.new options[:style] - hint = options[:hint] - if hint && ![:debug, :info, :info_long].include?(hint) - raise ArgumentError, "Unknown value %p for :hint; \ - expected :info, :info_long, :debug, false, or nil." % hint - end - - css_classes = TokenKinds - case options[:css] - when :class - @span_for_kind = Hash.new do |h, k| - if k.is_a? ::Symbol - kind = k_dup = k - else - kind = k.first - k_dup = k.dup - end - if kind != :space && (hint || css_class = css_classes[kind]) - title = HTML.token_path_to_hint hint, k if hint - css_class ||= css_classes[kind] - h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>" - else - h[k_dup] = nil - end - end - when :style - @span_for_kind = Hash.new do |h, k| - kind = k.is_a?(Symbol) ? k : k.first - h[k.is_a?(Symbol) ? k : k.dup] = - if kind != :space && (hint || css_classes[kind]) - title = HTML.token_path_to_hint hint, k if hint - style = @css.get_style Array(k).map { |c| css_classes[c] } - "<span#{title}#{" style=\"#{style}\"" if style}>" - end - end - else - raise ArgumentError, "Unknown value %p for :css." % options[:css] - end + @span_for_kinds = make_span_for_kinds(options[:css], options[:hint]) @set_last_opened = options[:hint] || options[:css] == :style end @@ -255,20 +217,10 @@ module Encoders public def text_token text, kind - if text =~ /#{HTML_ESCAPE_PATTERN}/o - text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } - end + style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind] - style = @span_for_kind[@last_opened ? [kind, *@opened] : kind] - - if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0 - close = '</span>' * c - reopen = '' - @opened.each_with_index do |k, index| - reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>') - end - text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}") - end + text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o + text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n") if style @out << style << text << '</span>' @@ -279,25 +231,19 @@ module Encoders # token groups, eg. strings def begin_group kind - @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>') + @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>') @opened << kind @last_opened = kind if @set_last_opened end def end_group kind - if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind) - warn 'Malformed token stream: Trying to close a token (%p) ' \ - 'that is not open. Open are: %p.' % [kind, @opened[1..-1]] - end - if @opened.pop - @out << '</span>' - @last_opened = @opened.last if @last_opened - end + check_group_nesting 'token group', kind if $CODERAY_DEBUG + close_span end # whole lines to be highlighted, eg. a deleted line in a diff def begin_line kind - if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind] + if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind] if style['class="'] @out << style.sub('class="', 'class="line ') else @@ -311,16 +257,71 @@ module Encoders end def end_line kind - if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind) - warn 'Malformed token stream: Trying to close a line (%p) ' \ - 'that is not open. Open are: %p.' % [kind, @opened[1..-1]] + check_group_nesting 'line', kind if $CODERAY_DEBUG + close_span + end + + protected + + def check_options! options + unless [false, nil, :debug, :info, :info_long].include? options[:hint] + raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]] + end + + unless [:class, :style].include? options[:css] + raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]] + end + + options[:break_lines] = true if options[:line_numbers] == :inline + end + + def css_class_for_kinds kinds + TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first] + end + + def style_for_kinds kinds + css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]] + @css.get_style_for_css_classes css_classes + end + + def make_span_for_kinds method, hint + Hash.new do |h, kinds| + h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin + css_class = css_class_for_kinds(kinds) + title = HTML.token_path_to_hint hint, kinds if hint + + if css_class || title + if method == :style + style = style_for_kinds(kinds) + "<span#{title}#{" style=\"#{style}\"" if style}>" + else + "<span#{title}#{" class=\"#{css_class}\"" if css_class}>" + end + end + end + end + end + + def check_group_nesting name, kind + if @opened.empty? || @opened.last != kind + warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]] end + end + + def break_lines text, style + reopen = '' + @opened.each_with_index do |k, index| + reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>') + end + text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}") + end + + def close_span if @opened.pop @out << '</span>' @last_opened = @opened.last if @last_opened end end - end end diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb index 6de4b46..164d7f8 100644 --- a/lib/coderay/encoders/html/css.rb +++ b/lib/coderay/encoders/html/css.rb @@ -11,7 +11,7 @@ module Encoders end def initialize style = :default - @classes = Hash.new + @styles = Hash.new style = CSS.load_stylesheet style @stylesheet = [ style::CSS_MAIN_STYLES, @@ -20,12 +20,12 @@ module Encoders parse style::TOKEN_COLORS end - def get_style styles - cl = @classes[styles.first] + def get_style_for_css_classes css_classes + cl = @styles[css_classes.first] return '' unless cl style = '' - 1.upto styles.size do |offset| - break if style = cl[styles[offset .. -1]] + 1.upto css_classes.size do |offset| + break if style = cl[css_classes[offset .. -1]] end # warn 'Style not found: %p' % [styles] if style.empty? return style @@ -52,8 +52,8 @@ module Encoders for selector in selectors.split(',') classes = selector.scan(/[-\w]+/) cl = classes.pop - @classes[cl] ||= Hash.new - @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';') + @styles[cl] ||= Hash.new + @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';') end end end diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb index e717429..332145b 100644 --- a/lib/coderay/encoders/html/numbering.rb +++ b/lib/coderay/encoders/html/numbering.rb @@ -1,15 +1,15 @@ module CodeRay module Encoders - + class HTML - + module Numbering # :nodoc: - + def self.number! output, mode = :table, options = {} return self unless mode - + options = DEFAULT_OPTIONS.merge options - + start = options[:line_number_start] unless start.is_a? Integer raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start @@ -56,12 +56,17 @@ module Encoders raise ArgumentError, 'Invalid value %p for :bolding; false or Integer expected.' % bold_every end - line_count = output.count("\n") - position_of_last_newline = output.rindex(RUBY_VERSION >= '1.9' ? /\n/ : ?\n) - if position_of_last_newline + if position_of_last_newline = output.rindex(RUBY_VERSION >= '1.9' ? /\n/ : ?\n) after_last_newline = output[position_of_last_newline + 1 .. -1] ends_with_newline = after_last_newline[/\A(?:<\/span>)*\z/] - line_count += 1 if not ends_with_newline + + if ends_with_newline + line_count = output.count("\n") + else + line_count = output.count("\n") + 1 + end + else + line_count = 1 end case mode @@ -74,30 +79,30 @@ module Encoders line_number += 1 "<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}" end - + when :table line_numbers = (start ... start + line_count).map(&bolding).join("\n") line_numbers << "\n" line_numbers_table_template = Output::TABLE.apply('LINE_NUMBERS', line_numbers) - + output.gsub!(/<\/div>\n/, '</div>') output.wrap_in! line_numbers_table_template output.wrapped_in = :div - + when :list raise NotImplementedError, 'The :list option is no longer available. Use :table.' - + else raise ArgumentError, 'Unknown value %p for mode: expected one of %p' % [mode, [:table, :inline]] end - + output end - + end - + end - + end end diff --git a/lib/coderay/encoders/html/output.rb b/lib/coderay/encoders/html/output.rb index 9132d94..de6f6ea 100644 --- a/lib/coderay/encoders/html/output.rb +++ b/lib/coderay/encoders/html/output.rb @@ -124,7 +124,7 @@ module Encoders TABLE = Template.new <<-TABLE <table class="CodeRay"><tr> - <td class="line-numbers" title="double click to toggle" ondblclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td> + <td class="line-numbers"><pre><%LINE_NUMBERS%></pre></td> <td class="code"><pre><%CONTENT%></pre></td> </tr></table> TABLE diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb index a0ceb3c..500e5d8 100644 --- a/lib/coderay/encoders/terminal.rb +++ b/lib/coderay/encoders/terminal.rb @@ -19,73 +19,73 @@ module CodeRay register_for :terminal TOKEN_COLORS = { - :annotation => '35', - :attribute_name => '33', - :attribute_value => '31', - :binary => '1;35', + :annotation => "\e[35m", + :attribute_name => "\e[33m", + :attribute_value => "\e[31m", + :binary => "\e[1;35m", :char => { - :self => '36', :delimiter => '1;34' + :self => "\e[36m", :delimiter => "\e[1;34m" }, - :class => '1;35', - :class_variable => '36', - :color => '32', - :comment => '37', - :complex => '1;34', - :constant => ['1;34', '4'], - :decoration => '35', - :definition => '1;32', - :directive => ['32', '4'], - :doc => '46', - :doctype => '1;30', - :doc_string => ['31', '4'], - :entity => '33', - :error => ['1;33', '41'], - :exception => '1;31', - :float => '1;35', - :function => '1;34', - :global_variable => '42', - :hex => '1;36', - :include => '33', - :integer => '1;34', - :key => '35', - :label => '1;15', - :local_variable => '33', - :octal => '1;35', - :operator_name => '1;29', - :predefined_constant => '1;36', - :predefined_type => '1;30', - :predefined => ['4', '1;34'], - :preprocessor => '36', - :pseudo_class => '1;34', + :class => "\e[1;35m", + :class_variable => "\e[36m", + :color => "\e[32m", + :comment => "\e[37m", + :complex => "\e[1;34m", + :constant => "\e[1;34m\e[4m", + :decoration => "\e[35m", + :definition => "\e[1;32m", + :directive => "\e[32m\e[4m", + :doc => "\e[46m", + :doctype => "\e[1;30m", + :doc_string => "\e[31m\e[4m", + :entity => "\e[33m", + :error => "\e[1;33m\e[41m", + :exception => "\e[1;31m", + :float => "\e[1;35m", + :function => "\e[1;34m", + :global_variable => "\e[42m", + :hex => "\e[1;36m", + :include => "\e[33m", + :integer => "\e[1;34m", + :key => "\e[35m", + :label => "\e[1;15m", + :local_variable => "\e[33m", + :octal => "\e[1;35m", + :operator_name => "\e[1;29m", + :predefined_constant => "\e[1;36m", + :predefined_type => "\e[1;30m", + :predefined => "\e[4m\e[1;34m", + :preprocessor => "\e[36m", + :pseudo_class => "\e[1;34m", :regexp => { - :self => '31', - :content => '31', - :delimiter => '1;29', - :modifier => '35', + :self => "\e[31m", + :content => "\e[31m", + :delimiter => "\e[1;29m", + :modifier => "\e[35m", }, - :reserved => '1;31', + :reserved => "\e[1;31m", :shell => { - :self => '42', - :content => '1;29', - :delimiter => '37', + :self => "\e[42m", + :content => "\e[1;29m", + :delimiter => "\e[37m", }, :string => { - :self => '32', - :modifier => '1;32', - :escape => '1;36', - :delimiter => '1;32', - :char => '1;36', + :self => "\e[32m", + :modifier => "\e[1;32m", + :escape => "\e[1;36m", + :delimiter => "\e[1;32m", + :char => "\e[1;36m", }, - :symbol => '1;32', - :tag => '1;34', - :type => '1;34', - :value => '36', - :variable => '1;34', + :symbol => "\e[1;32m", + :tag => "\e[1;34m", + :type => "\e[1;34m", + :value => "\e[36m", + :variable => "\e[1;34m", - :insert => '42', - :delete => '41', - :change => '44', - :head => '45' + :insert => "\e[42m", + :delete => "\e[41m", + :change => "\e[44m", + :head => "\e[45m" } TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved] TOKEN_COLORS[:method] = TOKEN_COLORS[:function] @@ -114,10 +114,10 @@ module CodeRay end end - @out << ansi_colorize(color) - @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color)) - @out << ansi_clear - @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self] + @out << color + @out << text.gsub("\n", "\e[0m\n" + color) + @out << "\e[0m" + @out << @subcolors[:self] if @subcolors else @out << text end @@ -134,7 +134,7 @@ module CodeRay # nothing to close else @opened.pop - @out << ansi_clear + @out << "\e[0m" @out << open_token(@opened.last) end end @@ -146,7 +146,7 @@ module CodeRay @opened.pop # whole lines to be highlighted, # eg. added/modified/deleted lines in a diff - @out << "\t" * 100 + ansi_clear + @out << (@line_filler ||= "\t" * 100 + "\e[0m") @out << open_token(@opened.last) end end @@ -157,23 +157,16 @@ module CodeRay if color = TOKEN_COLORS[kind] if Hash === color @subcolors = color - ansi_colorize(color[:self]) if color[:self] + color[:self] else @subcolors = {} - ansi_colorize(color) + color end else @subcolors = nil '' end end - - def ansi_colorize(color) - Array(color).map { |c| "\e[#{c}m" }.join - end - def ansi_clear - ansi_colorize(0) - end end end -end
\ No newline at end of file +end diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 637001b..a5d83ff 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -99,6 +99,7 @@ module CodeRay 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, + 'phtml' => :php, 'php' => :php, 'php3' => :php, 'php4' => :php, @@ -116,10 +117,10 @@ module CodeRay 'rpdf' => :ruby, 'ru' => :ruby, 'rxml' => :ruby, - # 'sch' => :scheme, + 'sass' => :sass, 'sql' => :sql, - # 'ss' => :scheme, 'tmproj' => :xml, + 'xaml' => :xml, 'xhtml' => :html, 'xml' => :xml, 'yaml' => :yaml, diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb index 137c1ab..d14c5a9 100644 --- a/lib/coderay/helpers/plugin.rb +++ b/lib/coderay/helpers/plugin.rb @@ -131,7 +131,7 @@ module CodeRay # A Hash of plugion_id => Plugin pairs. def plugin_hash - @plugin_hash ||= make_plugin_hash + @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map } end # Returns an array of all .rb files in the plugin path. @@ -158,7 +158,6 @@ module CodeRay # This is done automatically when plugin_path is called. def load_plugin_map mapfile = path_to '_map' - @plugin_map_loaded = true if File.exist? mapfile require mapfile true @@ -171,23 +170,16 @@ module CodeRay # Return a plugin hash that automatically loads plugins. def make_plugin_hash - @plugin_map_loaded ||= false Hash.new do |h, plugin_id| id = validate_id(plugin_id) path = path_to id begin require path rescue LoadError => boom - if @plugin_map_loaded - if h.has_key?(:default) - warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]] - h[:default] - else - raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] - end + if h.has_key?(:default) + h[:default] else - load_plugin_map - h[plugin_id] + raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom] end else # Plugin should have registered by now @@ -271,7 +263,6 @@ module CodeRay end def aliases - plugin_host.load_plugin_map plugin_host.plugin_hash.inject [] do |aliases, (key, _)| aliases << key if plugin_host[key] == self aliases diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 907cf00..b3f7e17 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -182,16 +182,9 @@ module CodeRay # Scan the code and returns all tokens in a Tokens object. def tokenize source = nil, options = {} options = @options.merge(options) - @tokens = options[:tokens] || @tokens || Tokens.new - @tokens.scanner = self if @tokens.respond_to? :scanner= - case source - when Array - self.string = self.class.normalize(source.join) - when nil - reset - else - self.string = self.class.normalize(source) - end + + set_tokens_from_options options + set_string_from_source source begin scan_tokens @tokens, options @@ -261,6 +254,22 @@ module CodeRay def setup # :doc: end + def set_string_from_source source + case source + when Array + self.string = self.class.normalize(source.join) + when nil + reset + else + self.string = self.class.normalize(source) + end + end + + def set_tokens_from_options options + @tokens = options[:tokens] || @tokens || Tokens.new + @tokens.scanner = self if @tokens.respond_to? :scanner= + end + # This is the central method, and commonly the only one a # subclass implements. # @@ -277,19 +286,15 @@ module CodeRay @binary_string = nil if defined? @binary_string end - # Scanner error with additional status information - def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller - raise ScanError, <<-EOE % [ + SCAN_ERROR_MESSAGE = <<-MESSAGE -***ERROR in %s: %s (after %d tokens) +***ERROR in %s: %s (after %s tokens) tokens: %s -current line: %d column: %d pos: %d -matched: %p state: %p -bol? = %p, eos? = %p +%s surrounding code: %p ~~ %p @@ -297,16 +302,43 @@ surrounding code: ***ERROR*** - EOE - File.basename(caller[0]), - msg, - tokens.respond_to?(:size) ? tokens.size : 0, - tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '', - line, column, pos, - matched, state, bol?, eos?, + MESSAGE + + def raise_inspect_arguments message, tokens, state, ambit + return File.basename(caller[0]), + message, + tokens_size(tokens), + tokens_last(tokens, 10).map(&:inspect).join("\n"), + scanner_state_info(state), binary_string[pos - ambit, ambit], - binary_string[pos, ambit], - ], backtrace + binary_string[pos, ambit] + end + + SCANNER_STATE_INFO = <<-INFO +current line: %d column: %d pos: %d +matched: %p state: %p +bol?: %p, eos?: %p + INFO + + def scanner_state_info state + SCANNER_STATE_INFO % [ + line, column, pos, + matched, state || 'No state given!', + bol?, eos?, + ] + end + + # Scanner error with additional status information + def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller + raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace + end + + def tokens_size tokens + tokens.size if tokens.respond_to?(:size) + end + + def tokens_last tokens, n + tokens.respond_to?(:last) ? tokens.last(n) : [] end # Shorthand for scan_until(/\z/). diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb index 8d24b99..84b6e8e 100644 --- a/lib/coderay/scanners/c.rb +++ b/lib/coderay/scanners/c.rb @@ -148,7 +148,7 @@ module Scanners encoder.text_token match, :char elsif match = scan(/ \\ | $ /x) encoder.end_group :string - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index 9da62f4..e61f56f 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -160,7 +160,7 @@ module Scanners encoder.text_token match, :char elsif match = scan(/ \\ | $ /x) encoder.end_group :string - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial label_expected = false else diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb index 7b731ef..732f9c5 100644 --- a/lib/coderay/scanners/css.rb +++ b/lib/coderay/scanners/css.rb @@ -7,27 +7,25 @@ module Scanners KINDS_NOT_LOC = [ :comment, - :class, :pseudo_class, :type, - :constant, :directive, + :class, :pseudo_class, :tag, + :id, :directive, :key, :value, :operator, :color, :float, :string, - :error, :important, + :error, :important, :type, ] # :nodoc: module RE # :nodoc: Hex = /[0-9a-fA-F]/ - Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too - Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/ - NMChar = /[-_a-zA-Z0-9]|#{Escape}/ - NMStart = /[_a-zA-Z]|#{Escape}/ - NL = /\r\n|\r|\n|\f/ - String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # TODO: buggy regexp - String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # TODO: buggy regexp + Unicode = /\\#{Hex}{1,6}\b/ # differs from standard because it allows uppercase hex too + Escape = /#{Unicode}|\\[^\n0-9a-fA-F]/ + NMChar = /[-_a-zA-Z0-9]/ + NMStart = /[_a-zA-Z]/ + String1 = /"(?:[^\n\\"]+|\\\n|#{Escape})*"?/ # TODO: buggy regexp + String2 = /'(?:[^\n\\']+|\\\n|#{Escape})*'?/ # TODO: buggy regexp String = /#{String1}|#{String2}/ HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/ - Color = /#{HexColor}/ - Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/ + Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/ Name = /#{NMChar}+/ Ident = /-?#{NMStart}#{NMChar}*/ AtKeyword = /@#{Ident}/ @@ -35,16 +33,15 @@ module Scanners reldimensions = %w[em ex px] absdimensions = %w[in cm mm pt pc] - Unit = Regexp.union(*(reldimensions + absdimensions + %w[s])) + Unit = Regexp.union(*(reldimensions + absdimensions + %w[s dpi dppx deg])) Dimension = /#{Num}#{Unit}/ - Comment = %r! /\* (?: .*? \*/ | .* ) !mx - Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/ + Function = /(?:url|alpha|attr|counters?)\((?:[^)\n]|\\\))*\)?/ - Id = /##{Name}/ + Id = /(?!#{HexColor}\b(?!-))##{Name}/ Class = /\.#{Name}/ - PseudoClass = /:#{Name}/ + PseudoClass = /::?#{Ident}/ AttributeSelector = /\[[^\]]*\]?/ end @@ -52,7 +49,7 @@ module Scanners def setup @state = :initial - @value_expected = nil + @value_expected = false end def scan_tokens encoder, options @@ -67,13 +64,13 @@ module Scanners elsif case states.last when :initial, :media if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox) - encoder.text_token match, :type + encoder.text_token match, :tag next elsif match = scan(RE::Class) encoder.text_token match, :class next elsif match = scan(RE::Id) - encoder.text_token match, :constant + encoder.text_token match, :id next elsif match = scan(RE::PseudoClass) encoder.text_token match, :pseudo_class @@ -158,7 +155,7 @@ module Scanners elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox) encoder.text_token match, :float - elsif match = scan(/#{RE::Color}/o) + elsif match = scan(/#{RE::HexColor}/o) encoder.text_token match, :color elsif match = scan(/! *important/) @@ -170,7 +167,7 @@ module Scanners elsif match = scan(RE::AtKeyword) encoder.text_token match, :directive - elsif match = scan(/ [+>:;,.=()\/] /x) + elsif match = scan(/ [+>~:;,.=()\/] /x) if match == ':' value_expected = true elsif match == ';' diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb index 9e899c3..af0f755 100644 --- a/lib/coderay/scanners/diff.rb +++ b/lib/coderay/scanners/diff.rb @@ -20,7 +20,7 @@ module Scanners line_kind = nil state = :initial - deleted_lines = 0 + deleted_lines_count = 0 scanners = Hash.new do |h, lang| h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true end @@ -30,7 +30,7 @@ module Scanners until eos? if match = scan(/\n/) - deleted_lines = 0 unless line_kind == :delete + deleted_lines_count = 0 unless line_kind == :delete if line_kind encoder.end_line line_kind line_kind = nil @@ -45,7 +45,7 @@ module Scanners if match = scan(/--- |\+\+\+ |=+|_+/) encoder.begin_line line_kind = :head encoder.text_token match, :head - if match = scan(/.*?(?=$|[\t\n\x00]| \(revision)/) + if match = scan(/[^\x00\n]+?(?=$|[\t\n]| \(revision)/) encoder.text_token match, :filename if options[:highlight_code] && match != '/dev/null' file_type = CodeRay::FileType.fetch(match, :text) @@ -99,37 +99,59 @@ module Scanners end next elsif match = scan(/-/) - deleted_lines += 1 - encoder.begin_line line_kind = :delete - encoder.text_token match, :delete - if options[:inline_diff] && deleted_lines == 1 && check(/(?>.*)\n\+(?>.*)$(?!\n\+)/) - content_scanner_entry_state = content_scanner.state - skip(/(.*)\n\+(.*)$/) - head, deletion, insertion, tail = diff self[1], self[2] - pre, deleted, post = content_scanner.tokenize [head, deletion, tail], :tokens => Tokens.new - encoder.tokens pre - unless deleted.empty? - encoder.begin_group :eyecatcher - encoder.tokens deleted - encoder.end_group :eyecatcher + deleted_lines_count += 1 + if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/) + deleted_lines = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) } + inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/) ; scan(/.*/) } + + deleted_lines_tokenized = [] + inserted_lines_tokenized = [] + for deleted_line, inserted_line in deleted_lines.zip(inserted_lines) + pre, deleted_part, inserted_part, post = diff deleted_line, inserted_line + content_scanner_entry_state = content_scanner.state + deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new) + content_scanner.state = content_scanner_entry_state || :initial + inserted_lines_tokenized << content_scanner.tokenize([pre, inserted_part, post], :tokens => Tokens.new) end - encoder.tokens post - encoder.end_line line_kind - encoder.text_token "\n", :space - encoder.begin_line line_kind = :insert - encoder.text_token '+', :insert - content_scanner.state = content_scanner_entry_state || :initial - pre, inserted, post = content_scanner.tokenize [head, insertion, tail], :tokens => Tokens.new - encoder.tokens pre - unless inserted.empty? - encoder.begin_group :eyecatcher - encoder.tokens inserted - encoder.end_group :eyecatcher + + for pre, deleted_part, post in deleted_lines_tokenized + encoder.begin_line :delete + encoder.text_token '-', :delete + encoder.tokens pre + unless deleted_part.empty? + encoder.begin_group :eyecatcher + encoder.tokens deleted_part + encoder.end_group :eyecatcher + end + encoder.tokens post + encoder.end_line :delete + encoder.text_token "\n", :space + end + + for pre, inserted_part, post in inserted_lines_tokenized + encoder.begin_line :insert + encoder.text_token '+', :insert + encoder.tokens pre + unless inserted_part.empty? + encoder.begin_group :eyecatcher + encoder.tokens inserted_part + encoder.end_group :eyecatcher + end + encoder.tokens post + changed_lines_count -= 1 + if changed_lines_count > 0 + encoder.end_line :insert + encoder.text_token "\n", :space + end end - encoder.tokens post + + line_kind = :insert + elsif match = scan(/.*/) + encoder.begin_line line_kind = :delete + encoder.text_token '-', :delete if options[:highlight_code] - if deleted_lines == 1 + if deleted_lines_count == 1 content_scanner_entry_state = content_scanner.state end content_scanner.tokenize match, :tokens => encoder unless match.empty? diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb index 49c346d..3ba3b79 100644 --- a/lib/coderay/scanners/html.rb +++ b/lib/coderay/scanners/html.rb @@ -101,7 +101,7 @@ module Scanners when :initial if match = scan(/<!--(?:.*?-->|.*)/m) encoder.text_token match, :comment - elsif match = scan(/<!DOCTYPE(?:.*?>|.*)/m) + elsif match = scan(/<!(\w+)(?:.*?>|.*)|\]>/m) encoder.text_token match, :doctype elsif match = scan(/<\?xml(?:.*?\?>|.*)/m) encoder.text_token match, :preprocessor diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index c1490ac..b282864 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -147,7 +147,7 @@ module Scanners elsif match = scan(/ \\ | $ /x) encoder.end_group state state = :initial - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder end diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb index 43ecb18..9eb0a0a 100644 --- a/lib/coderay/scanners/java_script.rb +++ b/lib/coderay/scanners/java_script.rb @@ -54,10 +54,17 @@ module Scanners protected + def setup + @state = :initial + end + def scan_tokens encoder, options - state = :initial - string_delimiter = nil + state, string_delimiter = options[:state] || @state + if string_delimiter + encoder.begin_group state + end + value_expected = true key_expected = false function_expected = false @@ -72,9 +79,10 @@ module Scanners value_expected = true if !value_expected && match.index(?\n) encoder.text_token match, :space - elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx) + elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .*() ) !mx) value_expected = true encoder.text_token match, :comment + state = :open_multi_line_comment if self[1] elsif check(/\.?\d/) key_expected = value_expected = false @@ -175,20 +183,36 @@ module Scanners encoder.text_token match, :content elsif match = scan(/ \\ | $ /x) encoder.end_group state - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? + string_delimiter = nil key_expected = value_expected = false state = :initial else - raise_inspect "else case \" reached; %p not handled." % peek(1), encoder + raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder end + when :open_multi_line_comment + if match = scan(%r! .*? \*/ !mx) + state = :initial + else + match = scan(%r! .+ !mx) + end + value_expected = true + encoder.text_token match, :comment if match + else - raise_inspect 'Unknown state', encoder + #:nocov: + raise_inspect 'Unknown state: %p' % [state], encoder + #:nocov: end end + if options[:keep_state] + @state = state, string_delimiter + end + if [:string, :regexp].include? state encoder.end_group state end diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb index 0c90c34..4e0f462 100644 --- a/lib/coderay/scanners/json.rb +++ b/lib/coderay/scanners/json.rb @@ -70,7 +70,7 @@ module Scanners encoder.text_token match, :content elsif match = scan(/ \\ | $ /x) encoder.end_group state - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb index 8acfff5..6c68834 100644 --- a/lib/coderay/scanners/php.rb +++ b/lib/coderay/scanners/php.rb @@ -1,4 +1,4 @@ -# encoding: ASCII-8BIT +# encoding: utf-8 module CodeRay module Scanners @@ -11,7 +11,6 @@ module Scanners register_for :php file_extension 'php' - encoding 'BINARY' KINDS_NOT_LOC = HTML::KINDS_NOT_LOC @@ -211,7 +210,7 @@ module Scanners HTML_INDICATOR = /<!DOCTYPE html|<(?:html|body|div|p)[> ]/i - IDENTIFIER = /[a-z_\x7f-\xFF][a-z0-9_\x7f-\xFF]*/i + IDENTIFIER = 'ä'[/[[:alpha:]]/] == 'ä' ? Regexp.new('[[:alpha:]_[^\0-\177]][[:alnum:]_[^\0-\177]]*') : Regexp.new('[a-z_\x7f-\xFF][a-z0-9_\x7f-\xFF]*', true) VARIABLE = /\$#{IDENTIFIER}/ OPERATOR = / diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb index cbdbbdb..a9492ab 100644 --- a/lib/coderay/scanners/python.rb +++ b/lib/coderay/scanners/python.rb @@ -133,7 +133,7 @@ module Scanners elsif match = scan(/ \\ | $ /x) encoder.end_group string_type string_type = nil - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial else raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb index 2be98a6..c282f31 100644 --- a/lib/coderay/scanners/ruby.rb +++ b/lib/coderay/scanners/ruby.rb @@ -94,18 +94,27 @@ module Scanners if !method_call_expected && match = scan(unicode ? /#{patterns::METHOD_NAME}/uo : /#{patterns::METHOD_NAME}/o) - value_expected = false + kind = patterns::IDENT_KIND[match] - if kind == :ident - if match[/\A[A-Z]/] && !(match[/[!?]$/] || match?(/\(/)) - kind = :constant + if value_expected != :colon_expected && scan(/:(?!:)/) + value_expected = true + encoder.text_token match, :key + encoder.text_token ':', :operator + else + value_expected = false + if kind == :ident + if match[/\A[A-Z]/] && !(match[/[!?]$/] || match?(/\(/)) + kind = :constant + end + elsif kind == :keyword + state = patterns::KEYWORD_NEW_STATE[match] + if patterns::KEYWORDS_EXPECTING_VALUE[match] + value_expected = match == 'when' ? :colon_expected : true + end end - elsif kind == :keyword - state = patterns::KEYWORD_NEW_STATE[match] - value_expected = true if patterns::KEYWORDS_EXPECTING_VALUE[match] + value_expected = true if !value_expected && check(/#{patterns::VALUE_FOLLOWS}/o) + encoder.text_token match, kind end - value_expected = true if !value_expected && check(/#{patterns::VALUE_FOLLOWS}/o) - encoder.text_token match, kind elsif method_call_expected && match = scan(unicode ? /#{patterns::METHOD_AFTER_DOT}/uo : @@ -119,9 +128,9 @@ module Scanners value_expected = check(/#{patterns::VALUE_FOLLOWS}/o) # OPERATORS # - elsif !method_call_expected && match = scan(/ (\.(?!\.)|::) | (?: \.\.\.? | ==?=? | [,\(\[\{] )() | [\)\]\}] /x) + elsif !method_call_expected && match = scan(/ (\.(?!\.)|::) | ( \.\.\.? | ==?=? | [,\(\[\{] ) | [\)\]\}] /x) method_call_expected = self[1] - value_expected = !method_call_expected && self[2] + value_expected = !method_call_expected && !!self[2] if inline_block_stack case match when '{' @@ -213,7 +222,7 @@ module Scanners encoder.text_token match, :integer elsif match = scan(/ %=? | <(?:<|=>?)? | \? /x) - value_expected = true + value_expected = match == '?' ? :colon_expected : true encoder.text_token match, :operator elsif match = scan(/`/) diff --git a/lib/coderay/scanners/ruby/patterns.rb b/lib/coderay/scanners/ruby/patterns.rb index a52198e..ed071d2 100644 --- a/lib/coderay/scanners/ruby/patterns.rb +++ b/lib/coderay/scanners/ruby/patterns.rb @@ -1,9 +1,9 @@ # encoding: utf-8 module CodeRay module Scanners - + module Ruby::Patterns # :nodoc: all - + KEYWORDS = %w[ and def end in or unless begin defined? ensure module redo super until @@ -12,7 +12,7 @@ module Scanners while alias class elsif if not return undef yield ] - + # See http://murfy.de/ruby-constants. PREDEFINED_CONSTANTS = %w[ nil true false self @@ -24,19 +24,19 @@ module Scanners RUBY_PLATFORM RUBY_RELEASE_DATE RUBY_REVISION RUBY_VERSION __FILE__ __LINE__ __ENCODING__ ] - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_CONSTANTS, :predefined_constant) - + KEYWORD_NEW_STATE = WordList.new(:initial). add(%w[ def ], :def_expected). add(%w[ undef ], :undef_expected). add(%w[ alias ], :alias_expected). add(%w[ class module ], :module_expected) - - IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/ - + + IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? Regexp.new('[[:alpha:]_[^\0-\177]][[:alnum:]_[^\0-\177]]*') : /[^\W\d]\w*/ + METHOD_NAME = / #{IDENT} [?!]? /ox METHOD_NAME_OPERATOR = / \*\*? # multiplication and power @@ -57,25 +57,25 @@ module Scanners GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox - + QUOTE_TO_TYPE = { '`' => :shell, '/'=> :regexp, } QUOTE_TO_TYPE.default = :string - + REGEXP_MODIFIERS = /[mousenix]*/ - + DECIMAL = /\d+(?:_\d+)*/ OCTAL = /0_?[0-7]+(?:_[0-7]+)*/ HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/ BINARY = /0b[01]+(?:_[01]+)*/ - + EXPONENT = / [eE] [+-]? #{DECIMAL} /ox FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox - + SYMBOL = / : (?: @@ -85,7 +85,7 @@ module Scanners ) /ox METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox - + SIMPLE_ESCAPE = / [abefnrstv] | [0-7]{1,3} @@ -110,7 +110,7 @@ module Scanners | \\ #{ESCAPE} ) /mox - + # NOTE: This is not completely correct, but # nobody needs heredoc delimiters ending with \n. HEREDOC_OPEN = / @@ -122,13 +122,13 @@ module Scanners ( [^\n]*? ) \3 # $4 = delim ) /mx - + RUBYDOC = / =begin (?!\S) .*? (?: \Z | ^=end (?!\S) [^\n]* ) /mx - + DATA = / __END__$ .*? @@ -136,7 +136,7 @@ module Scanners /mx RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo - + # Checks for a valid value to follow. This enables # value_expected in method calls without parentheses. VALUE_FOLLOWS = / diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb new file mode 100644 index 0000000..167051d --- /dev/null +++ b/lib/coderay/scanners/sass.rb @@ -0,0 +1,227 @@ +module CodeRay +module Scanners + + # A scanner for Sass. + class Sass < CSS + + register_for :sass + file_extension 'sass' + + STRING_CONTENT_PATTERN = { + "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, + '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/, + } + + protected + + def setup + @state = :initial + end + + def scan_tokens encoder, options + states = Array(options[:state] || @state) + string_delimiter = nil + + until eos? + + if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/)) + encoder.text_token self[1], :space if self[1] + encoder.begin_group :comment + encoder.text_token self[2], :delimiter + encoder.text_token self[3], :content if self[3] + if match = scan(/(?:\n+#{self[1]} .*)+/) + encoder.text_token match, :content + end + encoder.end_group :comment + elsif match = scan(/\n|[^\n\S]+\n?/) + encoder.text_token match, :space + if match.index(/\n/) + value_expected = false + states.pop if states.last == :include + end + + elsif states.last == :sass_inline && (match = scan(/\}/)) + encoder.text_token match, :inline_delimiter + encoder.end_group :inline + states.pop + + elsif case states.last + when :initial, :media, :sass_inline + if match = scan(/(?>#{RE::Ident})(?!\()/ox) + encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag) + next + elsif !value_expected && (match = scan(/\*/)) + encoder.text_token match, :tag + next + elsif match = scan(RE::Class) + encoder.text_token match, :class + next + elsif match = scan(RE::Id) + encoder.text_token match, :id + next + elsif match = scan(RE::PseudoClass) + encoder.text_token match, :pseudo_class + next + elsif match = scan(RE::AttributeSelector) + # TODO: Improve highlighting inside of attribute selectors. + encoder.text_token match[0,1], :operator + encoder.text_token match[1..-2], :attribute_name if match.size > 2 + encoder.text_token match[-1,1], :operator if match[-1] == ?] + next + elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o) + encoder.text_token match, :function + next + elsif match = scan(/@import\b/) + encoder.text_token match, :directive + states << :include + next + elsif match = scan(/@media\b/) + encoder.text_token match, :directive + # states.push :media_before_name + next + end + + when :block + if match = scan(/(?>#{RE::Ident})(?!\()/ox) + if value_expected + encoder.text_token match, :value + else + encoder.text_token match, :key + end + next + end + + when :string + if match = scan(STRING_CONTENT_PATTERN[string_delimiter]) + encoder.text_token match, :content + elsif match = scan(/['"]/) + encoder.text_token match, :delimiter + encoder.end_group :string + string_delimiter = nil + states.pop + elsif match = scan(/#\{/) + encoder.begin_group :inline + encoder.text_token match, :inline_delimiter + states.push :sass_inline + elsif match = scan(/ \\ | $ /x) + encoder.end_group :string + encoder.text_token match, :error unless match.empty? + states.pop + else + raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder + end + + when :include + if match = scan(/[^\s'",]+/) + encoder.text_token match, :include + next + end + + else + #:nocov: + raise_inspect 'Unknown state', encoder + #:nocov: + + end + + elsif match = scan(/\$#{RE::Ident}/o) + encoder.text_token match, :variable + next + + elsif match = scan(/&/) + encoder.text_token match, :local_variable + + elsif match = scan(/\+#{RE::Ident}/o) + encoder.text_token match, :include + value_expected = true + + elsif match = scan(/\/\*(?:.*?\*\/|.*)|\/\/.*/) + encoder.text_token match, :comment + + elsif match = scan(/#\{/) + encoder.begin_group :inline + encoder.text_token match, :inline_delimiter + states.push :sass_inline + + elsif match = scan(/\{/) + value_expected = false + encoder.text_token match, :operator + states.push :block + + elsif match = scan(/\}/) + value_expected = false + encoder.text_token match, :operator + if states.last == :block || states.last == :media + states.pop + end + + elsif match = scan(/['"]/) + encoder.begin_group :string + string_delimiter = match + encoder.text_token match, :delimiter + if states.include? :sass_inline + content = scan_until(/(?=#{string_delimiter}|\}|\z)/) + encoder.text_token content, :content unless content.empty? + encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/) + encoder.end_group :string + else + states.push :string + end + + elsif match = scan(/#{RE::Function}/o) + encoder.begin_group :function + start = match[/^[-\w]+\(/] + encoder.text_token start, :delimiter + if match[-1] == ?) + encoder.text_token match[start.size..-2], :content + encoder.text_token ')', :delimiter + else + encoder.text_token match[start.size..-1], :content + end + encoder.end_group :function + + elsif match = scan(/[a-z][-a-z_]*(?=\()/o) + encoder.text_token match, :predefined + + elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox) + encoder.text_token match, :float + + elsif match = scan(/#{RE::HexColor}/o) + encoder.text_token match, :color + + elsif match = scan(/! *(?:important|optional)/) + encoder.text_token match, :important + + elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/) + encoder.text_token match, :color + + elsif match = scan(/@else if\b|#{RE::AtKeyword}/) + encoder.text_token match, :directive + value_expected = true + + elsif match = scan(/ == | != | [-+*\/>~:;,.=()] /x) + if match == ':' + value_expected = true + elsif match == ';' + value_expected = false + end + encoder.text_token match, :operator + + else + encoder.text_token getch, :error + + end + + end + + if options[:keep_state] + @state = states + end + + encoder + end + + end + +end +end diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb index bcbffd5..b757278 100644 --- a/lib/coderay/scanners/sql.rb +++ b/lib/coderay/scanners/sql.rb @@ -148,7 +148,7 @@ module CodeRay module Scanners encoder.text_token string_content, :content string_content = '' end - encoder.text_token match, :error + encoder.text_token match, :error unless match.empty? state = :initial else raise "else case \" reached; %p not handled." % peek(1), encoder diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb index 259e458..6829686 100644 --- a/lib/coderay/styles/alpha.rb +++ b/lib/coderay/styles/alpha.rb @@ -39,6 +39,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } color: gray !important; text-decoration: none !important; } +.CodeRay .line-numbers pre { + word-break: normal; +} .CodeRay .line-numbers a:target { color: blue !important; } .CodeRay .line-numbers .highlighted { color: red !important; } .CodeRay .line-numbers .highlighted a { color: red !important; } @@ -78,14 +81,17 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; } .exception { color:#C00; font-weight:bold } .float { color:#60E } .function { color:#06B; font-weight:bold } +.function .delimiter { color:#024; font-weight:bold } .global-variable { color:#d70 } .hex { color:#02b } +.id { color:#33D; font-weight:bold } .imaginary { color:#f00 } .include { color:#B44; font-weight:bold } .inline { background-color: hsla(0,0%,0%,0.07); color: black } .inline-delimiter { font-weight: bold; color: #666 } .instance-variable { color:#33B } .integer { color:#00D } +.important { color:#D00 } .key .char { color: #60f } .key .delimiter { color: #404 } .key { color: #606 } diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb index a36d877..f7f19bb 100755 --- a/lib/coderay/token_kinds.rb +++ b/lib/coderay/token_kinds.rb @@ -39,6 +39,7 @@ module CodeRay :function => 'function', :global_variable => 'global-variable', :hex => 'hex', + :id => 'id', :imaginary => 'imaginary', :important => 'important', :include => 'include', @@ -88,6 +89,4 @@ module CodeRay TokenKinds[:method] = TokenKinds[:function] TokenKinds[:escape] = TokenKinds[:delimiter] TokenKinds[:docstring] = TokenKinds[:comment] - - TokenKinds.freeze end diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb index c747017..6957d69 100644 --- a/lib/coderay/tokens.rb +++ b/lib/coderay/tokens.rb @@ -93,6 +93,7 @@ module CodeRay # This method is used by @Scanner#tokenize@ when called with an Array # of source strings. The Diff encoder uses it for inline highlighting. def split_into_parts *sizes + return Array.new(sizes.size) { Tokens.new } if size == 2 && first == '' parts = [] opened = [] content = nil diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 87d1cff..4b4f085 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.0.8' + VERSION = '1.1.0' end diff --git a/rake_tasks/documentation.rake b/rake_tasks/documentation.rake index 0b7f810..d555022 100644 --- a/rake_tasks/documentation.rake +++ b/rake_tasks/documentation.rake @@ -14,12 +14,13 @@ Rake::RDocTask.new :doc do |rd| rd.main = 'lib/README'
rd.title = 'CodeRay Documentation'
- rd.options << '--line-numbers' << '--inline-source' << '--tab-width' << '2'
- rd.options << '--fmt' << ENV.fetch('format', 'html_coderay')
- require 'pathname'
- template = File.join ROOT, 'rake_helpers', 'coderay_rdoc_template.rb'
- rd.template = Pathname.new(template).expand_path.to_s
+ rd.options << '--line-numbers' << '--tab-width' << '2'
+ # rd.options << '--fmt' << ENV.fetch('format', 'html_coderay')
+ # require 'pathname'
+ # template = File.join ROOT, 'rake_helpers', 'coderay_rdoc_template.rb'
+ # rd.template = Pathname.new(template).expand_path.to_s
+ rd.main = 'README_INDEX.rdoc'
rd.rdoc_files.add 'README_INDEX.rdoc'
rd.rdoc_files.add Dir['lib']
rd.rdoc_dir = 'doc'
diff --git a/rake_tasks/generator.rake b/rake_tasks/generator.rake index 9f5c192..284adcb 100644 --- a/rake_tasks/generator.rake +++ b/rake_tasks/generator.rake @@ -1,11 +1,11 @@ namespace :generate do - desc 'generates a new scanner NAME=lang [ALT=alternative,plugin,ids] [EXT=file,extensions] [BASE=base lang] ' + desc 'generates a new scanner NAME=lang [ALT=alternative,plugin,ids] [EXT=file,extensions] [BASE=base lang]' task :scanner do raise 'I need a scanner name; use NAME=lang' unless scanner_class_name = ENV['NAME'] raise "Invalid lang: #{scanner_class_name}; use NAME=lang." unless /\A\w+\z/ === scanner_class_name - require 'active_support' + require 'active_support/all' lang = scanner_class_name.underscore - class_name = scanner_class_name.classify + class_name = scanner_class_name.camelize def scanner_file_for_lang lang File.join(LIB_ROOT, 'coderay', 'scanners', lang + '.rb') @@ -25,6 +25,7 @@ namespace :generate do File.open(scanner_file, 'w') do |file| file.write base_scanner. sub(/class \w+ < Scanner/, "class #{class_name} < Scanner"). + sub('# Scanner for JSON (JavaScript Object Notation).', "# A scanner for #{scanner_class_name}."). sub(/register_for :\w+/, "register_for :#{lang}"). sub(/file_extension '\S+'/, "file_extension '#{ENV.fetch('EXT', lang).split(',').first}'") end @@ -37,9 +38,9 @@ namespace :generate do test_suite_file = File.join(test_dir, 'suite.rb') unless File.exist? test_suite_file puts "Creating test suite file #{test_suite_file}..." - base_suite = File.read File.join(test_dir, '..', 'json', 'suite.rb') + base_suite = File.read File.join(test_dir, '..', 'ruby', 'suite.rb') File.open(test_suite_file, 'w') do |file| - file.write base_suite.sub(/class JSON/, "class #{class_name}") + file.write base_suite.sub(/class Ruby/, "class #{class_name}") end end @@ -51,7 +52,7 @@ namespace :generate do end end - if alternative_ids = ENV['ALT'] + if alternative_ids = ENV['ALT'] && alternative_ids != lang map_file = File.join(LIB_ROOT, 'coderay', 'scanners', '_map.rb') puts "Not automated. Remember to add your alternative plugin ids to #{map_file}:" for id in alternative_ids.split(',') @@ -59,17 +60,13 @@ namespace :generate do end end - print 'Add to SVN? [Y|n] ' + print 'Add to git? [Y|n] ' answer = $stdin.gets.chomp.downcase if answer.empty? || answer == 'y' - sh "svn add #{scanner_file}" - sh "svn add #{test_dir}" - svn_ignore = <<-SVN_IGNORE -*.actual.* -*.expected.html -*.debug.diff* - SVN_IGNORE - sh "svn pset svn:ignore '#{svn_ignore}' #{test_dir}" + sh "git add #{scanner_file}" + cd File.join('test', 'scanners') do + sh "git add #{lang}" + end end end end diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index f070ccc..a60699d 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -1,5 +1,4 @@ namespace :test do - desc 'run all sample tests' task :samples do ruby './sample/suite.rb' @@ -17,22 +16,33 @@ namespace :test do end scanner_suite = 'test/scanners/suite.rb' - task scanner_suite do - unless File.exist? scanner_suite - puts 'Scanner tests not found; downloading from Subversion...' - sh 'svn co http://svn.rubychan.de/coderay-scanner-tests/trunk/ test/scanners/' - puts 'Finished.' - end - end - desc 'run all scanner tests' task :scanners => :update_scanner_suite do ruby scanner_suite end - desc 'update scanner test suite from SVN' - task :update_scanner_suite => scanner_suite do - sh "svn up #{File.dirname(scanner_suite)}" + desc 'update scanner test suite from GitHub' + task :update_scanner_suite do + if File.exist? scanner_suite + Dir.chdir File.dirname(scanner_suite) do + if File.directory? '.git' + puts 'Updating scanner test suite...' + sh 'git pull' + elsif File.directory? '.svn' + raise <<-ERROR +Found the deprecated Subversion scanner test suite in ./#{File.dirname(scanner_suite)}. +Please rename or remove it and run again to use the GitHub repository: + + mv test/scanners test/scanners-old + ERROR + else + raise 'No scanner test suite found.' + end + end + else + puts 'Downloading scanner test suite...' + sh 'git clone https://github.com/rubychan/coderay-scanner-tests.git test/scanners/' + end end namespace :scanner do @@ -72,7 +82,6 @@ namespace :test do puts "Skipping." end end - end task :test => %w(test:functional test:units test:exe) diff --git a/test/executable/suite.rb b/test/executable/suite.rb index f3495d6..d386f4b 100644 --- a/test/executable/suite.rb +++ b/test/executable/suite.rb @@ -14,12 +14,13 @@ class TestCodeRayExecutable < Test::Unit::TestCase ROOT_DIR = Pathname.new(File.dirname(__FILE__)) + '..' + '..' EXECUTABLE = ROOT_DIR + 'bin' + 'coderay' + RUBY_COMMAND = RUBY_VERSION < '2.0.0' ? 'ruby -w' : 'ruby' # Ruby 2 currently throws warnings for bundler EXE_COMMAND = if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success? # use Nailgun - 'ruby --ng -wI%s %s' + "#{RUBY_COMMAND}--ng -I%s %s" else - 'ruby -wI%s %s' + "#{RUBY_COMMAND} -I%s %s" end % [ROOT_DIR + 'lib', EXECUTABLE] def coderay args, options = {} diff --git a/test/functional/examples.rb b/test/functional/examples.rb index ff64af3..15f9ca3 100755 --- a/test/functional/examples.rb +++ b/test/functional/examples.rb @@ -22,7 +22,7 @@ end CODE assert_equal <<-DIV, div <table class="CodeRay"><tr> - <td class="line-numbers" title="double click to toggle" ondblclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><a href="#n1" name="n1">1</a> + <td class="line-numbers"><pre><a href="#n1" name="n1">1</a> <a href="#n2" name="n2">2</a> <a href="#n3" name="n3">3</a> </pre></td> @@ -38,7 +38,7 @@ end <body> <table class="CodeRay"><tr> - <td class="line-numbers" title="double click to toggle" ondblclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre> + <td class="line-numbers"><pre><a href="#n1" name="n1">1</a> </pre></td> <td class="code"><pre>puts <span class="string"><span class="delimiter">"</span><span class="content">Hello, world!</span><span class="delimiter">"</span></span></pre></td> </tr></table> diff --git a/test/functional/for_redcloth.rb b/test/functional/for_redcloth.rb index e980667..9fd244e 100644 --- a/test/functional/for_redcloth.rb +++ b/test/functional/for_redcloth.rb @@ -1,5 +1,4 @@ require 'test/unit' -require File.expand_path('../../lib/assert_warning', __FILE__) $:.unshift File.expand_path('../../../lib', __FILE__) require 'coderay' @@ -66,19 +65,14 @@ class BasicTest < Test::Unit::TestCase # See http://jgarber.lighthouseapp.com/projects/13054/tickets/124-code-markup-does-not-allow-brackets. def test_for_redcloth_false_positive require 'coderay/for_redcloth' - assert_warning 'CodeRay::Scanners could not load plugin :project; falling back to :text' do - assert_equal '<p><code>[project]_dff.skjd</code></p>', - RedCloth.new('@[project]_dff.skjd@').to_html - end + assert_equal '<p><code>[project]_dff.skjd</code></p>', + RedCloth.new('@[project]_dff.skjd@').to_html # false positive, but expected behavior / known issue assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">_dff.skjd</span></p>", RedCloth.new('@[ruby]_dff.skjd@').to_html - assert_warning 'CodeRay::Scanners could not load plugin :project; falling back to :text' do - assert_equal <<-BLOCKCODE.chomp, + assert_equal <<-BLOCKCODE.chomp, RedCloth.new('bc. [project]_dff.skjd').to_html <pre><code>[project]_dff.skjd</code></pre> - BLOCKCODE - RedCloth.new('bc. [project]_dff.skjd').to_html - end + BLOCKCODE end end if defined? RedCloth
\ No newline at end of file diff --git a/test/unit/plugin.rb b/test/unit/plugin.rb index a1d06e1..41eec51 100755 --- a/test/unit/plugin.rb +++ b/test/unit/plugin.rb @@ -1,6 +1,5 @@ require 'test/unit' require 'pathname' -require File.expand_path('../../lib/assert_warning', __FILE__) $:.unshift File.expand_path('../../../lib', __FILE__) require 'coderay' @@ -39,9 +38,7 @@ class PluginScannerTest < Test::Unit::TestCase def test_default assert_nothing_raised do - assert_warning 'PluginScannerTest::PluginsWithDefault could not load plugin :gargamel; falling back to :default_plugin' do - assert_operator PluginsWithDefault[:gargamel], :<, PluginsWithDefault::Plugin - end + assert_operator PluginsWithDefault[:gargamel], :<, PluginsWithDefault::Plugin end assert_equal PluginsWithDefault::Default, PluginsWithDefault.default end |