new method Scanner#scan_rest (upported from 0.9.8), and Scanner::ScanError is now a StandardError

6 files changed, 19 insertions, 18 deletions

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 8f8ca77..00dce7d 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -52,7 +52,7 @@ module CodeRay
       plugin_host Scanners
       
       # Raised if a Scanner fails while scanning
-      ScanError = Class.new Exception
+      ScanError = Class.new StandardError
       
       # The default options for all scanner classes.
       #
@@ -282,8 +282,8 @@ surrounding code:
         EOE
           File.basename(caller[0]),
           msg,
-          tokens.size,
-          tokens.last(10).map { |t| t.inspect }.join("\n"),
+          tokens.respond_to?(:size) ? tokens.size : 0,
+          tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '',
           line, column, pos,
           matched, state, bol?, eos?,
           string[pos - ambit, ambit],
@@ -291,6 +291,14 @@ surrounding code:
         ]
       end
       
+      # Shorthand for scan_until(/\z/).
+      # This method also avoids a JRuby 1.9 mode bug.
+      def scan_rest
+        rest = self.rest
+        terminate
+        rest
+      end
+      
     end
     
   end
diff --git a/lib/coderay/scanners/nitro_xhtml.rb b/lib/coderay/scanners/nitro_xhtml.rb
index cf8c5aa..b67b60c 100644
--- a/lib/coderay/scanners/nitro_xhtml.rb
+++ b/lib/coderay/scanners/nitro_xhtml.rb
@@ -89,7 +89,7 @@ module Scanners
       
       until eos?
         
-        if (match = scan_until(/(?=#{START_OF_RUBY})/o) || match = scan_until(/\z/)) and not match.empty?
+        if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_rest) and not match.empty?
           @html_scanner.tokenize match
           
         elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index ed1095e..9968994 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -261,7 +261,7 @@ module Scanners
             label_expected = true
             states << :php
           else
-            match = scan_until(/(?=#{RE::PHP_START})/o) || scan_until(/\z/)
+            match = scan_until(/(?=#{RE::PHP_START})/o) || scan_rest
             @html_scanner.tokenize match unless match.empty?
           end
         
diff --git a/lib/coderay/scanners/rhtml.rb b/lib/coderay/scanners/rhtml.rb
index 2403edf..9bfab5c 100644
--- a/lib/coderay/scanners/rhtml.rb
+++ b/lib/coderay/scanners/rhtml.rb
@@ -44,7 +44,7 @@ module Scanners
       
       until eos?
         
-        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
+        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_rest) and not match.empty?
           @html_scanner.tokenize match, :tokens => encoder
           
         elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index 79e1e91..549ae87 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -8,13 +8,6 @@ module Scanners
   #
   # It is optimized for HTML highlighting, and is not very useful for
   # parsing or pretty printing.
-  #
-  # For now, I think it's better than the scanners in VIM or Syntax, or
-  # any highlighter I was able to find, except Caleb's RubyLexer.
-  #
-  # I hope it's also better than the rdoc/irb lexer.
-  # 
-  # Alias: +irb+
   class Ruby < Scanner
     
     register_for :ruby
@@ -31,10 +24,8 @@ module Scanners
     
     def scan_tokens encoder, options
       
-      patterns = Patterns  # avoid constant lookup
-      
       state = @state
-      if state.instance_of? self.class::StringState
+      if state && state.instance_of?(self.class::StringState)
         encoder.begin_group state.type
       end
       
@@ -50,6 +41,8 @@ module Scanners
       # def_object_stack = nil
       # def_object_paren_depth = 0
       
+      patterns = Patterns  # avoid constant lookup
+      
       unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
       
       until eos?
@@ -339,7 +332,7 @@ module Scanners
           
         else  # StringState
           
-          match = scan_until(state.pattern) || scan_until(/\z/)
+          match = scan_until(state.pattern) || scan_rest
           unless match.empty?
             encoder.text_token match, :content
             break if eos?
diff --git a/lib/coderay/scanners/scheme.rb b/lib/coderay/scanners/scheme.rb
index e2c36e6..ca3816e 100644
--- a/lib/coderay/scanners/scheme.rb
+++ b/lib/coderay/scanners/scheme.rb
@@ -84,7 +84,7 @@ module CodeRay
             if match = scan(/ \s+ | \\\n /x)
               encoder.text_token match, :space
             elsif match = scan(/['\(\[\)\]]|#\(/)
-              encoder.text_token match, :operator  # FIXME: was :operator_fat
+              encoder.text_token match, :operator
             elsif match = scan(/;.*/)
               encoder.text_token match, :comment
             elsif match = scan(/#\\(?:newline|space|.?)/)