New: *Simple Diff Scanner* (closes #22).

* Highlights unified diffs, especially like the ones svn diff outputs.
* Changes to make highlighting of whole lines were necessary.
* I added two example files.

More changes:
* Added token classes :head, :delete, :insert, and :change along with styles.
* Added two new special token types: :begin_line and :end_line. They mark token groups that explicitly span whole lines and should be highlighted as such.
* The HTML encoder converts these new tokens to DIVs. May need more work.
* The Debug Encoder uses square brackets for line tokens.
* Some cleanups.

1 files changed, 2241 insertions, 0 deletions

diff --git a/test/scanners/diff/coderay200vs250.in.diff b/test/scanners/diff/coderay200vs250.in.diff
new file mode 100644
index 0000000..f7b99ca
--- /dev/null
+++ b/test/scanners/diff/coderay200vs250.in.diff
@@ -0,0 +1,2241 @@
+Index: lib/coderay/token_classes.rb
+===================================================================
+--- lib/coderay/token_classes.rb	(revision 0)
++++ lib/coderay/token_classes.rb	(revision 250)
+@@ -0,0 +1,71 @@
++module CodeRay
++  class Tokens
++    ClassOfKind = Hash.new do |h, k|
++      h[k] = k.to_s
++    end
++    ClassOfKind.update with = {
++      :attribute_name => 'an',
++      :attribute_name_fat => 'af',
++      :attribute_value => 'av',
++      :attribute_value_fat => 'aw',
++      :bin => 'bi',
++      :char => 'ch',
++      :class => 'cl',
++      :class_variable => 'cv',
++      :color => 'cr',
++      :comment => 'c',
++      :constant => 'co',
++      :content => 'k',
++      :definition => 'df',
++      :delimiter => 'dl',
++      :directive => 'di',
++      :doc => 'do',
++      :doc_string => 'ds',
++      :entity => 'en',
++      :error => 'er',
++      :escape => 'e',
++      :exception => 'ex',
++      :float => 'fl',
++      :function => 'fu',
++      :global_variable => 'gv',
++      :hex => 'hx',
++      :include => 'ic',
++      :inline => 'il',
++      :inline_delimiter => 'idl',
++      :instance_variable => 'iv',
++      :integer => 'i',
++      :interpreted => 'in',
++      :label => 'la',
++      :local_variable => 'lv',
++      :modifier => 'mod',
++      :oct => 'oc',
++      :operator_fat => 'of',
++      :pre_constant => 'pc',
++      :pre_type => 'pt',
++      :predefined => 'pd',
++      :preprocessor => 'pp',
++      :regexp => 'rx',
++      :reserved => 'r',
++      :shell => 'sh',
++      :string => 's',
++      :symbol => 'sy',
++      :tag => 'ta',
++      :tag_fat => 'tf',
++      :tag_special => 'ts',
++      :type => 'ty',
++      :variable => 'v',
++      :xml_text => 'xt',
++
++      :ident => :NO_HIGHLIGHT, # 'id'
++      #:operator => 'op',
++      :operator => :NO_HIGHLIGHT,  # 'op'
++      :space => :NO_HIGHLIGHT,  # 'sp'
++      :plain => :NO_HIGHLIGHT,
++    }
++    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
++    ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
++    ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
++    ClassOfKind[:escape] = ClassOfKind[:delimiter]
++    #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
++  end
++end
+\ No newline at end of file
+
+Property changes on: lib/coderay/token_classes.rb
+___________________________________________________________________
+Added: svn:executable
+   + *
+
+Index: lib/coderay/encoder.rb
+===================================================================
+--- lib/coderay/encoder.rb	(revision 200)
++++ lib/coderay/encoder.rb	(revision 250)
+@@ -1,3 +1,5 @@
++require "stringio"
++
+ module CodeRay
+ 
+   # This module holds the Encoder class and its subclasses.
+@@ -40,7 +42,7 @@
+         # downcase class name instead.
+         def const_missing sym
+           if sym == :FILE_EXTENSION
+-            sym.to_s.downcase
++            plugin_id
+           else
+             super
+           end
+@@ -130,13 +132,15 @@
+       # By default, it calls text_token or block_token, depending on
+       # whether +text+ is a String.
+       def token text, kind
+-        if text.instance_of? ::String  # Ruby 1.9: :open.is_a? String
+-          text_token text, kind
+-        elsif text.is_a? ::Symbol
+-          block_token text, kind
+-        else
+-          raise 'Unknown token text type: %p' % text
+-        end
++        out =
++          if text.is_a? ::String  # Ruby 1.9: :open.is_a? String
++            text_token text, kind
++          elsif text.is_a? ::Symbol
++            block_token text, kind
++          else
++            raise 'Unknown token text type: %p' % text
++          end
++        @out << out if defined?(@out) && @out
+       end
+ 
+       def text_token text, kind
+@@ -164,7 +168,8 @@
+       # The already created +tokens+ object must be used; it can be a
+       # TokenStream or a Tokens object.
+       def compile tokens, options
+-        tokens.each(&self)
++        tokens.each { |text, kind| token text, kind }  # FIXME for Ruby 1.9?
++        #tokens.each(&self)
+       end
+ 
+     end
+Index: lib/coderay/encoders/xml.rb
+===================================================================
+--- lib/coderay/encoders/xml.rb	(revision 200)
++++ lib/coderay/encoders/xml.rb	(revision 250)
+@@ -22,7 +22,6 @@
+   protected
+ 
+     def setup options
+-      @out = ''
+       @doc = REXML::Document.new
+       @doc << REXML::XMLDecl.new
+       @tab_width = options[:tab_width]
+@@ -33,7 +32,7 @@
+       @doc.write @out, options[:pretty], options[:transitive], true
+       @out
+     end
+-
++    
+     def text_token text, kind
+       if kind == :space
+         token = @node
+Index: lib/coderay/encoders/html/classes.rb
+===================================================================
+--- lib/coderay/encoders/html/classes.rb	(revision 200)
++++ lib/coderay/encoders/html/classes.rb	(revision 250)
+@@ -1,77 +0,0 @@
+-module CodeRay
+-module Encoders
+-
+-  class HTML
+-
+-    ClassOfKind = Hash.new do |h, k|
+-      h[k] = k.to_s
+-    end
+-    ClassOfKind.update with = {
+-      :attribute_name => 'an',
+-      :attribute_name_fat => 'af',
+-      :attribute_value => 'av',
+-      :attribute_value_fat => 'aw',
+-      :bin => 'bi',
+-      :char => 'ch',
+-      :class => 'cl',
+-      :class_variable => 'cv',
+-      :color => 'cr',
+-      :comment => 'c',
+-      :constant => 'co',
+-      :content => 'k',
+-      :definition => 'df',
+-      :delimiter => 'dl',
+-      :directive => 'di',
+-      :doc => 'do',
+-      :doc_string => 'ds',
+-      :entity => 'en',
+-      :error => 'er',
+-      :escape => 'e',
+-      :exception => 'ex',
+-      :float => 'fl',
+-      :function => 'fu',
+-      :global_variable => 'gv',
+-      :hex => 'hx',
+-      :include => 'ic',
+-      :inline => 'il',
+-      :inline_delimiter => 'idl',
+-      :instance_variable => 'iv',
+-      :integer => 'i',
+-      :interpreted => 'in',
+-      :label => 'la',
+-      :local_variable => 'lv',
+-      :modifier => 'mod',
+-      :oct => 'oc',
+-      :operator_name => 'on',
+-      :pre_constant => 'pc',
+-      :pre_type => 'pt',
+-      :predefined => 'pd',
+-      :preprocessor => 'pp',
+-      :regexp => 'rx',
+-      :reserved => 'r',
+-      :shell => 'sh',
+-      :string => 's',
+-      :symbol => 'sy',
+-      :tag => 'ta',
+-      :tag_fat => 'tf',
+-      :tag_special => 'ts',
+-      :type => 'ty',
+-      :variable => 'v',
+-      :xml_text => 'xt',
+-
+-      :ident => :NO_HIGHLIGHT, # 'id'
+-      #:operator => 'op',
+-      :operator => :NO_HIGHLIGHT,  # 'op'
+-      :space => :NO_HIGHLIGHT,  # 'sp'
+-      :plain => :NO_HIGHLIGHT,
+-    }
+-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+-    ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
+-    ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
+-    ClassOfKind[:escape] = ClassOfKind[:delimiter]
+-    #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+-
+-  end
+-
+-end
+-end
+Index: lib/coderay/encoders/html/numerization.rb
+===================================================================
+--- lib/coderay/encoders/html/numerization.rb	(revision 200)
++++ lib/coderay/encoders/html/numerization.rb	(revision 250)
+@@ -51,12 +51,12 @@
+         case mode
+         when :inline
+           max_width = (start + line_count).to_s.size
+-          line = start
++          line_number = start
+           gsub!(/^/) do
+-            line_number = bolding.call line
+-            indent = ' ' * (max_width - line.to_s.size)
+-            res = "<span class=\"no\">#{indent}#{line_number}</span> "
+-            line += 1
++            line_number_text = bolding.call line_number
++            indent = ' ' * (max_width - line_number.to_s.size)  # TODO: Optimize (10^x)
++            res = "<span class=\"no\">#{indent}#{line_number_text}</span> "
++            line_number += 1
+             res
+           end
+ 
+Index: lib/coderay/encoders/tokens.rb
+===================================================================
+--- lib/coderay/encoders/tokens.rb	(revision 200)
++++ lib/coderay/encoders/tokens.rb	(revision 250)
+@@ -33,9 +33,9 @@
+ 
+     FILE_EXTENSION = 'tok'
+ 
+-    protected
+-    def token *args
+-      @out << CodeRay::Tokens.write_token(*args)
++  protected
++    def token text, kind
++      @out << CodeRay::Tokens.write_token(text, kind)
+     end
+ 
+   end
+Index: lib/coderay/encoders/html.rb
+===================================================================
+--- lib/coderay/encoders/html.rb	(revision 200)
++++ lib/coderay/encoders/html.rb	(revision 250)
+@@ -1,3 +1,5 @@
++require "set"
++
+ module CodeRay
+ module Encoders
+ 
+@@ -10,7 +12,8 @@
+   #
+   #  require 'coderay'
+   #  puts CodeRay.scan('Some /code/', :ruby).html  #-> a HTML page
+-  #  puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span) #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
++  #  puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
++  #  #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
+   #  puts CodeRay.scan('Some /code/', :ruby).span  #-> the same
+   #  
+   #  puts CodeRay.scan('Some code', :ruby).html(
+@@ -55,7 +58,8 @@
+   #
+   # === :hint
+   # Include some information into the output using the title attribute.
+-  # Can be :info (show token type on mouse-over), :info_long (with full path) or :debug (via inspect).
++  # Can be :info (show token type on mouse-over), :info_long (with full path)
++  # or :debug (via inspect).
+   #
+   # Default: false
+   class HTML < Encoder
+@@ -82,7 +86,7 @@
+       :hint => false,
+     }
+ 
+-    helper :classes, :output, :css
++    helper :output, :css
+ 
+     attr_reader :css
+ 
+@@ -115,11 +119,14 @@
+         end
+     }
+ 
++    TRANSPARENT_TOKEN_KINDS = [
++      :delimiter, :modifier, :content, :escape, :inline_delimiter,
++    ].to_set
++
+     # Generate a hint about the given +classes+ in a +hint+ style.
+     #
+     # +hint+ may be :info, :info_long or :debug.
+     def self.token_path_to_hint hint, classes
+-      return '' unless hint
+       title =
+         case hint
+         when :info
+@@ -129,7 +136,7 @@
+         when :debug
+           classes.inspect
+         end
+-      " title=\"#{title}\""
++      title ? " title=\"#{title}\"" : ''
+     end
+ 
+     def setup options
+@@ -143,42 +150,45 @@
+ 
+       hint = options[:hint]
+       if hint and not [:debug, :info, :info_long].include? hint
+-        raise ArgumentError, "Unknown value %p for :hint; expected :info, :debug, false or nil." % hint
++        raise ArgumentError, "Unknown value %p for :hint; \
++          expected :info, :debug, false, or nil." % hint
+       end
+ 
+       case options[:css]
+ 
+       when :class
+         @css_style = Hash.new do |h, k|
+-          if k.is_a? Array
+-            type = k.first
+-          else
+-            type = k
+-          end
+-          c = ClassOfKind[type]
++          c = CodeRay::Tokens::ClassOfKind[k.first]
+           if c == :NO_HIGHLIGHT and not hint
+-            h[k] = false
++            h[k.dup] = false
+           else
+-            title = HTML.token_path_to_hint hint, (k[1..-1] << k.first)
+-            h[k] = '<span%s class="%s">' % [title, c]
++            title = if hint
++              HTML.token_path_to_hint(hint, k[1..-1] << k.first)
++            else
++              ''
++            end
++            if c == :NO_HIGHLIGHT
++              h[k.dup] = '<span%s>' % [title]
++            else
++              h[k.dup] = '<span%s class="%s">' % [title, c]
++            end
+           end
+         end
+ 
+       when :style
+         @css_style = Hash.new do |h, k|
+-          if k.is_a? Array
++          if k.is_a? ::Array
+             styles = k.dup
+           else
+             styles = [k]
+           end
+           type = styles.first
+-          classes = styles.map { |c| ClassOfKind[c] }
++          classes = styles.map { |c| Tokens::ClassOfKind[c] }
+           if classes.first == :NO_HIGHLIGHT and not hint
+             h[k] = false
+           else
+-            styles.shift if [:delimiter, :modifier, :content, :escape].include? styles.first
++            styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
+             title = HTML.token_path_to_hint hint, styles
+-            classes.delete 'il'
+             style = @css[*classes]
+             h[k] =
+               if style
+@@ -198,7 +208,9 @@
+     def finish options
+       not_needed = @opened.shift
+       @out << '</span>' * @opened.size
+-      warn '%d tokens still open: %p' % [@opened.size, @opened] unless @opened.empty?
++      unless @opened.empty?
++        warn '%d tokens still open: %p' % [@opened.size, @opened]
++      end
+ 
+       @out.extend Output
+       @out.css = @css
+@@ -229,8 +241,9 @@
+           if @opened.empty?
+             # nothing to close
+           else
+-            if @opened.size == 1 or @opened.last != type
+-              raise 'Malformed token stream: Trying to close a token (%p) that is not open. Open are: %p.' % [type, @opened[1..-1]] if $DEBUG
++            if $DEBUG and (@opened.size == 1 or @opened.last != type)
++              raise 'Malformed token stream: Trying to close a token (%p) \
++                that is not open. Open are: %p.' % [type, @opened[1..-1]]
+             end
+             @out << '</span>'
+             @opened.pop
+Index: lib/coderay/encoders/text.rb
+===================================================================
+--- lib/coderay/encoders/text.rb	(revision 200)
++++ lib/coderay/encoders/text.rb	(revision 250)
+@@ -14,13 +14,12 @@
+ 
+   protected
+     def setup options
+-      super
++      @out = ''
+       @sep = options[:separator]
+     end
+ 
+     def token text, kind
+-      return unless text.respond_to? :to_str
+-      @out << text + @sep
++      @out << text + @sep if text.is_a? ::String
+     end
+ 
+     def finish options
+Index: lib/coderay/encoders/debug.rb
+===================================================================
+--- lib/coderay/encoders/debug.rb	(revision 200)
++++ lib/coderay/encoders/debug.rb	(revision 250)
+@@ -19,19 +19,14 @@
+ 
+   protected
+     def text_token text, kind
+-      @out <<
+-        if kind == :space
+-          text
+-        else
+-          text = text.gsub(/[)\\]/, '\\\\\0')
+-          "#{kind}(#{text})"
+-        end
++      if kind == :space
++        text
++      else
++        text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
++        "#{kind}(#{text})"
++      end
+     end
+ 
+-    def block_token action, kind
+-      @out << super
+-    end
+-
+     def open_token kind
+       "#{kind}<"
+     end
+Index: lib/coderay/encoders/statistic.rb
+===================================================================
+--- lib/coderay/encoders/statistic.rb	(revision 200)
++++ lib/coderay/encoders/statistic.rb	(revision 250)
+@@ -28,19 +28,15 @@
+       @type_stats[kind].count += 1
+       @type_stats[kind].size += text.size
+       @type_stats['TOTAL'].size += text.size
++      @type_stats['TOTAL'].count += 1
+     end
+ 
+     # TODO Hierarchy handling
+     def block_token action, kind
+-      #@content_type = kind
++      @type_stats['TOTAL'].count += 1
+       @type_stats['open/close'].count += 1
+     end
+ 
+-    def token text, kind
+-      super
+-      @type_stats['TOTAL'].count += 1
+-    end
+-
+     STATS = <<-STATS
+ 
+ Code Statistics
+Index: lib/coderay/encoders/_map.rb
+===================================================================
+--- lib/coderay/encoders/_map.rb	(revision 200)
++++ lib/coderay/encoders/_map.rb	(revision 250)
+@@ -2,7 +2,8 @@
+ module Encoders
+ 
+   map :stats => :statistic,
+-    :plain => :text
++    :plain => :text,
++    :tex => :latex
+ 
+ end
+ end
+Index: lib/coderay/helpers/filetype.rb
+===================================================================
+--- lib/coderay/helpers/filetype.rb	(revision 200)
++++ lib/coderay/helpers/filetype.rb	(revision 250)
+@@ -1,180 +0,0 @@
+-# =FileType
+-#
+-# A simple filetype recognizer
+-#
+-# Author: murphy (mail to murphy cYcnus de)
+-#
+-# Version: 0.1 (2005.september.1)
+-#
+-# == Documentation
+-#
+-#  # determine the type of the given
+-#   lang = FileType[ARGV.first]
+-#  
+-#   # return :plaintext if the file type is unknown
+-#   lang = FileType.fetch ARGV.first, :plaintext
+-#  
+-#   # try the shebang line, too
+-#   lang = FileType.fetch ARGV.first, :plaintext, true
+-module FileType
+-
+-  UnknownFileType = Class.new Exception
+-
+-  class << self
+-
+-    # Try to determine the file type of the file.
+-    #
+-    # +filename+ is a relative or absolute path to a file.
+-    #
+-    # The file itself is only accessed when +read_shebang+ is set to true.
+-    # That means you can get filetypes from files that don't exist.
+-    def [] filename, read_shebang = false
+-      name = File.basename filename
+-      ext = File.extname name
+-      ext.sub!(/^\./, '')  # delete the leading dot
+-
+-      type =
+-        TypeFromExt[ext] ||
+-        TypeFromExt[ext.downcase] ||
+-        TypeFromName[name] ||
+-        TypeFromName[name.downcase]
+-      type ||= shebang(filename) if read_shebang
+-
+-      type
+-    end
+-
+-    def shebang filename
+-      begin
+-        File.open filename, 'r' do |f|
+-          first_line = f.gets
+-          first_line[TypeFromShebang]
+-        end
+-      rescue IOError
+-        nil
+-      end
+-    end
+-
+-    # This works like Hash#fetch.
+-    #
+-    # If the filetype cannot be found, the +default+ value
+-    # is returned.
+-    def fetch filename, default = nil, read_shebang = false
+-      if default and block_given?
+-        warn 'block supersedes default value argument'
+-      end
+-
+-      unless type = self[filename, read_shebang]
+-        return yield if block_given?
+-        return default if default
+-        raise UnknownFileType, 'Could not determine type of %p.' % filename
+-      end
+-      type
+-    end
+-
+-  end
+-
+-  TypeFromExt = {
+-    'rb' => :ruby,
+-    'rbw' => :ruby,
+-    'rake' => :ruby,
+-    'cpp' => :c,
+-    'c' => :c,
+-    'h' => :c,
+-    'xml' => :xml,
+-    'htm' => :html,
+-    'html' => :html,
+-    'xhtml' => :xhtml,
+-    'rhtml' => :rhtml,
+-    'yaml' => :yaml,
+-    'yml' => :yaml,
+-  }
+-
+-  TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
+-
+-  TypeFromName = {
+-    'Rakefile' => :ruby,
+-    'Rantfile' => :ruby,
+-  }
+-
+-end
+-
+-if $0 == __FILE__
+-  $VERBOSE = true
+-  eval DATA.read, nil, $0, __LINE__+4
+-end
+-
+-__END__
+-
+-require 'test/unit'
+-
+-class TC_FileType < Test::Unit::TestCase
+-
+-  def test_fetch
+-    assert_raise FileType::UnknownFileType do
+-      FileType.fetch ''
+-    end
+-
+-    assert_throws :not_found do
+-      FileType.fetch '.' do
+-        throw :not_found
+-      end
+-    end
+-
+-    assert_equal :default, FileType.fetch('c', :default)
+-
+-    stderr, fake_stderr = $stderr, Object.new
+-    $err = ''
+-    def fake_stderr.write x
+-      $err << x
+-    end
+-    $stderr = fake_stderr
+-    FileType.fetch('c', :default) { }
+-    assert_equal "block supersedes default value argument\n", $err
+-    $stderr = stderr
+-  end
+-
+-  def test_ruby
+-    assert_equal :ruby, FileType['test.rb']
+-    assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
+-    assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
+-    assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
+-    assert_equal :ruby, FileType['./lib/tasks\repository.rake']
+-    assert_not_equal :ruby, FileType['test_rb']
+-    assert_not_equal :ruby, FileType['Makefile']
+-    assert_not_equal :ruby, FileType['set.rb/set']
+-    assert_not_equal :ruby, FileType['~/projects/blabla/rb']
+-  end
+-
+-  def test_c
+-    assert_equal :c, FileType['test.c']
+-    assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
+-    assert_not_equal :c, FileType['test_c']
+-    assert_not_equal :c, FileType['Makefile']
+-    assert_not_equal :c, FileType['set.h/set']
+-    assert_not_equal :c, FileType['~/projects/blabla/c']
+-  end
+-
+-  def test_html
+-    assert_equal :html, FileType['test.htm']
+-    assert_equal :xhtml, FileType['test.xhtml']
+-    assert_equal :xhtml, FileType['test.html.xhtml']
+-    assert_equal :rhtml, FileType['_form.rhtml']
+-  end
+-
+-  def test_yaml
+-    assert_equal :yaml, FileType['test.yml']
+-    assert_equal :yaml, FileType['test.yaml']
+-    assert_equal :yaml, FileType['my.html.yaml']
+-    assert_not_equal :yaml, FileType['YAML']
+-  end
+-
+-  def test_shebang
+-    dir = './test'
+-    if File.directory? dir
+-      Dir.chdir dir do
+-        assert_equal :c, FileType['test.c']
+-      end
+-    end
+-  end
+-
+-end
+Index: lib/coderay/helpers/plugin.rb
+===================================================================
+--- lib/coderay/helpers/plugin.rb	(revision 200)
++++ lib/coderay/helpers/plugin.rb	(revision 250)
+@@ -1,3 +1,5 @@
++module CodeRay
++  
+ # = PluginHost
+ #
+ # $Id$
+@@ -20,7 +22,7 @@
+ #
+ #    Generators[:fancy]  #-> FancyGenerator
+ #    # or
+-#    require_plugin 'Generators/fancy'
++#    CodeRay.require_plugin 'Generators/fancy'
+ module PluginHost
+ 
+   # Raised if Encoders::[] fails because:
+@@ -310,17 +312,18 @@
+ 
+ end
+ 
+-
+ # Convenience method for plugin loading.
+ # The syntax used is:
+ #
+-#  require_plugin '<Host ID>/<Plugin ID>'
++#  CodeRay.require_plugin '<Host ID>/<Plugin ID>'
+ #
+ # Returns the loaded plugin.
+-def require_plugin path
++def self.require_plugin path
+   host_id, plugin_id = path.split '/', 2
+   host = PluginHost.host_by_id(host_id)
+   raise PluginHost::HostNotFound,
+     "No host for #{host_id.inspect} found." unless host
+   host.load plugin_id
+ end
++
++end
+\ No newline at end of file
+Index: lib/coderay/helpers/file_type.rb
+===================================================================
+--- lib/coderay/helpers/file_type.rb	(revision 0)
++++ lib/coderay/helpers/file_type.rb	(revision 250)
+@@ -0,0 +1,210 @@
++#!/usr/bin/env ruby
++module CodeRay
++
++# = FileType
++#
++# A simple filetype recognizer.
++#
++# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
++#
++# License:: LGPL / ask the author
++# Version:: 0.1 (2005-09-01)
++#
++# == Documentation
++#
++#  # determine the type of the given
++#   lang = FileType[ARGV.first]
++#  
++#   # return :plaintext if the file type is unknown
++#   lang = FileType.fetch ARGV.first, :plaintext
++#  
++#   # try the shebang line, too
++#   lang = FileType.fetch ARGV.first, :plaintext, true
++module FileType
++
++  UnknownFileType = Class.new Exception
++
++  class << self
++
++    # Try to determine the file type of the file.
++    #
++    # +filename+ is a relative or absolute path to a file.
++    #
++    # The file itself is only accessed when +read_shebang+ is set to true.
++    # That means you can get filetypes from files that don't exist.
++    def [] filename, read_shebang = false
++      name = File.basename filename
++      ext = File.extname(name).sub(/^\./, '')  # from last dot, delete the leading dot
++      ext2 = filename[/\.(.*)/, 1]  # from first dot
++
++      type =
++        TypeFromExt[ext.downcase] ||
++        (TypeFromExt[ext2.downcase] if ext2) ||
++        TypeFromName[name] ||
++        TypeFromName[name.downcase]
++      type ||= shebang(filename) if read_shebang
++
++      type
++    end
++
++    def shebang filename
++      begin
++        File.open filename, 'r' do |f|
++          if first_line = f.gets
++            if type = first_line[TypeFromShebang]
++              type.to_sym
++            end
++          end
++        end
++      rescue IOError
++        nil
++      end
++    end
++
++    # This works like Hash#fetch.
++    #
++    # If the filetype cannot be found, the +default+ value
++    # is returned.
++    def fetch filename, default = nil, read_shebang = false
++      if default and block_given?
++        warn 'block supersedes default value argument'
++      end
++
++      unless type = self[filename, read_shebang]
++        return yield if block_given?
++        return default if default
++        raise UnknownFileType, 'Could not determine type of %p.' % filename
++      end
++      type
++    end
++
++  end
++
++  TypeFromExt = {
++    'rb' => :ruby,
++    'rbw' => :ruby,
++    'rake' => :ruby,
++    'mab' => :ruby,
++    'cpp' => :c,
++    'c' => :c,
++    'h' => :c,
++    'xml' => :xml,
++    'htm' => :html,
++    'html' => :html,
++    'xhtml' => :xhtml,
++    'raydebug' => :debug,
++    'rhtml' => :rhtml,
++    'html.erb' => :rhtml,
++    'ss' => :scheme,
++    'sch' => :scheme,
++    'yaml' => :yaml,
++    'yml' => :yaml,
++  }
++
++  TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
++
++  TypeFromName = {
++    'Rakefile' => :ruby,
++    'Rantfile' => :ruby,
++  }
++
++end
++
++end
++
++if $0 == __FILE__
++  $VERBOSE = true
++  eval DATA.read, nil, $0, __LINE__+4
++end
++
++__END__
++require 'test/unit'
++
++class TC_FileType < Test::Unit::TestCase
++  
++  include CodeRay
++  
++  def test_fetch
++    assert_raise FileType::UnknownFileType do
++      FileType.fetch ''
++    end
++
++    assert_throws :not_found do
++      FileType.fetch '.' do
++        throw :not_found
++      end
++    end
++
++    assert_equal :default, FileType.fetch('c', :default)
++
++    stderr, fake_stderr = $stderr, Object.new
++    $err = ''
++    def fake_stderr.write x
++      $err << x
++    end
++    $stderr = fake_stderr
++    FileType.fetch('c', :default) { }
++    assert_equal "block supersedes default value argument\n", $err
++    $stderr = stderr
++  end
++
++  def test_ruby
++    assert_equal :ruby, FileType['test.rb']
++    assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
++    assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
++    assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
++    assert_equal :ruby, FileType['./lib/tasks\repository.rake']
++    assert_not_equal :ruby, FileType['test_rb']
++    assert_not_equal :ruby, FileType['Makefile']
++    assert_not_equal :ruby, FileType['set.rb/set']
++    assert_not_equal :ruby, FileType['~/projects/blabla/rb']
++  end
++
++  def test_c
++    assert_equal :c, FileType['test.c']
++    assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
++    assert_not_equal :c, FileType['test_c']
++    assert_not_equal :c, FileType['Makefile']
++    assert_not_equal :c, FileType['set.h/set']
++    assert_not_equal :c, FileType['~/projects/blabla/c']
++  end
++
++  def test_html
++    assert_equal :html, FileType['test.htm']
++    assert_equal :xhtml, FileType['test.xhtml']
++    assert_equal :xhtml, FileType['test.html.xhtml']
++    assert_equal :rhtml, FileType['_form.rhtml']
++    assert_equal :rhtml, FileType['_form.html.erb']
++  end
++
++  def test_yaml
++    assert_equal :yaml, FileType['test.yml']
++    assert_equal :yaml, FileType['test.yaml']
++    assert_equal :yaml, FileType['my.html.yaml']
++    assert_not_equal :yaml, FileType['YAML']
++  end
++
++  def test_no_shebang
++    dir = './test'
++    if File.directory? dir
++      Dir.chdir dir do
++        assert_equal :c, FileType['test.c']
++      end
++    end
++  end
++  
++  def test_shebang_empty_file
++    require 'tmpdir'
++    tmpfile = File.join(Dir.tmpdir, 'bla')
++    File.open(tmpfile, 'w') { }  # touch
++    assert_equal nil, FileType[tmpfile]
++  end
++  
++  def test_shebang
++    require 'tmpdir'
++    tmpfile = File.join(Dir.tmpdir, 'bla')
++    File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
++    assert_equal :ruby, FileType[tmpfile, true]
++  end
++
++end
+
+Property changes on: lib/coderay/helpers/file_type.rb
+___________________________________________________________________
+Added: svn:keywords
+   + Id Rev
+
+Index: lib/coderay/helpers/gzip_simple.rb
+===================================================================
+--- lib/coderay/helpers/gzip_simple.rb	(revision 200)
++++ lib/coderay/helpers/gzip_simple.rb	(revision 250)
+@@ -46,6 +46,7 @@
+   end
+ end
+ 
++
+ # String extensions to use the GZip module.
+ #
+ # The methods gzip and gunzip provide an even more simple
+Index: lib/coderay/helpers/word_list.rb
+===================================================================
+--- lib/coderay/helpers/word_list.rb	(revision 200)
++++ lib/coderay/helpers/word_list.rb	(revision 250)
+@@ -1,15 +1,19 @@
++module CodeRay
++
+ # = WordList
++# 
++# <b>A Hash subclass designed for mapping word lists to token types.</b>
++# 
++# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
+ #
+-# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy cYcnus de>
+-#
+ # License:: LGPL / ask the author
+-# Version:: 1.0 (2006-Feb-3)
++# Version:: 1.1 (2006-Oct-19)
+ #
+ # A WordList is a Hash with some additional features.
+ # It is intended to be used for keyword recognition.
+ #
+ # WordList is highly optimized to be used in Scanners,
+-# typically to decide whether a given ident is a keyword.
++# typically to decide whether a given ident is a special token.
+ #
+ # For case insensitive words use CaseIgnoringWordList.
+ #
+@@ -47,25 +51,30 @@
+ #      ...
+ class WordList < Hash
+ 
+-  # Create a WordList for the given +words+.
+-  #
+-  # This WordList responds to [] with +true+, if the word is
+-  # in +words+, and with +false+ otherwise.
+-  def self.for words
+-    new.add words
+-  end
+-
+   # Creates a new WordList with +default+ as default value.
+-  def initialize default = false, &block
+-    super default, &block
++  # 
++  # You can activate +caching+ to store the results for every [] request.
++  # 
++  # With caching, methods like +include?+ or +delete+ may no longer behave
++  # as you expect. Therefore, it is recommended to use the [] method only.
++  def initialize default = false, caching = false, &block
++    if block
++      raise ArgumentError, 'Can\'t combine block with caching.' if caching
++      super(&block)
++    else
++      if caching
++        super() do |h, k|
++          h[k] = h.fetch k, default
++        end
++      else
++        super default
++      end
++    end
+   end
+ 
+-  # Checks if a word is included.
+-  def include? word
+-    has_key? word
+-  end
+-
+   # Add words to the list and associate them with +kind+.
++  # 
++  # Returns +self+, so you can concat add calls.
+   def add words, kind = true
+     words.each do |word|
+       self[word] = kind
+@@ -78,24 +87,30 @@
+ 
+ # A CaseIgnoringWordList is like a WordList, only that
+ # keys are compared case-insensitively.
++# 
++# Ignoring the text case is realized by sending the +downcase+ message to
++# all keys.
++# 
++# Caching usually makes a CaseIgnoringWordList faster, but it has to be
++# activated explicitely.
+ class CaseIgnoringWordList < WordList
+ 
+-  # Creates a new WordList with +default+ as default value.
+-  #
+-  # Text case is ignored.
+-  def initialize default = false, &block
+-    block ||= proc do |h, k|
+-      h[k] = h.fetch k.downcase, default
++  # Creates a new case-insensitive WordList with +default+ as default value.
++  # 
++  # You can activate caching to store the results for every [] request.
++  def initialize default = false, caching = false
++    if caching
++      super(default, false) do |h, k|
++        h[k] = h.fetch k.downcase, default
++      end
++    else
++      def self.[] key  # :nodoc:
++        super(key.downcase)
++      end
+     end
+-    super default
+   end
+ 
+-  # Checks if a word is included.
+-  def include? word
+-    has_key? word.downcase
+-  end
+-
+-  # Add words to the list and associate them with +kind+.
++  # Add +words+ to the list and associate them with +kind+.
+   def add words, kind = true
+     words.each do |word|
+       self[word.downcase] = kind
+@@ -104,3 +119,5 @@
+   end
+ 
+ end
++
++end
+\ No newline at end of file
+Index: lib/coderay/styles/cycnus.rb
+===================================================================
+--- lib/coderay/styles/cycnus.rb	(revision 200)
++++ lib/coderay/styles/cycnus.rb	(revision 250)
+@@ -42,12 +42,14 @@
+     MAIN
+ 
+     TOKEN_COLORS = <<-'TOKENS'
++.debug { color:white ! important; background:blue ! important; }
++
+ .af { color:#00C }
+ .an { color:#007 }
+ .av { color:#700 }
+ .aw { color:#C00 }
+ .bi { color:#509; font-weight:bold }
+-.c  { color:#888 }
++.c  { color:#666; }
+ 
+ .ch { color:#04D }
+ .ch .k { color:#04D }
+@@ -83,7 +85,7 @@
+ .la { color:#970; font-weight:bold }
+ .lv { color:#963 }
+ .oc { color:#40E; font-weight:bold }
+-.on { color:#000; font-weight:bold }
++.of { color:#000; font-weight:bold }
+ .op { }
+ .pc { color:#038; font-weight:bold }
+ .pd { color:#369; font-weight:bold }
+Index: lib/coderay/styles/murphy.rb
+===================================================================
+--- lib/coderay/styles/murphy.rb	(revision 200)
++++ lib/coderay/styles/murphy.rb	(revision 250)
+@@ -47,7 +47,7 @@
+ .av { color:#700; }
+ .aw { color:#C00; }
+ .bi { color:#509; font-weight:bold; }
+-.c  { color:#666; }
++.c  { color:#555; background-color: black; }
+ 
+ .ch { color:#88F; }
+ .ch .k { color:#04D; }
+@@ -77,7 +77,7 @@
+ .la { color:#970; font-weight:bold; }
+ .lv { color:#963; }
+ .oc { color:#40E; font-weight:bold; }
+-.on { color:#000; font-weight:bold; }
++.of { color:#000; font-weight:bold; }
+ .op { }
+ .pc { color:#08f; font-weight:bold; }
+ .pd { color:#369; font-weight:bold; }
+Index: lib/coderay/tokens.rb
+===================================================================
+--- lib/coderay/tokens.rb	(revision 200)
++++ lib/coderay/tokens.rb	(revision 250)
+@@ -115,7 +115,7 @@
+     #   tokens.each_text_token { |text, kind| text.replace html_escape(text) }
+     def each_text_token
+       each do |text, kind|
+-        next unless text.respond_to? :to_str
++        next unless text.is_a? ::String
+         yield text, kind
+       end
+     end
+@@ -252,7 +252,7 @@
+     #
+     # You can configure the level of compression,
+     # but the default value 7 should be what you want
+-    # in most cases as it is a good comprimise between
++    # in most cases as it is a good compromise between
+     # speed and compression rate.
+     #
+     # See GZip module.
+@@ -267,9 +267,20 @@
+     # Should be equal to the input size before
+     # scanning.
+     def text_size
+-      map { |t, k| t }.join.size
++      size = 0
++      each_text_token do |t, k|
++        size + t.size
++      end
++      size
+     end
+ 
++    # The total size of the tokens.
++    # Should be equal to the input size before
++    # scanning.
++    def text
++      map { |t, k| t if t.is_a? ::String }.join
++    end
++
+     # Include this module to give an object an #undump
+     # method.
+     #
+@@ -342,7 +353,7 @@
+     #
+     # Returns self.
+     def << token
+-      @callback.call token
++      @callback.call(*token)
+       @size += 1
+       self
+     end
+@@ -365,4 +376,8 @@
+ 
+   end
+ 
++  
++  # Token name abbreviations
++  require 'coderay/token_classes'
++
+ end
+Index: lib/coderay/duo.rb
+===================================================================
+--- lib/coderay/duo.rb	(revision 200)
++++ lib/coderay/duo.rb	(revision 250)
+@@ -4,26 +4,84 @@
+   #
+   # $Id: scanner.rb 123 2006-03-21 14:46:34Z murphy $
+   #
+-  # TODO: Doc.
++  # A Duo is a convenient way to use CodeRay. You just create a Duo,
++  # giving it a lang (language of the input code) and a format (desired
++  # output format), and call Duo#highlight with the code.
++  # 
++  # Duo makes it easy to re-use both scanner and encoder for a repetitive
++  # task. It also provides a very easy interface syntax:
++  # 
++  #   require 'coderay'
++  #   CodeRay::Duo[:python, :div].highlight 'import this'
++  # 
++  # Until you want to do uncommon things with CodeRay, I recommend to use
++  # this method, since it takes care of everything.
+   class Duo
+ 
+-    attr_accessor :scanner, :encoder
+-
+-    def initialize lang, format, options = {}
+-      @scanner = CodeRay.scanner lang, CodeRay.get_scanner_options(options)
+-      @encoder = CodeRay.encoder format, options
++    attr_accessor :lang, :format, :options
++    
++    # Create a new Duo, holding a lang and a format to highlight code.
++    # 
++    # simple:
++    #   CodeRay::Duo[:ruby, :page].highlight 'bla 42'
++    # 
++    # streaming:
++    #   CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
++    # 
++    # with options:
++    #   CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
++    # 
++    # alternative syntax without options:
++    #   CodeRay::Duo[:ruby => :statistic].encode 'class << self; end'
++    # 
++    # alternative syntax with options:
++    #   CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc'
++    # 
++    # The options are forwarded to scanner and encoder
++    # (see CodeRay.get_scanner_options).
++    def initialize lang = nil, format = nil, options = {}
++      if format == nil and lang.is_a? Hash and lang.size == 1
++        @lang = lang.keys.first
++        @format = lang[@lang]
++      else
++        @lang = lang
++        @format = format
++      end
++      @options = options
+     end
+ 
+     class << self
++      # To allow calls like Duo[:ruby, :html].highlight.
+       alias [] new
+     end
+ 
+-    def encode code
+-      @scanner.string = code
+-      @encoder.encode_tokens(scanner.tokenize)
++    # The scanner of the duo. Only created once.
++    def scanner
++      @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
+     end
++    
++    # The encoder of the duo. Only created once.
++    def encoder
++      @encoder ||= CodeRay.encoder @format, @options
++    end
++    
++    # Tokenize and highlight the code using +scanner+ and +encoder+.
++    #
++    # If the :stream option is set, the Duo will go into streaming mode,
++    # saving memory for the cost of time.
++    def encode code, options = { :stream => false }
++      stream = options.delete :stream
++      options = @options.merge options
++      if stream
++        encoder.encode_stream(code, @lang, options)
++      else
++        scanner.code = code
++        encoder.encode_tokens(scanner.tokenize, options)
++      end
++    end
+     alias highlight encode
+ 
+   end
+ 
+ end
++
+Index: lib/coderay/scanner.rb
+===================================================================
+--- lib/coderay/scanner.rb	(revision 200)
++++ lib/coderay/scanner.rb	(revision 250)
+@@ -66,8 +66,18 @@
+         end
+ 
+         def normify code
+-          code = code.to_s.to_unix
++          code = code.to_s
++          code.force_encoding 'binary' if code.respond_to? :force_encoding
++          code.to_unix
+         end
++        
++        def file_extension extension = nil
++          if extension
++            @file_extension = extension.to_s
++          else
++            @file_extension ||= plugin_id.to_s
++          end
++        end        
+ 
+       end
+ 
+@@ -117,9 +127,6 @@
+         setup
+       end
+ 
+-      # More mnemonic accessor name for the input string.
+-      alias code string
+-
+       def reset
+         super
+         reset_instance
+@@ -131,6 +138,10 @@
+         reset_instance
+       end
+ 
++      # More mnemonic accessor name for the input string.
++      alias code string
++      alias code= string=
++
+       # Scans the code and returns all tokens in a Tokens object.
+       def tokenize new_string=nil, options = {}
+         options = @options.merge(options)
+@@ -148,6 +159,11 @@
+       def tokens
+         @cached_tokens ||= tokenize
+       end
++      
++      # Whether the scanner is in streaming mode.
++      def streaming?
++        !!@options[:stream]
++      end
+ 
+       # Traverses the tokens.
+       def each &block
+@@ -195,7 +211,7 @@
+         raise ScanError, <<-EOE % [
+ 
+ 
+-***ERROR in %s: %s
++***ERROR in %s: %s (after %d tokens)
+ 
+ tokens:
+ %s
+@@ -211,13 +227,14 @@
+ ***ERROR***
+ 
+         EOE
+-        File.basename(caller[0]),
+-        msg,
+-        tokens.last(10).map { |t| t.inspect }.join("\n"),
+-        line, pos,
+-        matched, state, bol?, eos?,
+-        string[pos-ambit,ambit],
+-        string[pos,ambit],
++          File.basename(caller[0]),
++          msg,
++          tokens.size,
++          tokens.last(10).map { |t| t.inspect }.join("\n"),
++          line, pos,
++          matched, state, bol?, eos?,
++          string[pos-ambit,ambit],
++          string[pos,ambit],
+         ]
+       end
+ 
+Index: lib/coderay/for_redcloth.rb
+===================================================================
+--- lib/coderay/for_redcloth.rb	(revision 0)
++++ lib/coderay/for_redcloth.rb	(revision 250)
+@@ -0,0 +1,72 @@
++module CodeRay  # :nodoc:
++  
++  # A little hack to enable CodeRay highlighting in RedCloth.
++  # 
++  # Usage:
++  #  require 'coderay'
++  #  require 'coderay/for_redcloth'
++  #  RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
++  # 
++  # Make sure you have RedCloth 4.0.3 activated, for example by calling
++  #  require 'rubygems'
++  # before RedCloth is loaded and before calling CodeRay.for_redcloth.
++  module ForRedCloth
++    
++    def self.install
++      gem 'RedCloth', '>= 4.0.3' rescue nil
++      require 'redcloth'
++      raise 'CodeRay.for_redcloth needs RedCloth 4.0.3 or later.' unless RedCloth::VERSION.to_s >= '4.0.3'
++      RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
++      RedCloth::Formatters::HTML.module_eval do
++        def unescape(html)
++          replacements = {
++            '&amp;' => '&',
++            '&quot;' => '"',
++            '&gt;' => '>',
++            '&lt;' => '<',
++          }
++          html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
++        end
++        undef_method :code, :bc_open, :bc_close, :escape_pre
++        def code(opts)  # :nodoc:
++          opts[:block] = true
++          if opts[:lang] && !filter_coderay
++            require 'coderay'
++            @in_bc ||= nil
++            format = @in_bc ? :div : :span
++            highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
++            highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
++            highlighted_code = unescape(highlighted_code) unless @in_bc
++            highlighted_code
++          else
++            "<code#{pba(opts)}>#{opts[:text]}</code>"
++          end
++        end
++        def bc_open(opts)  # :nodoc:
++          opts[:block] = true
++          @in_bc = opts
++          opts[:lang] ? '' : "<pre#{pba(opts)}>"
++        end
++        def bc_close(opts)  # :nodoc:
++          @in_bc = nil
++          opts[:lang] ? '' : "</pre>\n"
++        end
++        def escape_pre(text)
++          if @in_bc ||= nil
++            text
++          else
++            html_esc(text, :html_escape_preformatted)
++          end
++        end
++      end
++    end
++
++    module TextileDoc  # :nodoc:
++      attr_accessor :filter_coderay
++    end
++    
++  end
++  
++end
++
++CodeRay::ForRedCloth.install
+\ No newline at end of file
+Index: lib/coderay/scanners/ruby/patterns.rb
+===================================================================
+--- lib/coderay/scanners/ruby/patterns.rb	(revision 200)
++++ lib/coderay/scanners/ruby/patterns.rb	(revision 250)
+@@ -14,19 +14,14 @@
+ 
+     DEF_KEYWORDS = %w[ def ]
+     UNDEF_KEYWORDS = %w[ undef ]
++    ALIAS_KEYWORDS = %w[ alias ]
+     MODULE_KEYWORDS = %w[class module]
+     DEF_NEW_STATE = WordList.new(:initial).
+       add(DEF_KEYWORDS, :def_expected).
+       add(UNDEF_KEYWORDS, :undef_expected).
++      add(ALIAS_KEYWORDS, :alias_expected).
+       add(MODULE_KEYWORDS, :module_expected)
+ 
+-    IDENTS_ALLOWING_REGEXP = %w[
+-      and or not while until unless if then elsif when sub sub! gsub gsub!
+-      scan slice slice! split
+-    ]
+-    REGEXP_ALLOWED = WordList.new(false).
+-      add(IDENTS_ALLOWING_REGEXP, :set)
+-
+     PREDEFINED_CONSTANTS = %w[
+       nil true false self
+       DATA ARGV ARGF __FILE__ __LINE__
+@@ -41,19 +36,20 @@
+     METHOD_NAME = / #{IDENT} [?!]? /ox
+     METHOD_NAME_OPERATOR = /
+       \*\*?           # multiplication and power
+-      | [-+]@?        # plus, minus
+-      | [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
++      | [-+~]@?       # plus, minus, tilde with and without @
++      | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
+       | \[\]=?        # array getter and setter
+       | << | >>       # append or shift left, shift right
+       | <=?>? | >=?   # comparison, rocket operator
+-      | ===?          # simple equality and case equality
++      | ===? | =~     # simple equality, case equality, match
++      | ![~=@]?       # negation with and without @, not-equal and not-match
+     /ox
+     METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
+     INSTANCE_VARIABLE = / @ #{IDENT} /ox
+     CLASS_VARIABLE = / @@ #{IDENT} /ox
+     OBJECT_VARIABLE = / @@? #{IDENT} /ox
+     GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+-    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
++    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
+     VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
+ 
+     QUOTE_TO_TYPE = {
+@@ -73,7 +69,7 @@
+     EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+     FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
+     FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
+-    NUMERIC = / [-+]? (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
++    NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
+ 
+     SYMBOL = /
+       :
+@@ -83,6 +79,7 @@
+       | ['"]
+       )
+     /ox
++    METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
+ 
+     # TODO investigste \M, \c and \C escape sequences
+     # (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )
+@@ -111,7 +108,7 @@
+       (?:
+         ( [A-Za-z_0-9]+ )  # $2 = delim
+       |
+-        ( ["'`] )          # $3 = quote, type
++        ( ["'`\/] )        # $3 = quote, type
+         ( [^\n]*? ) \3     # $4 = delim
+       )
+     /mx
+@@ -129,15 +126,14 @@
+     /mx
+     
+     # Checks for a valid value to follow. This enables
+-    # fancy_allowed in method calls.
++    # value_expected in method calls without parentheses.
+     VALUE_FOLLOWS = /
+-      \s+
++      (?>[ \t\f\v]+)
+       (?:
+         [%\/][^\s=]
+-      |
+-        <<-?\S
+-      |
+-        #{CHARACTER}
++      | <<-?\S
++      | [-+] \d
++      | #{CHARACTER}
+       )
+     /x
+ 
+Index: lib/coderay/scanners/ruby.rb
+===================================================================
+--- lib/coderay/scanners/ruby.rb	(revision 200)
++++ lib/coderay/scanners/ruby.rb	(revision 250)
+@@ -18,6 +18,7 @@
+     include Streamable
+ 
+     register_for :ruby
++    file_extension 'rb'
+ 
+     helper :patterns
+ 
+@@ -90,15 +91,15 @@
+             end
+ 
+           when '#'
+-            case peek(1)[0]
+-            when ?{
++            case peek(1)
++            when '{'
+               inline_block_stack << [state, depth, heredocs]
+               value_expected = true
+               state = :initial
+               depth = 1
+               tokens << [:open, :inline]
+               tokens << [match + getch, :inline_delimiter]
+-            when ?$, ?@
++            when '$', '@'
+               tokens << [match, :escape]
+               last_state = state  # scan one token as normal code, then return here
+               state = :initial
+@@ -121,36 +122,37 @@
+ # }}}
+         else
+ # {{{
+-          if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
+-            ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
+-            case m = match[0]
+-            when ?\s, ?\t, ?\f
+-              match << scan(/\s*/) unless eos? or heredocs
+-              kind = :space
+-            when ?\n, ?\\
+-              kind = :space
+-              if m == ?\n
+-                value_expected = true  # FIXME not quite true
+-                state = :initial if state == :undef_comma_expected
+-              end
+-              if heredocs
+-                unscan  # heredoc scanning needs \n at start
+-                state = heredocs.shift
+-                tokens << [:open, state.type]
+-                heredocs = nil if heredocs.empty?
+-                next
+-              else
+-                match << scan(/\s*/) unless eos?
+-              end
+-            when ?#, ?=, ?_
+-              kind = :comment
+-              value_expected = true
++          if match = scan(/[ \t\f]+/)
++            kind = :space
++            match << scan(/\s*/) unless eos? || heredocs
++            value_expected = true if match.index(?\n) # FIXME not quite true
++            tokens << [match, kind]
++            next
++            
++          elsif match = scan(/\\?\n/)
++            kind = :space
++            if match == "\n"
++              value_expected = true  # FIXME not quite true
++              state = :initial if state == :undef_comma_expected
++            end
++            if heredocs
++              unscan  # heredoc scanning needs \n at start
++              state = heredocs.shift
++              tokens << [:open, state.type]
++              heredocs = nil if heredocs.empty?
++              next
+             else
+-              raise_inspect 'else-case _ reached, because case %p was
+-                not handled' % [matched[0].chr], tokens
++              match << scan(/\s*/) unless eos?
+             end
+             tokens << [match, kind]
+             next
++          
++          elsif match = scan(/\#.*/) or
++            ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
++              kind = :comment
++              value_expected = true
++              tokens << [match, kind]
++              next
+ 
+           elsif state == :initial
+ 
+@@ -167,19 +169,19 @@
+                 end
+               end
+               ## experimental!
+-              value_expected = :set if
+-                patterns::REGEXP_ALLOWED[match] or check(/#{patterns::VALUE_FOLLOWS}/o)
++              value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
+             
+             elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o)
+               kind = :ident
+               value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
+ 
+             # OPERATORS #
+-            elsif not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)
++            # TODO: match (), [], {} as one single operator
++            elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
+               if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
+                 value_expected = :set
+               end
+-              last_token_dot = :set if match == '.' or match == '::'
++              last_token_dot = :set if self[1]
+               kind = :operator
+               unless inline_block_stack.empty?
+                 case match
+@@ -210,8 +212,9 @@
+               interpreted = true
+               state = patterns::StringState.new :regexp, interpreted, match
+ 
+-            elsif match = scan(/#{patterns::NUMERIC}/o)
+-              kind = if self[1] then :float else :integer end
++            # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
++            elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
++              kind = self[1] ? :float : :integer
+ 
+             elsif match = scan(/#{patterns::SYMBOL}/o)
+               case delim = match[1]
+@@ -285,6 +288,18 @@
+               next
+             end
+ 
++          elsif state == :module_expected
++            if match = scan(/<</)
++              kind = :operator
++            else
++              state = :initial
++              if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
++                kind = :class
++              else
++                next
++              end
++            end
++
+           elsif state == :undef_expected
+             state = :undef_comma_expected
+             if match = scan(/#{patterns::METHOD_NAME_EX}/o)
+@@ -306,6 +321,15 @@
+               next
+             end
+ 
++          elsif state == :alias_expected
++            if match = scan(/(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
++              tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
++              tokens << [self[2], :space]
++              tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
++            end
++            state = :initial
++            next
++
+           elsif state == :undef_comma_expected
+             if match = scan(/,/)
+               kind = :operator
+@@ -315,24 +339,14 @@
+               next
+             end
+ 
+-          elsif state == :module_expected
+-            if match = scan(/<</)
+-              kind = :operator
+-            else
+-              state = :initial
+-              if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
+-                kind = :class
+-              else
+-                next
+-              end
+-            end
+-
+           end
+ # }}}
++          
++          unless kind == :error
++            value_expected = value_expected == :set
++            last_token_dot = last_token_dot == :set
++          end
+ 
+-          value_expected = value_expected == :set
+-          last_token_dot = last_token_dot == :set
+-
+           if $DEBUG and not kind
+             raise_inspect 'Error token %p in line %d' %
+               [[match, kind], line], tokens, state
+Index: lib/coderay/scanners/c.rb
+===================================================================
+--- lib/coderay/scanners/c.rb	(revision 200)
++++ lib/coderay/scanners/c.rb	(revision 250)
+@@ -4,6 +4,8 @@
+   class C < Scanner
+ 
+     register_for :c
++    
++    include Streamable
+ 
+     RESERVED_WORDS = [
+       'asm', 'break', 'case', 'continue', 'default', 'do', 'else',
+@@ -42,7 +44,7 @@
+ 
+         kind = nil
+         match = nil
+-
++        
+         case state
+ 
+         when :initial
+Index: lib/coderay/scanners/scheme.rb
+===================================================================
+--- lib/coderay/scanners/scheme.rb	(revision 0)
++++ lib/coderay/scanners/scheme.rb	(revision 250)
+@@ -0,0 +1,142 @@
++module CodeRay
++  module Scanners
++
++    # Scheme scanner for CodeRay (by closure).
++    # Thanks to murphy for putting CodeRay into public.
++    class Scheme < Scanner
++      
++      register_for :scheme
++      file_extension :scm
++
++      CORE_FORMS = %w[
++        lambda let let* letrec syntax-case define-syntax let-syntax
++        letrec-syntax begin define quote if or and cond case do delay
++        quasiquote set! cons force call-with-current-continuation call/cc
++      ]
++
++      IDENT_KIND = CaseIgnoringWordList.new(:ident).
++        add(CORE_FORMS, :reserved)
++      
++      #IDENTIFIER_INITIAL = /[a-z!@\$%&\*\/\:<=>\?~_\^]/i
++      #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
++      #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
++      IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./
++      DIGIT = /\d/
++      DIGIT10 = DIGIT
++      DIGIT16 = /[0-9a-f]/i
++      DIGIT8 = /[0-7]/
++      DIGIT2 = /[01]/
++      RADIX16 = /\#x/i
++      RADIX8 = /\#o/i
++      RADIX2 = /\#b/i
++      RADIX10 = /\#d/i
++      EXACTNESS = /#i|#e/i
++      SIGN = /[\+-]?/
++      EXP_MARK = /[esfdl]/i
++      EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
++      SUFFIX = /#{EXP}?/
++      PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
++      PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
++      PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
++      PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
++      UINT10 = /#{DIGIT10}+#*/
++      UINT16 = /#{DIGIT16}+#*/
++      UINT8 = /#{DIGIT8}+#*/
++      UINT2 = /#{DIGIT2}+#*/
++      DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
++      UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
++      UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
++      UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
++      UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
++      REAL10 = /#{SIGN}#{UREAL10}/
++      REAL16 = /#{SIGN}#{UREAL16}/
++      REAL8 = /#{SIGN}#{UREAL8}/
++      REAL2 = /#{SIGN}#{UREAL2}/
++      IMAG10 = /i|#{UREAL10}i/
++      IMAG16 = /i|#{UREAL16}i/
++      IMAG8 = /i|#{UREAL8}i/
++      IMAG2 = /i|#{UREAL2}i/
++      COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
++      COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
++      COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
++      COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
++      NUM10 = /#{PREFIX10}?#{COMPLEX10}/
++      NUM16 = /#{PREFIX16}#{COMPLEX16}/
++      NUM8 = /#{PREFIX8}#{COMPLEX8}/
++      NUM2 = /#{PREFIX2}#{COMPLEX2}/
++      NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
++    
++    private
++      def scan_tokens tokens,options
++        
++        state = :initial
++        ident_kind = IDENT_KIND
++        
++        until eos?
++          kind = match = nil
++          
++          case state
++          when :initial
++            if scan(/ \s+ | \\\n /x)
++              kind = :space
++            elsif scan(/['\(\[\)\]]|#\(/)
++              kind = :operator_fat
++            elsif scan(/;.*/)
++              kind = :comment
++            elsif scan(/#\\(?:newline|space|.?)/)
++              kind = :char
++            elsif scan(/#[ft]/)
++              kind = :pre_constant
++            elsif scan(/#{IDENTIFIER}/o)
++              kind = ident_kind[matched]
++            elsif scan(/\./)
++              kind = :operator
++            elsif scan(/"/)
++              tokens << [:open, :string]
++              state = :string
++              tokens << ['"', :delimiter]
++              next
++            elsif scan(/#{NUM}/o) and not matched.empty?
++              kind = :integer
++            elsif getch
++              kind = :error
++            end
++            
++          when :string
++            if scan(/[^"\\]+/) or scan(/\\.?/)
++              kind = :content
++            elsif scan(/"/)
++              tokens << ['"', :delimiter]
++              tokens << [:close, :string]
++              state = :initial
++              next
++            else
++              raise_inspect "else case \" reached; %p not handled." % peek(1),
++                tokens, state
++            end
++            
++          else
++            raise "else case reached"
++          end
++          
++          match ||= matched
++          if $DEBUG and not kind
++            raise_inspect 'Error token %p in line %d' %
++            [[match, kind], line], tokens
++          end
++          raise_inspect 'Empty token', tokens, state unless match
++          
++          tokens << [match, kind]
++          
++        end  # until eos
++        
++        if state == :string
++          tokens << [:close, :string]
++        end
++        
++        tokens
++        
++      end #scan_tokens
++    end #class
++  end #module scanners
++end #module coderay
+\ No newline at end of file
+Index: lib/coderay/scanners/delphi.rb
+===================================================================
+--- lib/coderay/scanners/delphi.rb	(revision 200)
++++ lib/coderay/scanners/delphi.rb	(revision 250)
+@@ -29,13 +29,18 @@
+       'virtual', 'write', 'writeonly'
+     ]
+ 
+-    IDENT_KIND = CaseIgnoringWordList.new(:ident).
++    IDENT_KIND = CaseIgnoringWordList.new(:ident, caching=true).
+       add(RESERVED_WORDS, :reserved).
+       add(DIRECTIVES, :directive)
++    
++    NAME_FOLLOWS = CaseIgnoringWordList.new(false, caching=true).
++      add(%w(procedure function .))
+ 
++  private
+     def scan_tokens tokens, options
+ 
+       state = :initial
++      last_token = ''
+ 
+       until eos?
+ 
+@@ -45,19 +50,29 @@
+         if state == :initial
+           
+           if scan(/ \s+ /x)
+-            kind = :space
++            tokens << [matched, :space]
++            next
+             
+           elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
+-            kind = :preprocessor
++            tokens << [matched, :preprocessor]
++            next
+             
+           elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
+-            kind = :comment
++            tokens << [matched, :comment]
++            next
+             
+-          elsif scan(/ [-+*\/=<>:;,.@\^|\(\)\[\]]+ /x)
++          elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
+             kind = :operator
++          
++          elsif match = scan(/\./)
++            kind = :operator
++            if last_token == 'end'
++              tokens << [match, kind]
++              next
++            end
+             
+           elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+-            kind = IDENT_KIND[match]
++            kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
+             
+           elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
+             tokens << [:open, :char]
+@@ -101,6 +116,7 @@
+             state = :initial
+             next
+           elsif scan(/\n/)
++            tokens << [:close, :string]
+             kind = :error
+             state = :initial
+           else
+@@ -119,6 +135,7 @@
+         end
+         raise_inspect 'Empty token', tokens unless match
+ 
++        last_token = match
+         tokens << [match, kind]
+         
+       end
+Index: lib/coderay/scanners/debug.rb
+===================================================================
+--- lib/coderay/scanners/debug.rb	(revision 0)
++++ lib/coderay/scanners/debug.rb	(revision 250)
+@@ -0,0 +1,60 @@
++module CodeRay
++module Scanners
++
++  # = Debug Scanner
++  class Debug < Scanner
++
++    include Streamable
++    register_for :debug
++
++  protected
++    def scan_tokens tokens, options
++
++      opened_tokens = []
++
++      until eos?
++
++        kind = nil
++        match = nil
++
++          if scan(/\s+/)
++            tokens << [matched, :space]
++            next
++            
++          elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
++            kind = self[1].to_sym
++            match = self[2].gsub(/\\(.)/, '\1')
++            
++          elsif scan(/ (\w+) < /x)
++            kind = self[1].to_sym
++            opened_tokens << kind
++            match = :open
++            
++          elsif scan(/ > /x)
++            kind = opened_tokens.pop
++            match = :close
++            
++          else
++            kind = :error
++            getch
++
++          end
++                  
++        match ||= matched
++        if $DEBUG and not kind
++          raise_inspect 'Error token %p in line %d' %
++            [[match, kind], line], tokens
++        end
++        raise_inspect 'Empty token', tokens unless match
++
++        tokens << [match, kind]
++        
++      end
++      
++      tokens
++    end
++
++  end
++
++end
++end
+Index: lib/coderay/scanners/rhtml.rb
+===================================================================
+--- lib/coderay/scanners/rhtml.rb	(revision 200)
++++ lib/coderay/scanners/rhtml.rb	(revision 250)
+@@ -51,10 +51,10 @@
+           start_tag = match[/\A<%[-=]?/]
+           end_tag = match[/-?%?>?\z/]
+           tokens << [:open, :inline]
+-          tokens << [start_tag, :delimiter]
++          tokens << [start_tag, :inline_delimiter]
+           code = match[start_tag.size .. -1 - end_tag.size]
+           @ruby_scanner.tokenize code
+-          tokens << [end_tag, :delimiter] unless end_tag.empty?
++          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+           tokens << [:close, :inline]
+ 
+         else
+Index: lib/coderay/scanners/nitro_xhtml.rb
+===================================================================
+--- lib/coderay/scanners/nitro_xhtml.rb	(revision 200)
++++ lib/coderay/scanners/nitro_xhtml.rb	(revision 250)
+@@ -95,20 +95,20 @@
+           delimiter = CLOSING_PAREN[start_tag[1,1]]
+           end_tag = match[-1,1] == delimiter ? delimiter : ''
+           tokens << [:open, :inline]
+-          tokens << [start_tag, :delimiter]
++          tokens << [start_tag, :inline_delimiter]
+           code = match[start_tag.size .. -1 - end_tag.size]
+           @ruby_scanner.tokenize code
+-          tokens << [end_tag, :delimiter] unless end_tag.empty?
++          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+           tokens << [:close, :inline]
+ 
+         elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
+           start_tag = '<?r'
+           end_tag = match[-2,2] == '?>' ? '?>' : ''
+           tokens << [:open, :inline]
+-          tokens << [start_tag, :delimiter]
++          tokens << [start_tag, :inline_delimiter]
+           code = match[start_tag.size .. -(end_tag.size)-1]
+           @ruby_scanner.tokenize code
+-          tokens << [end_tag, :delimiter] unless end_tag.empty?
++          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+           tokens << [:close, :inline]
+ 
+         elsif entity = scan(/#{NITRO_ENTITY}/o)
+Index: lib/coderay/scanners/plaintext.rb
+===================================================================
+--- lib/coderay/scanners/plaintext.rb	(revision 200)
++++ lib/coderay/scanners/plaintext.rb	(revision 250)
+@@ -4,6 +4,8 @@
+   class Plaintext < Scanner
+ 
+     register_for :plaintext, :plain
++    
++    include Streamable
+ 
+     def scan_tokens tokens, options
+       text = (scan_until(/\z/) || '')
+Index: lib/coderay.rb
+===================================================================
+--- lib/coderay.rb	(revision 200)
++++ lib/coderay.rb	(revision 250)
+@@ -24,8 +24,8 @@
+ #
+ # == Usage
+ #
+-# Remember you need RubyGems to use CodeRay. Run Ruby with -rubygems option
+-# if required.
++# Remember you need RubyGems to use CodeRay, unless you have it in your load path. Run Ruby with
++# -rubygems option if required.
+ #
+ # === Highlight Ruby code in a string as html
+ # 
+@@ -44,19 +44,15 @@
+ # 
+ # You can include this div in your page. The used CSS styles can be printed with
+ # 
+-#   % ruby -rcoderay -e "print CodeRay::Encoders[:html]::CSS"
++#   % coderay_stylesheet
+ # 
+ # === Highlight without typing too much
+-#
++# 
+ # If you are one of the hasty (or lazy, or extremely curious) people, just run this file:
+-#
+-#   % ruby -rubygems coderay.rb
+ # 
+-# If the output was to fast for you, try
++#   % ruby -rubygems /path/to/coderay/coderay.rb > example.html
+ # 
+-#   % ruby -rubygems coderay.rb > example.html
+-#
+-# and look at the file it created.
++# and look at the file it created in your browser.
+ # 
+ # = CodeRay Module
+ #
+@@ -111,7 +107,7 @@
+ #
+ # CodeRay.scan_stream:: Scan in stream mode.
+ #
+-#  == All-in-One Encoding
++# == All-in-One Encoding
+ #
+ # CodeRay.encode:: Highlight a string with a given input and output format.
+ #
+@@ -121,11 +117,16 @@
+ # for this Encoder must only be done once.
+ #
+ # CodeRay.encoder:: Create an Encoder instance with format and options.
++# CodeRay.scanner:: Create an Scanner instance for lang, with '' as default code.
+ #
+-# There is no CodeRay.scanner method because Scanners are bound to an input string
+-# on creation; you can't re-use them with another string.
++# To make use of CodeRay.scanner, use CodeRay::Scanner::code=.
+ #
+-#  The scanning methods provide more flexibility; we recommend to use these.
++# The scanning methods provide more flexibility; we recommend to use these.
++# 
++# == Reusing Scanners and Encoders
++# 
++# If you want to re-use scanners and encoders (because that is faster), see
++# CodeRay::Duo for the most convenient (and recommended) interface.
+ module CodeRay
+ 
+   # Version: Major.Minor.Teeny[.Revision]
+@@ -133,7 +134,7 @@
+   # Minor: odd for beta, even for stable
+   # Teeny: development state
+   # Revision: Subversion Revision number (generated on rake)
+-  VERSION = '0.7.4'
++  VERSION = '0.7.9'
+ 
+   require 'coderay/tokens'
+   require 'coderay/scanner'
+@@ -170,7 +171,7 @@
+     def scan_file filename, lang = :auto, options = {}, &block
+       file = IO.read filename
+       if lang == :auto
+-        require 'coderay/helpers/filetype'
++        require 'coderay/helpers/file_type'
+         lang = FileType.fetch filename, :plaintext, true
+       end
+       scan file, lang, options = {}, &block
+@@ -314,6 +315,7 @@
+ # Run a test script.
+ if $0 == __FILE__
+   $stderr.print 'Press key to print demo.'; gets
+-  code = File.read($0)[/module CodeRay.*/m]
++  # Just use this file as an example of Ruby code.
++  code = File.read(__FILE__)[/module CodeRay.*/m]
+   print CodeRay.scan(code, :ruby).html
+ end
+
+Property changes on: lib
+___________________________________________________________________
+Added: svn:externals
+   + term http://term-ansicolor.rubyforge.org/svn/trunk/lib/term/
+
+