80 files changed, 723 insertions, 2922 deletions
diff --git a/Changes.textile b/Changes.textile
index ff9e6a9..fcee29b 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -12,6 +12,7 @@ h2. Changes in 1.1
 * Remove double-click toggle handler from HTML table output
 * Fixes to CSS scanner (floats, pseudoclasses)
 * CSS scanner uses @:id@ and @:tag@ now [#27]
+* Removed @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency. Nobody was using this, right?
 * Add .xaml file type [#121, thanks to Kozman Bálint]
 * @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
 * New token type @:id@ for CSS/Sass [#27]
@@ -20,6 +21,7 @@ h2. Changes in 1.1
 * Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
 * Fixed @:docstring@ token type style
 * @Plugin@ does not warn about fallback when default is defined
+* @Debug@ encoder refactored; use @DebugLint@ if you want strict checking now
 
 h2. Changes in 1.0.9
 
diff --git a/bench/bench.rb b/bench/bench.rb
index 1889eed..1958c73 100644
--- a/bench/bench.rb
+++ b/bench/bench.rb
@@ -14,11 +14,10 @@ require 'coderay'
 lang = ARGV.fetch(0) do
   puts <<-HELP
 Usage:
-  ruby bench.rb (c|ruby|dump) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
+  ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
 
   SIZE defaults to 100 kB (= 100,000 bytes).
   SIZE = 0 means the whole input.
-  SIZE is ignored when dump is input.
 
 -p generates a profile (slow! use with SIZE = 1)
 -o shows the output
@@ -48,10 +47,6 @@ if format == 'comp'
   end
 end
 
-$dump_input = lang == 'dump'
-$dump_output = format == 'dump'
-require 'coderay/helpers/gzip_simple.rb' if $dump_input
-
 def here fn = nil
   return MYDIR unless fn
   File.join here, fn
@@ -66,59 +61,38 @@ N.times do
 
   data = nil
   File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
-  if $dump_input
-    @size = CodeRay::Tokens.load(data).text.size
-  else
-    raise 'Example file is empty.' if data.empty?
-    unless @size.zero?
-      data += data until data.size >= @size
-      data = data[0, @size]
-    end
-    @size = data.size
+  raise 'Example file is empty.' if data.empty?
+  unless @size.zero?
+    data += data until data.size >= @size
+    data = data[0, @size]
   end
-
+  @size = data.size
+  
   options = {
     :tab_width => 2,
     # :line_numbers => :inline,
     :css => $style ? :style : :class,
   }
-  $hl = CodeRay.encoder(format, options) unless $dump_output
+  $hl = CodeRay.encoder(format, options)
   time = bm.report('CodeRay') do
     if $stream || true
-      if $dump_input
-        raise 'Can\'t stream dump.'
-      elsif $dump_output
-        raise 'Can\'t dump stream.'
-      end
       $o = $hl.encode(data, lang, options)
     else
-      if $dump_input
-        tokens = CodeRay::Tokens.load data
-      else
-        tokens = CodeRay.scan(data, lang)
-      end
+      tokens = CodeRay.scan(data, lang)
       tokens.optimize! if $optimize
-      if $dump_output
-        $o = tokens.optimize.dump
-      else
-        $o = tokens.encode($hl)
-      end
+      $o = tokens.encode($hl)
     end
   end
-  $file_created = here('test.' +
-    ($dump_output ? 'dump' : $hl.file_extension))
+  $file_created = here('test.' + $hl.file_extension)
   File.open($file_created, 'wb') do |f|
     # f.write $o
   end
-  Dir.chdir(here) do
-    FileUtils.copy 'test.dump', 'example.dump' if $dump_output
-  end
-
+  
   time_real = time.real
-
+  
   puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
   puts $o if ARGV.include? '-o'
-
+  
 end
 end
 puts "Files created: #$file_created"
diff --git a/bin/coderay b/bin/coderay
index 9aecb88..889ae72 100755
--- a/bin/coderay
+++ b/bin/coderay
@@ -155,8 +155,9 @@ when 'highlight', nil
         puts boom.message
       end
       # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
-    rescue CodeRay::Scanners::Scanner::ScanError  # FIXME: rescue Errno::EPIPE
-      # this is sometimes raised by pagers; ignore [TODO: wtf?]
+    rescue CodeRay::Scanners::Scanner::ScanError
+      # this is sometimes raised by pagers; ignore
+    # FIXME: rescue Errno::EPIPE
     ensure
       file.close if output_file
     end
diff --git a/lib/coderay.rb b/lib/coderay.rb
index 88c7cc2..24ae5a2 100644
--- a/lib/coderay.rb
+++ b/lib/coderay.rb
@@ -166,7 +166,6 @@ module CodeRay
     #
     # See also demo/demo_simple.
     def scan code, lang, options = {}, &block
-      # FIXME: return a proxy for direct-stream encoding
       TokensProxy.new code, lang, options, block
     end
     
diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index c03d3fb..f4db330 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -9,7 +9,6 @@ module Encoders
   #
   # You cannot fully restore the tokens information from the
   # output, because consecutive :space tokens are merged.
-  # Use Tokens#dump for caching purposes.
   # 
   # See also: Scanners::Debug
   class Debug < Encoder
@@ -18,38 +17,26 @@ module Encoders
     
     FILE_EXTENSION = 'raydebug'
     
-    def initialize options = {}
-      super
-      @opened = []
-    end
-    
     def text_token text, kind
-      raise 'empty token' if $CODERAY_DEBUG && text.empty?
       if kind == :space
         @out << text
       else
-        # TODO: Escape (
-        text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/)
-        @out << kind.to_s << '(' << text << ')'
+        text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
+        text = text.gsub(')',  '\\\\)')    if text.index(')')
+        @out << "#{kind}(#{text})"
       end
     end
     
     def begin_group kind
-      @opened << kind
-      @out << kind.to_s << '<'
+      @out << "#{kind}<"
     end
     
     def end_group kind
-      if @opened.last != kind
-        puts @out
-        raise "we are inside #{@opened.inspect}, not #{kind}"
-      end
-      @opened.pop
       @out << '>'
     end
     
     def begin_line kind
-      @out << kind.to_s << '['
+      @out << "#{kind}["
     end
     
     def end_line kind
diff --git a/lib/coderay/encoders/debug_lint.rb b/lib/coderay/encoders/debug_lint.rb
new file mode 100644
index 0000000..eeb2a92
--- /dev/null
+++ b/lib/coderay/encoders/debug_lint.rb
@@ -0,0 +1,55 @@
+module CodeRay
+module Encoders
+  
+  # = Debug Lint Encoder
+  #
+  # Debug encoder with additional checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::Debug
+  class DebugLint < Debug
+    
+    register_for :debug_lint
+    
+    InvalidTokenStream = Class.new StandardError
+    EmptyToken = Class.new InvalidTokenStream
+    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
+    
+    def initialize options = {}
+      super
+      @opened = []
+    end
+    
+    def text_token text, kind
+      raise EmptyToken, 'empty token' if text.empty?
+      super
+    end
+    
+    def begin_group kind
+      @opened << kind
+      super
+    end
+    
+    def end_group kind
+      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      super
+    end
+    
+    def begin_line kind
+      @opened << kind
+      super
+    end
+    
+    def end_line kind
+      raise IncorrectTokenGroupNesting, "We are inside #{@opened.inspect}, not #{kind}" if @opened.pop != kind
+      super
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/encoders/html/numbering.rb b/lib/coderay/encoders/html/numbering.rb
index 332145b..a1b9c04 100644
--- a/lib/coderay/encoders/html/numbering.rb
+++ b/lib/coderay/encoders/html/numbering.rb
@@ -26,7 +26,7 @@ module Encoders
               "<a href=\"##{anchor}\" name=\"#{anchor}\">#{line}</a>"
             end
           else
-            proc { |line| line.to_s }  # :to_s.to_proc in Ruby 1.8.7+
+            :to_s.to_proc
           end
         
         bold_every = options[:bold_every]
@@ -75,7 +75,7 @@ module Encoders
           line_number = start
           output.gsub!(/^.*$\n?/) do |line|
             line_number_text = bolding.call line_number
-            indent = ' ' * (max_width - line_number.to_s.size)  # TODO: Optimize (10^x)
+            indent = ' ' * (max_width - line_number.to_s.size)
             line_number += 1
             "<span class=\"line-numbers\">#{indent}#{line_number_text}</span>#{line}"
           end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index 2315d9e..b2f8b83 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -67,7 +67,6 @@ Token Types (%d):
       @type_stats['TOTAL'].count += 1
     end
     
-    # TODO Hierarchy handling
     def begin_group kind
       block_token ':begin_group', kind
     end
diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index 9894b91..c7ae014 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -19,105 +19,135 @@ module CodeRay
       register_for :terminal
       
       TOKEN_COLORS = {
-        :annotation => "\e[35m",
-        :attribute_name => "\e[33m",
+        :debug => "\e[1;37;44m",
+        
+        :annotation => "\e[34m",
+        :attribute_name => "\e[35m",
         :attribute_value => "\e[31m",
-        :binary => "\e[1;35m",
+        :binary => {
+          :self => "\e[31m",
+          :char => "\e[1;31m",
+          :delimiter => "\e[1;31m",
+        },
         :char => {
-          :self => "\e[36m", :delimiter => "\e[1;34m"
+          :self => "\e[35m",
+          :delimiter => "\e[1;35m"
         },
-        :class => "\e[1;35m",
+        :class => "\e[1;35;4m",
         :class_variable => "\e[36m",
         :color => "\e[32m",
-        :comment => "\e[37m",
-        :complex => "\e[1;34m",
-        :constant => "\e[1;34m\e[4m",
-        :decoration => "\e[35m",
-        :definition => "\e[1;32m",
-        :directive => "\e[32m\e[4m",
-        :doc => "\e[46m",
-        :doctype => "\e[1;30m",
-        :docstring => "\e[31m\e[4m",
-        :entity => "\e[33m",
-        :error => "\e[1;33m\e[41m",
+        :comment => {
+          :self => "\e[1;30m",
+          :char => "\e[37m",
+          :delimiter => "\e[37m",
+        },
+        :constant => "\e[1;34;4m",
+        :decorator => "\e[35m",
+        :definition => "\e[1;33m",
+        :directive => "\e[33m",
+        :docstring => "\e[31m",
+        :doctype => "\e[1;34m",
+        :done => "\e[1;30;2m",
+        :entity => "\e[31m",
+        :error => "\e[1;37;41m",
         :exception => "\e[1;31m",
         :float => "\e[1;35m",
         :function => "\e[1;34m",
-        :global_variable => "\e[42m",
+        :global_variable => "\e[1;32m",
         :hex => "\e[1;36m",
-        :include => "\e[33m",
+        :id => "\e[1;34m",
+        :include => "\e[31m",
         :integer => "\e[1;34m",
-        :key => "\e[35m",
-        :label => "\e[1;15m",
+        :imaginary => "\e[1;34m",
+        :important => "\e[1;31m",
+        :key => {
+          :self => "\e[35m",
+          :char => "\e[1;35m",
+          :delimiter => "\e[1;35m",
+        },
+        :keyword => "\e[32m",
+        :label => "\e[1;33m",
         :local_variable => "\e[33m",
-        :octal => "\e[1;35m",
-        :operator_name => "\e[1;29m",
+        :namespace => "\e[1;35m",
+        :octal => "\e[1;34m",
+        :predefined => "\e[36m",
         :predefined_constant => "\e[1;36m",
-        :predefined_type => "\e[1;30m",
-        :predefined => "\e[4m\e[1;34m",
-        :preprocessor => "\e[36m",
+        :predefined_type => "\e[1;32m",
+        :preprocessor => "\e[1;36m",
         :pseudo_class => "\e[1;34m",
         :regexp => {
-          :self => "\e[31m",
-          :content => "\e[31m",
-          :delimiter => "\e[1;29m",
+          :self => "\e[35m",
+          :delimiter => "\e[1;35m",
           :modifier => "\e[35m",
+          :char => "\e[1;35m",
         },
-        :reserved => "\e[1;31m",
+        :reserved => "\e[32m",
         :shell => {
-          :self => "\e[42m",
-          :content => "\e[1;29m",
-          :delimiter => "\e[37m",
+          :self => "\e[33m",
+          :char => "\e[1;33m",
+          :delimiter => "\e[1;33m",
+          :escape => "\e[1;33m",
         },
         :string => {
-          :self => "\e[32m",
-          :modifier => "\e[1;32m",
-          :escape => "\e[1;36m",
-          :delimiter => "\e[1;32m",
-          :char => "\e[1;36m",
+          :self => "\e[31m",
+          :modifier => "\e[1;31m",
+          :char => "\e[1;35m",
+          :delimiter => "\e[1;31m",
+          :escape => "\e[1;31m",
+        },
+        :symbol => {
+          :self => "\e[33m",
+          :delimiter => "\e[1;33m",
         },
-        :symbol => "\e[1;32m",
-        :tag => "\e[1;34m",
+        :tag => "\e[32m",
         :type => "\e[1;34m",
         :value => "\e[36m",
-        :variable => "\e[1;34m",
+        :variable => "\e[34m",
         
-        :insert => "\e[42m",
-        :delete => "\e[41m",
-        :change => "\e[44m",
-        :head => "\e[45m"
+        :insert => {
+          :self => "\e[42m",
+          :insert => "\e[1;32;42m",
+          :eyecatcher => "\e[102m",
+        },
+        :delete => {
+          :self => "\e[41m",
+          :delete => "\e[1;31;41m",
+          :eyecatcher => "\e[101m",
+        },
+        :change => {
+          :self => "\e[44m",
+          :change => "\e[37;44m",
+        },
+        :head => {
+          :self => "\e[45m",
+          :filename => "\e[37;45m"
+        },
       }
+      
       TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
-      TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
-      TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] =
-        TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+      TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
       
     protected
       
       def setup(options)
         super
         @opened = []
-        @subcolors = nil
+        @color_scopes = [TOKEN_COLORS]
       end
       
     public
       
       def text_token text, kind
-        if color = (@subcolors || TOKEN_COLORS)[kind]
-          if Hash === color
-            if color[:self]
-              color = color[:self]
-            else
-              @out << text
-              return
-            end
-          end
+        if color = @color_scopes.last[kind]
+          color = color[:self] if color.is_a? Hash
           
           @out << color
-          @out << text.gsub("\n", "\e[0m\n" + color)
+          @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text)
           @out << "\e[0m"
-          @out << @subcolors[:self] if @subcolors
+          if outer_color = @color_scopes.last[:self]
+            @out << outer_color
+          end
         else
           @out << text
         end
@@ -130,40 +160,33 @@ module CodeRay
       alias begin_line begin_group
       
       def end_group kind
-        if @opened.empty?
-          # nothing to close
-        else
-          @opened.pop
+        if @opened.pop
+          @color_scopes.pop
           @out << "\e[0m"
-          @out << open_token(@opened.last)
+          if outer_color = @color_scopes.last[:self]
+            @out << outer_color
+          end
         end
       end
       
       def end_line kind
-        if @opened.empty?
-          # nothing to close
-        else
-          @opened.pop
-          # whole lines to be highlighted,
-          # eg. added/modified/deleted lines in a diff
-          @out << (@line_filler ||= "\t" * 100 + "\e[0m")
-          @out << open_token(@opened.last)
-        end
+        @out << (@line_filler ||= "\t" * 100)
+        end_group kind
       end
       
     private
       
       def open_token kind
-        if color = TOKEN_COLORS[kind]
-          if Hash === color
-            @subcolors = color
+        if color = @color_scopes.last[kind]
+          if color.is_a? Hash
+            @color_scopes << color
             color[:self]
           else
-            @subcolors = {}
+            @color_scopes << @color_scopes.last
             color
           end
         else
-          @subcolors = nil
+          @color_scopes << @color_scopes.last
           ''
         end
       end
diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index a5d83ff..19f27ac 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -77,54 +77,56 @@ module CodeRay
     end
     
     TypeFromExt = {
-      'c'        => :c,
-      'cfc'      => :xml,
-      'cfm'      => :xml,
-      'clj'      => :clojure,
-      'css'      => :css,
-      'diff'     => :diff,
-      'dpr'      => :delphi,
-      'erb'      => :erb,
-      'gemspec'  => :ruby,
-      'groovy'   => :groovy,
-      'gvy'      => :groovy,
-      'h'        => :c,
-      'haml'     => :haml,
-      'htm'      => :html,
-      'html'     => :html,
-      'html.erb' => :erb,
-      'java'     => :java,
-      'js'       => :java_script,
-      'json'     => :json,
-      'mab'      => :ruby,
-      'pas'      => :delphi,
-      'patch'    => :diff,
-      'phtml'    => :php,
-      'php'      => :php,
-      'php3'     => :php,
-      'php4'     => :php,
-      'php5'     => :php,
-      'prawn'    => :ruby,
-      'py'       => :python,
-      'py3'      => :python,
-      'pyw'      => :python,
-      'rake'     => :ruby,
-      'raydebug' => :raydebug,
-      'rb'       => :ruby,
-      'rbw'      => :ruby,
-      'rhtml'    => :erb,
-      'rjs'      => :ruby,
-      'rpdf'     => :ruby,
-      'ru'       => :ruby,
-      'rxml'     => :ruby,
-      'sass'     => :sass,
-      'sql'      => :sql,
-      'tmproj'   => :xml,
-      'xaml'     => :xml,
-      'xhtml'    => :html,
-      'xml'      => :xml,
-      'yaml'     => :yaml,
-      'yml'      => :yaml,
+      'c'         => :c,
+      'cfc'       => :xml,
+      'cfm'       => :xml,
+      'clj'       => :clojure,
+      'css'       => :css,
+      'diff'      => :diff,
+      'dpr'       => :delphi,
+      'erb'       => :erb,
+      'gemspec'   => :ruby,
+      'groovy'    => :groovy,
+      'gvy'       => :groovy,
+      'h'         => :c,
+      'haml'      => :haml,
+      'htm'       => :html,
+      'html'      => :html,
+      'html.erb'  => :erb,
+      'java'      => :java,
+      'js'        => :java_script,
+      'json'      => :json,
+      'mab'       => :ruby,
+      'pas'       => :delphi,
+      'patch'     => :diff,
+      'phtml'     => :php,
+      'php'       => :php,
+      'php3'      => :php,
+      'php4'      => :php,
+      'php5'      => :php,
+      'prawn'     => :ruby,
+      'py'        => :python,
+      'py3'       => :python,
+      'pyw'       => :python,
+      'rake'      => :ruby,
+      'raydebug'  => :raydebug,
+      'rb'        => :ruby,
+      'rbw'       => :ruby,
+      'rhtml'     => :erb,
+      'rjs'       => :ruby,
+      'rpdf'      => :ruby,
+      'ru'        => :ruby,
+      'rxml'      => :ruby,
+      'sass'      => :sass,
+      'sql'       => :sql,
+      'taskpaper' => :taskpaper,
+      'template'  => :json,  # AWS CloudFormation template
+      'tmproj'    => :xml,
+      'xaml'      => :xml,
+      'xhtml'     => :html,
+      'xml'       => :xml,
+      'yaml'      => :yaml,
+      'yml'       => :yaml,
     }
     for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
       TypeFromExt[cpp_alias] = :cpp
diff --git a/lib/coderay/helpers/gzip.rb b/lib/coderay/helpers/gzip.rb
deleted file mode 100644
index 245014a..0000000
--- a/lib/coderay/helpers/gzip.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-module CodeRay
-  
-  # A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
-  module GZip
-    
-    require 'zlib'
-    
-    # The default zipping level. 7 zips good and fast.
-    DEFAULT_GZIP_LEVEL = 7
-    
-    # Unzips the given string +s+.
-    #
-    # Example:
-    #   require 'gzip_simple'
-    #   print GZip.gunzip(File.read('adresses.gz'))
-    def GZip.gunzip s
-      Zlib::Inflate.inflate s
-    end
-    
-    # Zips the given string +s+.
-    #
-    # Example:
-    #   require 'gzip_simple'
-    #   File.open('adresses.gz', 'w') do |file
-    #     file.write GZip.gzip('Mum: 0123 456 789', 9)
-    #   end
-    #
-    # If you provide a +level+, you can control how strong
-    # the string is compressed:
-    # - 0: no compression, only convert to gzip format
-    # - 1: compress fast
-    # - 7: compress more, but still fast (default)
-    # - 8: compress more, slower
-    # - 9: compress best, very slow
-    def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
-      Zlib::Deflate.new(level).deflate s, Zlib::FINISH
-    end
-    
-  end
-  
-end
diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 4e0f462..3754a9b 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -14,15 +14,17 @@ module Scanners
     
     ESCAPE = / [bfnrt\\"\/] /x  # :nodoc:
     UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x  # :nodoc:
+    KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /x
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     # See http://json.org/ for a definition of the JSON lexic/grammar.
     def scan_tokens encoder, options
-      
-      state = :initial
-      stack = []
-      key_expected = false
+      state = options[:state] || @state
       
       until eos?
         
@@ -32,18 +34,11 @@ module Scanners
           if match = scan(/ \s+ /x)
             encoder.text_token match, :space
           elsif match = scan(/"/)
-            state = key_expected ? :key : :string
+            state = check(/#{KEY}/o) ? :key : :string
             encoder.begin_group state
             encoder.text_token match, :delimiter
           elsif match = scan(/ [:,\[{\]}] /x)
             encoder.text_token match, :operator
-            case match
-            when ':' then key_expected = false
-            when ',' then key_expected = true if stack.last == :object
-            when '{' then stack << :object; key_expected = true
-            when '[' then stack << :array
-            when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
-            end
           elsif match = scan(/ true | false | null /x)
             encoder.text_token match, :value
           elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x)
@@ -82,6 +77,10 @@ module Scanners
         end
       end
       
+      if options[:keep_state]
+        @state = state
+      end
+      
       if [:string, :key].include? state
         encoder.end_group state
       end
diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index a9492ab..09c8b6e 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -157,12 +157,12 @@ module Scanners
             encoder.text_token match, :operator
           
           elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
+            modifiers = self[1]
             string_delimiter = self[2]
-            string_type = docstring_coming ? :docstring : :string
+            string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string)
             docstring_coming = false if docstring_coming
             encoder.begin_group string_type
             string_raw = false
-            modifiers = self[1]
             unless modifiers.empty?
               string_raw = !!modifiers.index(?r)
               encoder.text_token modifiers, :modifier
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index 7b2381d..c05ffd5 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -56,25 +56,26 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .annotation { color:#007 }
 .attribute-name { color:#b48 }
 .attribute-value { color:#700 }
-.binary { color:#509 }
+.binary { color:#549 }
+.binary .char { color:#325 }
+.binary .delimiter { color:#325 }
+.char { color:#D20 }
 .char .content { color:#D20 }
 .char .delimiter { color:#710 }
-.char { color:#D20 }
 .class { color:#B06; font-weight:bold }
 .class-variable { color:#369 }
 .color { color:#0A0 }
 .comment { color:#777 }
 .comment .char { color:#444 }
 .comment .delimiter { color:#444 }
-.complex { color:#A08 }
 .constant { color:#036; font-weight:bold }
 .decorator { color:#B0B }
 .definition { color:#099; font-weight:bold }
 .delimiter { color:black }
 .directive { color:#088; font-weight:bold }
-.doc { color:#970 }
-.doc-string { color:#D42; font-weight:bold }
+.docstring { color:#D42; }
 .doctype { color:#34b }
+.done { text-decoration: line-through; color: gray }
 .entity { color:#800; font-weight:bold }
 .error { color:#F00; background-color:#FAA }
 .escape  { color:#666 }
@@ -85,16 +86,16 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .global-variable { color:#d70 }
 .hex { color:#02b }
 .id  { color:#33D; font-weight:bold }
-.imaginary { color:#f00 }
 .include { color:#B44; font-weight:bold }
 .inline { background-color: hsla(0,0%,0%,0.07); color: black }
 .inline-delimiter { font-weight: bold; color: #666 }
 .instance-variable { color:#33B }
 .integer  { color:#00D }
+.imaginary { color:#f00 }
 .important { color:#D00 }
+.key { color: #606 }
 .key .char { color: #60f }
 .key .delimiter { color: #404 }
-.key { color: #606 }
 .keyword { color:#080; font-weight:bold }
 .label { color:#970; font-weight:bold }
 .local-variable { color:#963 }
@@ -106,33 +107,33 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .predefined-type { color:#0a5; font-weight:bold }
 .preprocessor { color:#579 }
 .pseudo-class { color:#00C; font-weight:bold }
+.regexp { background-color:hsla(300,100%,50%,0.06); }
 .regexp .content { color:#808 }
 .regexp .delimiter { color:#404 }
 .regexp .modifier { color:#C2C }
-.regexp { background-color:hsla(300,100%,50%,0.06); }
 .reserved { color:#080; font-weight:bold }
+.shell { background-color:hsla(120,100%,50%,0.06); }
 .shell .content { color:#2B2 }
 .shell .delimiter { color:#161 }
-.shell { background-color:hsla(120,100%,50%,0.06); }
+.string { background-color:hsla(0,100%,50%,0.05); }
 .string .char { color: #b0b }
 .string .content { color: #D20 }
 .string .delimiter { color: #710 }
 .string .modifier { color: #E40 }
-.string { background-color:hsla(0,100%,50%,0.05); }
+.symbol { color:#A60 }
 .symbol .content { color:#A60 }
 .symbol .delimiter { color:#630 }
-.symbol { color:#A60 }
 .map .content { color:#808 }
 .map .delimiter { color:#40A}
 .map { background-color:hsla(200,100%,50%,0.06); }
-.tag { color:#070 }
+.tag { color:#070; font-weight:bold }
 .type { color:#339; font-weight:bold }
-.value { color: #088; }
-.variable  { color:#037 }
+.value { color: #088 }
+.variable { color:#037 }
 
 .insert { background: hsla(120,100%,50%,0.12) }
 .delete { background: hsla(0,100%,50%,0.12) }
-.change { color: #bbf; background: #007; }
+.change { color: #bbf; background: #007 }
 .head { color: #f8f; background: #505 }
 .head .filename { color: white; }
 
@@ -143,8 +144,6 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .delete .delete { color: #c00; background:transparent; font-weight:bold }
 .change .change { color: #88f }
 .head .head { color: #f4f }
-
-.done { text-decoration: line-through; color: gray }
     TOKENS
 
   end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index 923c859..f1696df 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -10,80 +10,78 @@ module CodeRay
   TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
 
   TokenKinds.update(  # :nodoc:
-    :annotation          => 'annotation',
-    :attribute_name      => 'attribute-name',
-    :attribute_value     => 'attribute-value',
-    :binary              => 'bin',
-    :char                => 'char',
-    :class               => 'class',
-    :class_variable      => 'class-variable',
-    :color               => 'color',
-    :comment             => 'comment',
-    :complex             => 'complex',
-    :constant            => 'constant',
-    :content             => 'content',
-    :debug               => 'debug',
-    :decorator           => 'decorator',
-    :definition          => 'definition',
-    :delimiter           => 'delimiter',
-    :directive           => 'directive',
-    :doc                 => 'doc',
-    :doctype             => 'doctype',
-    :docstring           => 'doc-string',
-    :done                => 'done',
-    :entity              => 'entity',
-    :error               => 'error',
-    :escape              => 'escape',
-    :exception           => 'exception',
-    :filename            => 'filename',
-    :float               => 'float',
-    :function            => 'function',
-    :global_variable     => 'global-variable',
-    :hex                 => 'hex',
-    :id                  => 'id',
-    :imaginary           => 'imaginary',
-    :important           => 'important',
-    :include             => 'include',
-    :inline              => 'inline',
-    :inline_delimiter    => 'inline-delimiter',
-    :instance_variable   => 'instance-variable',
-    :integer             => 'integer',
-    :key                 => 'key',
-    :keyword             => 'keyword',
-    :label               => 'label',
-    :local_variable      => 'local-variable',
-    :map                 => 'map',
-    :modifier            => 'modifier',
-    :namespace           => 'namespace',
-    :octal               => 'octal',
-    :predefined          => 'predefined',
-    :predefined_constant => 'predefined-constant',
-    :predefined_type     => 'predefined-type',
-    :preprocessor        => 'preprocessor',
-    :pseudo_class        => 'pseudo-class',
-    :regexp              => 'regexp',
-    :reserved            => 'reserved',
-    :shell               => 'shell',
-    :string              => 'string',
-    :symbol              => 'symbol',
-    :tag                 => 'tag',
-    :type                => 'type',
-    :value               => 'value',
-    :variable            => 'variable',
-
-    :change              => 'change',
-    :delete              => 'delete',
-    :head                => 'head',
-    :insert              => 'insert',
-
-    :eyecatcher          => 'eyecatcher',
-
-    :ident               => false,
-    :operator            => false,
-
-    :space               => false,
-    :plain               => false
+    :debug               => 'debug',              # highlight for debugging (white on blue background)
+    
+    :annotation          => 'annotation',         # Groovy, Java
+    :attribute_name      => 'attribute-name',     # HTML, CSS
+    :attribute_value     => 'attribute-value',    # HTML
+    :binary              => 'binary',             # Python, Ruby
+    :char                => 'char',               # most scanners, also inside of strings
+    :class               => 'class',              # lots of scanners, for different purposes also in CSS
+    :class_variable      => 'class-variable',     # Ruby, YAML
+    :color               => 'color',              # CSS
+    :comment             => 'comment',            # most scanners
+    :constant            => 'constant',           # PHP, Ruby
+    :content             => 'content',            # inside of strings, most scanners
+    :decorator           => 'decorator',          # Python
+    :definition          => 'definition',         # CSS
+    :delimiter           => 'delimiter',          # inside strings, comments and other types
+    :directive           => 'directive',          # lots of scanners
+    :doctype             => 'doctype',            # Goorvy, HTML, Ruby, YAML
+    :docstring           => 'docstring',          # Python
+    :done                => 'done',               # Taskpaper
+    :entity              => 'entity',             # HTML
+    :error               => 'error',              # invalid token, most scanners
+    :escape              => 'escape',             # Ruby (string inline variables like #$foo, #@bar)
+    :exception           => 'exception',          # Java, PHP, Python
+    :filename            => 'filename',           # Diff
+    :float               => 'float',              # most scanners
+    :function            => 'function',           # CSS, JavaScript, PHP
+    :global_variable     => 'global-variable',    # Ruby, YAML
+    :hex                 => 'hex',                # hexadecimal number; lots of scanners
+    :id                  => 'id',                 # CSS
+    :imaginary           => 'imaginary',          # Python
+    :important           => 'important',          # CSS, Taskpaper
+    :include             => 'include',            # C, Groovy, Java, Python, Sass
+    :inline              => 'inline',             # nested code, eg. inline string evaluation; lots of scanners
+    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why use inline_delimiter?
+    :instance_variable   => 'instance-variable',  # Ruby
+    :integer             => 'integer',            # most scanners
+    :key                 => 'key',                # lots of scanners, used together with :value
+    :keyword             => 'keyword',            # reserved word that's actually implemented; most scanners
+    :label               => 'label',              # C, PHP
+    :local_variable      => 'local-variable',     # local and magic variables; some scanners
+    :map                 => 'map',                # Lua tables
+    :modifier            => 'modifier',           # used inside on strings; lots of scanners
+    :namespace           => 'namespace',          # Clojure, Java, Taskpaper
+    :octal               => 'octal',              # lots of scanners
+    :predefined          => 'predefined',         # predefined function: lots of scanners
+    :predefined_constant => 'predefined-constant',# lots of scanners
+    :predefined_type     => 'predefined-type',    # C, Java, PHP
+    :preprocessor        => 'preprocessor',       # C, Delphi, HTML
+    :pseudo_class        => 'pseudo-class',       # CSS
+    :regexp              => 'regexp',             # Groovy, JavaScript, Ruby
+    :reserved            => 'reserved',           # most scanners
+    :shell               => 'shell',              # Ruby
+    :string              => 'string',             # most scanners
+    :symbol              => 'symbol',             # Clojure, Ruby, YAML
+    :tag                 => 'tag',                # CSS, HTML
+    :type                => 'type',               # CSS, Java, SQL, YAML
+    :value               => 'value',              # used together with :key; CSS, JSON, YAML
+    :variable            => 'variable',           # Sass, SQL, YAML
+    
+    :change              => 'change',             # Diff
+    :delete              => 'delete',             # Diff
+    :head                => 'head',               # Diff, YAML
+    :insert              => 'insert',             # Diff
+    :eyecatcher          => 'eyecatcher',         # Diff
+    
+    :ident               => false,                # almost all scanners
+    :operator            => false,                # almost all scanners
+    
+    :space               => false,                # almost all scanners
+    :plain               => false                 # almost all scanners
   )
-
-  TokenKinds[:method]    = TokenKinds[:function]
+  
+  TokenKinds[:method] = TokenKinds[:function]
 end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 6957d69..54358d4 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -1,27 +1,22 @@
 module CodeRay
   
-  # GZip library for writing and reading token dumps.
-  autoload :GZip, coderay_path('helpers', 'gzip')
-  
-  # = Tokens  TODO: Rewrite!
-  #
-  # The Tokens class represents a list of tokens returnd from
-  # a Scanner.
+  # The Tokens class represents a list of tokens returned from
+  # a Scanner. It's actually just an Array with a few helper methods.
   #
-  # A token is not a special object, just a two-element Array
+  # A token itself is not a special object, just a two-element Array
   # consisting of
   # * the _token_ _text_ (the original source of the token in a String) or
   #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
   #
-  # A token looks like this:
+  # It looks like this:
   #
   #   ['# It looks like this', :comment]
   #   ['3.1415926', :float]
   #   ['$^', :error]
   #
   # Some scanners also yield sub-tokens, represented by special
-  # token actions, namely begin_group and end_group.
+  # token actions, for example :begin_group and :end_group.
   #
   # The Ruby scanner, for example, splits "a string" into:
   #
@@ -33,23 +28,17 @@ module CodeRay
   #   [:end_group, :string]
   #  ]
   #
-  # Tokens is the interface between Scanners and Encoders:
-  # The input is split and saved into a Tokens object. The Encoder
-  # then builds the output from this object.
-  #
-  # Thus, the syntax below becomes clear:
+  # Tokens can be used to save the output of a Scanners in a simple
+  # Ruby object that can be send to an Encoder later:
   #
-  #   CodeRay.scan('price = 2.59', :ruby).html
-  #   # the Tokens object is here -------^
-  #
-  # See how small it is? ;)
+  #   tokens = CodeRay.scan('price = 2.59', :ruby).tokens
+  #   tokens.encode(:html)
+  #   tokens.html
+  #   CodeRay.encoder(:html).encode_tokens(tokens)
   #
   # Tokens gives you the power to handle pre-scanned code very easily:
-  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
-  # that you put in your DB.
-  # 
-  # It also allows you to generate tokens directly (without using a scanner),
-  # to load them from a file, and still use any Encoder that CodeRay provides.
+  # You can serialize it to a JSON string and store it in a database, pass it
+  # around to encode it more than once, send it to other algorithms...
   class Tokens < Array
     
     # The Scanner instance that created the tokens.
@@ -58,8 +47,7 @@ module CodeRay
     # Encode the tokens using encoder.
     #
     # encoder can be
-    # * a symbol like :html oder :statistic
-    # * an Encoder class
+    # * a plugin name like :html oder 'statistic'
     # * an Encoder object
     #
     # options are passed to the encoder.
@@ -157,53 +145,11 @@ module CodeRay
       parts
     end
     
-    # Dumps the object into a String that can be saved
-    # in files or databases.
-    #
-    # The dump is created with Marshal.dump;
-    # In addition, it is gzipped using GZip.gzip.
-    #
-    # The returned String object includes Undumping
-    # so it has an #undump method. See Tokens.load.
-    #
-    # You can configure the level of compression,
-    # but the default value 7 should be what you want
-    # in most cases as it is a good compromise between
-    # speed and compression rate.
-    #
-    # See GZip module.
-    def dump gzip_level = 7
-      dump = Marshal.dump self
-      dump = GZip.gzip dump, gzip_level
-      dump.extend Undumping
-    end
-    
     # Return the actual number of tokens.
     def count
       size / 2
     end
     
-    # Include this module to give an object an #undump
-    # method.
-    #
-    # The string returned by Tokens.dump includes Undumping.
-    module Undumping
-      # Calls Tokens.load with itself.
-      def undump
-        Tokens.load self
-      end
-    end
-    
-    # Undump the object using Marshal.load, then
-    # unzip it using GZip.gunzip.
-    #
-    # The result is commonly a Tokens object, but
-    # this is not guaranteed.
-    def Tokens.load dump
-      dump = GZip.gunzip dump
-      @dump = Marshal.load dump
-    end
-    
     alias text_token push
     def begin_group kind; push :begin_group, kind end
     def end_group kind; push :end_group, kind end
diff --git a/rake_helpers/ca.rb b/rake_helpers/ca.rb
deleted file mode 100644
index ba6fe1e..0000000
--- a/rake_helpers/ca.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-#!c:/ruby/bin/rubyw
-# Hy-Ca 0.2 by murphy
-
-module Hy
-  def self.ca str
-    str.gsub! %r-^(\s*)(?://)(.*)?-, '\1/*\2*/'
-    str.gsub! %r-\s*/\*.*?\*/\n?-m, ''
-    str.gsub!(/<<(.*?)>>/m) do
-      begin
-        eval $1
-        ''
-      rescue Exception => boom
-        "<<\n#{boom}\n>>"
-      end
-    end
-    
-    str.gsub!(/\$([\w_]+)/m) do
-      begin
-        eval $1
-      rescue
-        ''
-      end
-    end
-
-    str
-  end
-end
-
-begin
-  if file = ENV['PATH_TRANSLATED']
-    puts "Content-Type: text/css"
-    puts
-    ca = File.read file
-  else
-    ca = ARGF.read
-  end
-  print Hy.ca(ca)
-rescue => boom
-  p boom
-end if __FILE__ == $0
diff --git a/rake_helpers/coderay_rdoc_template.rb b/rake_helpers/coderay_rdoc_template.rb
deleted file mode 100644
index 21a3231..0000000
--- a/rake_helpers/coderay_rdoc_template.rb
+++ /dev/null
@@ -1,636 +0,0 @@
-# This goes to /lib/ruby/1.8/rdoc/generators/template/html/
-module RDoc::Page
-
-FONTS = "Tahoma, Verdana, sans-serif"
-
-require 'rake_helpers/ca.rb'
-
-Hy.ca <<CA
-<<
-	BACKGROUND = '#322'
-
-	GOLD = '#e92'
-	DARK_GOLD = '#630'
-	FIRE = '#fc3'
-	MORE_FIRE = '#fe0'
-	LIGHT_BLUE = '#CCF'
-	DARK_BLUE = '#004'
-	FLARE = LIGHT_BLUE
-	DARK_GRAY = '#444'
-	PURPLE = '#d8f'
-	DARK_PURPLE = '#303'
-
-	LINK = LIGHT_BLUE
-	VISITED = '#ccc'
-	HOVER = '#f44'
-
-	BORDER = '3px double gray'
->>
-CA
-
-require 'coderay'
-
-STYLE = Hy.ca <<CSS + CodeRay::Encoders[:html]::CSS.new.stylesheet
-a { text-decoration: none; }
-a:link { color: $LINK }
-a:visited { color: $VISITED }
-a:hover, a:active, a:focus { color: $HOVER }
-
-body {
-  background-color: $BACKGROUND;
-  color: $GOLD;
-  margin: 0px;
-  font-family: %fonts%;
-}
-
-img { border: $BORDER; }
-
-hr {
-	background-color: none; color: none;
-	height: 3px;
-	border: 1px solid gray;
-	margin: 0.5em 0em;
-	padding: 0px;
-}
-
-tt { color: $LIGHT_BLUE; font-size: 1em; }
-
-.sectiontitle {
-	font-size: 1.2em;
-	font-weight: bold;
-	font-color: white;
-	border: $BORDER;
-	padding: 0.2em;
-	margin: 1em auto; margin-top: 3em;
-	text-align: center;
-	width: 75%;
-	background-color: $DARK_BLUE;
-}
-
-.attr-rw {
-  padding-left: 1em;
-  padding-right: 1em;
-  text-align: center;
-  color: silver;
-}
-
-.attr-name {
-  font-weight: bold;
-}
-
-.attr-desc {
-}
-
-.attr-value {
-  font-family: monospace;
-	color: $LIGHT_BLUE;
-	font-size: 1em;
-}
-
-.banner {
-	border-collapse: collapse;
-  font-size: small;
-  background: $DARK_PURPLE;
-  color: silver;
-  border: 0px;
-  border-bottom: $BORDER;
-  padding: 0.5em;
-  margin-bottom: 1em;
-}
-.xbanner table { border-collapse: collapse; }
-
-.banner td {
-  color: silver;
-	background-color: transparent;
-	padding: 0.2em 0.5em;
-}
-.banner td.ruby-chan {
-	vertical-align: bottom;
-	padding: 0px;
-	width: 1px;
-}
-
-.file-title-prefix { }
-
-td.file-title {
-  font-size: 140%;
-  font-weight: bold;
-  color: $PURPLE;
-}
-
-.dyn-source {
-  display: none;
-  margin: 0.5em;
-}
-
-.method {
-  margin-left: 1em;
-  margin-right: 1em;
-  margin-bottom: 1em;
-  border: 1px solid white;
-	color: $MORE_FIRE;
-	background: $DARK_PURPLE;
-}
-
-.description pre {
-	border: 1px solid gray;
-	background: $DARK_BLUE;
-	color: white;
-	padding: 0.5em;
-}
-
-.method .title {
-  font-family: monospace;
-  font-size: larger;
-  color: $PURPLE;
-  background: $DARK_GRAY;
-  border-bottom: $BORDER;
-  margin: 0px; padding: 0.1em 0.5em;
-}
-
-.method .description, .method .sourcecode {
-  margin: 0.2em 1em;
-}
-.method p {
-	color: $GOLD;
-}
-
-.description p, .sourcecode p {
-  margin-bottom: 0.5em;
-}
-
-.method .sourcecode p.source-link {
-  margin-top: 0.5em;
-  font-style: normal;
-}
-.arrow { font-size: larger; }
-
-.method .aka {
-  margin-top: 0.3em;
-  margin-left: 1em;
-  color: $FIRE;
-}
-
-#content {
-  margin: 2em; margin-top: 0px;
-}
-
-#description p {
-  margin-bottom: 0.5em;
-}
-
-h1 {
-	font-size: 1.5em;
-	font-weight: bold;
-	color: $FLARE;
-	border: $BORDER;
-	padding: 0.25em;
-	margin: 1em auto;
-	text-align: center;
-	width: 33%;
-	background-color: $DARK_BLUE;
-}
-h1 a:link, h1 a:visited { color: $FLARE }
-
-h2 {
-	margin-bottom: 0.5em;
-	margin-top: 2em;
-  font-size: 1.2em;
-  font-weight: bold;
-  color: $FIRE;
-}
-
-h3, h4, h5, h6 {
-	margin-bottom: 0.4em;
-	margin-top: 1.5em;
-	padding: 0px;
-  border: 0px;
-  color: $FIRE;
-  font-size: 1em;
-}
-
-.sourcecode > pre {
-	border: 1px solid silver;
-	background: #112;
-	color: white;
-	padding: 0.5em;
-}
-CSS
-
-XHTML_FRAMESET_PREAMBLE = #<?xml version="1.0" encoding="%charset%"?>
-%{<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Frameset//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd">
-}
-
-XHTML_PREAMBLE =
-%{<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-}
-
-HEADER = XHTML_PREAMBLE + <<ENDHEADER
-<html>
-  <head>
-    <title>%title%</title>
-    <meta http-equiv="Content-Type" content="text/html; charset=%charset%" />
-    <link rel="stylesheet" href="%style_url%" type="text/css" media="screen" />
-
-    <script language="JavaScript" type="text/javascript">
-    // <![CDATA[
-
-        function toggleSource( id )
-        {
-          var elem
-          var link
-
-          if( document.getElementById )
-          {
-            elem = document.getElementById( id )
-            link = document.getElementById( "l_" + id )
-          }
-          else if ( document.all )
-          {
-            elem = eval( "document.all." + id )
-            link = eval( "document.all.l_" + id )
-          }
-          else
-            return false;
-
-          if( elem.style.display == "block" )
-          {
-            elem.style.display = "none"
-            link.innerHTML = "show source"
-          }
-          else
-          {
-            elem.style.display = "block"
-            link.innerHTML = "hide source"
-          }
-        }
-
-        function openCode( url )
-        {
-          window.open( url, "SOURCE_CODE", "width=400,height=400,scrollbars=yes" )
-        }
-      // ]]>
-    </script>
-  </head>
-
-  <body>
-ENDHEADER
-
-FILE_PAGE = <<HTML
-<table width="100%" border="0" cellpadding="0" cellspacing="0" class="banner">
-  <tr><td>
-    <table width="100%" border="0" cellpadding="0" cellspacing="0">
-    <tr>
-      <td class="file-title" colspan="2"><span class="file-title-prefix">File</span> %short_name%</td>
-		</tr>
-    <tr>
-    	<td>
-        <table border="0" cellspacing="0" cellpadding="2">
-          <tr>
-            <td>Path:</td>
-            <td>%full_path%
-IF:cvsurl
-				&nbsp;(<a href="%cvsurl%">CVS</a>)
-ENDIF:cvsurl
-            </td>
-          </tr>
-          <tr>
-            <td>Modified:</td>
-            <td>%dtm_modified%</td>
-          </tr>
-        </table>
-      </td></tr>
-    </table>
-  </td>
-  <td class="ruby-chan">
-	  <a href="http://rubychan.de"><img src="http://cycnus.de/ruby/pics/ruby-doc-chan.gif" alt="Ruby-Chan" style="border: 0px" /></a>
-  </td>
- </tr>
-</table><br />
-HTML
-
-###################################################################
-
-CLASS_PAGE = <<HTML
-<table width="100%" border="0" cellpadding="0" cellspacing="0" class="banner">
-	<tr>
-  	<td class="file-title"><span class="file-title-prefix">%classmod%</span> %full_name%</td>
-		<td rowspan="2" class="ruby-chan">
-	  <a href="http://rubychan.de"><img src="http://cycnus.de/ruby/pics/ruby-doc-chan.gif" alt="Ruby-Chan" style="border: 0px" /></a>
-		</td>
-	</tr>
-	<tr>
- 	 <td>
-    <table cellspacing="0" cellpadding="2">
-      <tr valign="top">
-        <td>In:</td>
-        <td>
-START:infiles
-HREF:full_path_url:full_path:
-IF:cvsurl
-&nbsp;(<a href="%cvsurl%">CVS</a>)
-ENDIF:cvsurl
-END:infiles
-        </td>
-      </tr>
-IF:parent
-    <tr>
-      <td>Parent:</td>
-      <td>
-IF:par_url
-        <a href="%par_url%">
-ENDIF:par_url
-%parent%
-IF:par_url
-         </a>
-ENDIF:par_url
-     </td>
-   </tr>
-ENDIF:parent
-         </table>
-        </td>
-        </tr>
-      </table>
-HTML
-
-###################################################################
-
-METHOD_LIST = <<HTML
-  <div id="content">
-IF:diagram
-  <table cellpadding="0" cellspacing="0" border="0" width="100%"><tr><td align="center">
-    %diagram%
-  </td></tr></table>
-ENDIF:diagram
-
-IF:description
-  <div class="description">%description%</div>
-ENDIF:description
-
-IF:requires
-  <div class="sectiontitle">Required Files</div>
-  <ul>
-START:requires
-  <li>HREF:aref:name:</li>
-END:requires
-  </ul>
-ENDIF:requires
-
-IF:toc
-  <div class="sectiontitle">Contents</div>
-  <ul>
-START:toc
-  <li><a href="#%href%">%secname%</a></li>
-END:toc
-  </ul>
-ENDIF:toc
-
-IF:methods
-  <div class="sectiontitle">Methods</div>
-  <ul>
-START:methods
-  <li>HREF:aref:name:</li>
-END:methods
-  </ul>
-ENDIF:methods
-
-IF:includes
-<div class="sectiontitle">Included Modules</div>
-<ul>
-START:includes
-  <li>HREF:aref:name:</li>
-END:includes
-</ul>
-ENDIF:includes
-
-START:sections
-IF:sectitle
-<div class="sectiontitle"><a nem="%secsequence%">%sectitle%</a></div>
-IF:seccomment
-<div class="description">
-%seccomment%
-</div>
-ENDIF:seccomment
-ENDIF:sectitle
-
-IF:classlist
-  <div class="sectiontitle">Classes and Modules</div>
-  %classlist%
-ENDIF:classlist
-
-IF:constants
-  <div class="sectiontitle">Constants</div>
-  <table border="0" cellpadding="5">
-START:constants
-  <tr valign="top">
-    <td class="attr-name">%name%</td>
-    <td>=</td>
-    <td class="attr-value">%value%</td>
-  </tr>
-IF:desc
-  <tr valign="top">
-    <td>&nbsp;</td>
-    <td colspan="2" class="attr-desc">%desc%</td>
-  </tr>
-ENDIF:desc
-END:constants
-  </table>
-ENDIF:constants
-
-IF:attributes
-  <div class="sectiontitle">Attributes</div>
-  <table border="0" cellpadding="5">
-START:attributes
-  <tr valign="top">
-    <td class="attr-rw">
-IF:rw
-[%rw%]
-ENDIF:rw
-    </td>
-    <td class="attr-name">%name%</td>
-    <td class="attr-desc">%a_desc%</td>
-  </tr>
-END:attributes
-  </table>
-ENDIF:attributes
-
-IF:method_list
-START:method_list
-IF:methods
-<div class="sectiontitle">%type% %category% methods</div>
-START:methods
-<div class="method">
-  <div class="title">
-IF:callseq
-    <a name="%aref%"></a><b>%callseq%</b>
-ENDIF:callseq
-IFNOT:callseq
-    <a name="%aref%"></a><b>%name%</b>%params%
-ENDIF:callseq
-IF:codeurl
-[ <a href="javascript:openCode("%codeurl%")">source</a> ]
-ENDIF:codeurl
-  </div>
-IF:m_desc
-  <div class="description">
-  %m_desc%
-  </div>
-ENDIF:m_desc
-IF:aka
-<div class="aka">
-  --- This method is also aliased as
-START:aka
-  <a href="%aref%">%name%</a>
-END:aka
-  ---
-</div>
-ENDIF:aka
-IF:sourcecode
-<div class="sourcecode">
-  <p class="source-link"><span class="arrow">&rarr;</span> <a href="javascript:toggleSource('%aref%_source')" id="l_%aref%_source">show source</a></p>
-  <div id="%aref%_source" class="dyn-source">
-%sourcecode%
-  </div>
-</div>
-ENDIF:sourcecode
-</div>
-END:methods
-ENDIF:methods
-END:method_list
-ENDIF:method_list
-END:sections
-</div>
-HTML
-
-FOOTER = <<ENDFOOTER
-  </body>
-</html>
-ENDFOOTER
-
-BODY = HEADER + <<ENDBODY
-  !INCLUDE! <!-- banner header -->
-
-  <div id="bodyContent">
-    #{METHOD_LIST}
-  </div>
-
-  #{FOOTER}
-ENDBODY
-
-########################## Source code ##########################
-
-SRC_PAGE = XHTML_PREAMBLE + <<HTML
-<html>
-<head><title>%title%</title>
-<meta http-equiv="Content-Type" content="text/html; charset=%charset%" />
-<style type="text/css">
-.ruby-comment    { color: green; font-style: italic }
-.ruby-constant   { color: #4433aa; font-weight: bold; }
-.ruby-identifier { color: #222222;  }
-.ruby-ivar       { color: #2233dd; }
-.ruby-keyword    { color: #3333FF; font-weight: bold }
-.ruby-node       { color: #777777; }
-.ruby-operator   { color: #111111;  }
-.ruby-regexp     { color: #662222; }
-.ruby-value      { color: #662222; font-style: italic }
-  .kw { color: #3333FF; font-weight: bold }
-  .cmt { color: green; font-style: italic }
-  .str { color: #662222; font-style: italic }
-  .re  { color: #662222; }
-</style>
-</head>
-<body bgcolor="white">
-<pre>%code%</pre>
-</body>
-</html>
-HTML
-
-########################## Index ################################
-
-FR_INDEX_BODY = <<HTML
-!INCLUDE!
-HTML
-
-FILE_INDEX = XHTML_PREAMBLE + <<HTML
-<html>
-<head><title>List</title>
-<meta http-equiv="Content-Type" content="text/html; charset=%charset%" />
-<style type="text/css">
-<!--
-#{ Hy.ca <<CA
-  body {
-    background-color: $DARK_PURPLE;
-    font-family: #{FONTS};
-    color: white;
-    margin: 0px;
-  }
-  .banner {
-    background: $DARK_BLUE;
-    color: $GOLD;
-    padding: 0em 0.2em;
-    border-bottom: $BORDER;
-    font-size: smaller;
-    font-weight: bold;
-    text-align: center;
-  }
-  .entries {
-    margin: 0.25em 0em 0em 0.5em;
-    font-size: 75%;
-  }
-  a { text-decoration: none; white-space: nowrap; }
-  a:link { color: $LINK; }
-  a:visited { color: $VISITED; }
-  a:hover, a:active, a:focus { color: $HOVER; }
-CA
-}
--->
-</style>
-<base target="docwin" />
-</head>
-<body>
-<div class="banner">%list_title%</div>
-<div class="entries">
-START:entries
-<a href="%href%">%name%</a><br />
-END:entries
-</div>
-</body></html>
-HTML
-
-CLASS_INDEX = FILE_INDEX
-METHOD_INDEX = FILE_INDEX
-
-INDEX = XHTML_FRAMESET_PREAMBLE + <<HTML
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>%title%</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=%charset%" />
-</head>
-
-<frameset cols="20%,*">
-    <frameset rows="15%,35%,50%">
-        <frame src="fr_file_index.html"   title="Files" name="Files" />
-        <frame src="fr_class_index.html"  name="Classes" />
-        <frame src="fr_method_index.html" name="Methods" />
-    </frameset>
-IF:inline_source
-      <frame  src="%initial_page%" name="docwin" />
-ENDIF:inline_source
-IFNOT:inline_source
-    <frameset rows="80%,20%">
-      <frame  src="%initial_page%" name="docwin" />
-      <frame  src="blank.html" name="source" />
-    </frameset>
-ENDIF:inline_source
-    <noframes>
-          <body bgcolor="white">
-            Click <a href="html/index.html">here</a> for a non-frames
-            version of this page.
-          </body>
-    </noframes>
-</frameset>
-
-</html>
-HTML
-
-end
diff --git a/rake_helpers/html_coderay_generator.rb b/rake_helpers/html_coderay_generator.rb
deleted file mode 100644
index 3c77705..0000000
--- a/rake_helpers/html_coderay_generator.rb
+++ /dev/null
@@ -1,1517 +0,0 @@
-# We're responsible for generating all the HTML files
-# from the object tree defined in code_objects.rb. We
-# generate:
-#
-# [files]   an html file for each input file given. These
-#           input files appear as objects of class
-#           TopLevel
-#
-# [classes] an html file for each class or module encountered.
-#           These classes are not grouped by file: if a file
-#           contains four classes, we'll generate an html
-#           file for the file itself, and four html files 
-#           for the individual classes. 
-#
-# [indices] we generate three indices for files, classes,
-#           and methods. These are displayed in a browser
-#           like window with three index panes across the
-#           top and the selected description below
-#
-# Method descriptions appear in whatever entity (file, class,
-# or module) that contains them.
-#
-# We generate files in a structure below a specified subdirectory,
-# normally +doc+.
-#
-#  opdir
-#     |
-#     |___ files
-#     |       |__  per file summaries
-#     |
-#     |___ classes
-#             |__ per class/module descriptions
-#
-# HTML is generated using the Template class.
-#
-
-require 'ftools'
-
-require 'rdoc/options'
-require 'rdoc/template'
-require 'rdoc/markup/simple_markup'
-require 'rdoc/markup/simple_markup/to_html'
-require 'cgi'
-
-module Generators
-
-  # Name of sub-direcories that hold file and class/module descriptions
-
-  FILE_DIR  = "files"
-  CLASS_DIR = "classes"
-  CSS_NAME  = "rdoc-style.css"
-  
-
-  ##
-  # Build a hash of all items that can be cross-referenced.
-  # This is used when we output required and included names: 
-  # if the names appear in this hash, we can generate
-  # an html cross reference to the appropriate description.
-  # We also use this when parsing comment blocks: any decorated 
-  # words matching an entry in this list are hyperlinked.
-
-  class AllReferences
-    @@refs = {}
-    
-    def AllReferences::reset
-      @@refs = {}
-    end
-
-    def AllReferences.add(name, html_class)
-      @@refs[name] = html_class
-    end
-
-    def AllReferences.[](name)
-      @@refs[name]
-    end
-
-    def AllReferences.keys
-      @@refs.keys
-    end
-  end
-
-
-  ##
-  # Subclass of the SM::ToHtml class that supports looking
-  # up words in the AllReferences list. Those that are
-  # found (like AllReferences in this comment) will
-  # be hyperlinked
-
-  class HyperlinkHtml < SM::ToHtml
-    # We need to record the html path of our caller so we can generate
-    # correct relative paths for any hyperlinks that we find
-    def initialize(from_path, context)
-      super()
-      @from_path = from_path
-
-      @parent_name = context.parent_name
-      @parent_name += "::" if @parent_name
-      @context = context
-    end
-
-    # We're invoked when any text matches the CROSSREF pattern
-    # (defined in MarkUp). If we fine the corresponding reference,
-    # generate a hyperlink. If the name we're looking for contains
-    # no punctuation, we look for it up the module/class chain. For
-    # example, HyperlinkHtml is found, even without the Generators::
-    # prefix, because we look for it in module Generators first.
-
-    def handle_special_CROSSREF(special)
-      name = special.text
-      if name[0,1] == '#'
-        lookup = name[1..-1]
-        name = lookup unless Options.instance.show_hash
-      else
-        lookup = name
-      end
-
-      if /([A-Z].*)[.\#](.*)/ =~ lookup
-        container = $1
-        method = $2
-        ref = @context.find_symbol(container, method)
-      else
-        ref = @context.find_symbol(lookup)
-      end
-
-      if ref and ref.document_self
-        "<a href=\"#{ref.as_href(@from_path)}\">#{name}</a>"
-      else
-        name
-      end
-    end
-
-
-    # Generate a hyperlink for url, labeled with text. Handle the
-    # special cases for img: and link: described under handle_special_HYPEDLINK
-    def gen_url(url, text)
-      if url =~ /([A-Za-z]+):(.*)/
-        type = $1
-        path = $2
-      else
-        type = "http"
-        path = url
-        url  = "http://#{url}"
-      end
-
-      if type == "link"
-        if path[0,1] == '#'     # is this meaningful?
-          url = path
-        else
-          url = HTMLGenerator.gen_url(@from_path, path)
-        end
-      end
-
-      if (type == "http" || type == "link") && 
-          url =~ /\.(gif|png|jpg|jpeg|bmp)$/
-
-        "<img src=\"#{url}\">"
-      else
-        "<a href=\"#{url}\">#{text.sub(%r{^#{type}:/*}, '')}</a>"
-      end
-    end
-
-    # And we're invoked with a potential external hyperlink mailto:
-    # just gets inserted. http: links are checked to see if they
-    # reference an image. If so, that image gets inserted using an
-    # <img> tag. Otherwise a conventional <a href> is used.  We also
-    # support a special type of hyperlink, link:, which is a reference
-    # to a local file whose path is relative to the --op directory.
-
-    def handle_special_HYPERLINK(special)
-      url = special.text
-      gen_url(url, url)
-    end
-
-    # HEre's a hypedlink where the label is different to the URL
-    #  <label>[url]
-    #
-    
-    def handle_special_TIDYLINK(special)
-      text = special.text
-#      unless text =~ /(\S+)\[(.*?)\]/
-      unless text =~ /\{(.*?)\}\[(.*?)\]/ or text =~ /(\S+)\[(.*?)\]/ 
-        return text
-      end
-      label = $1
-      url   = $2
-      gen_url(url, label)
-    end
-
-  end
-
-
-  
-  #####################################################################
-  #
-  # Handle common markup tasks for the various Html classes
-  #
-
-  module MarkUp
-
-    # Convert a string in markup format into HTML. We keep a cached
-    # SimpleMarkup object lying around after the first time we're
-    # called per object.
-
-    def markup(str, remove_para=false)
-      return '' unless str
-      unless defined? @markup
-        @markup = SM::SimpleMarkup.new
-
-        # class names, variable names, file names, or instance variables
-        @markup.add_special(/(
-                               \b([A-Z]\w*(::\w+)*[.\#]\w+)  #    A::B.meth
-                             | \b([A-Z]\w+(::\w+)*)       #    A::B..
-                             | \#\w+[!?=]?                #    #meth_name 
-                             | \b\w+([_\/\.]+\w+)+[!?=]?  #    meth_name
-                             )/x, 
-                            :CROSSREF)
-
-        # external hyperlinks
-        @markup.add_special(/((link:|https?:|mailto:|ftp:|www\.)\S+\w)/, :HYPERLINK)
-
-        # and links of the form  <text>[<url>]
-        @markup.add_special(/(((\{.*?\})|\b\S+?)\[\S+?\.\S+?\])/, :TIDYLINK)
-#        @markup.add_special(/\b(\S+?\[\S+?\.\S+?\])/, :TIDYLINK)
-
-      end
-      unless defined? @html_formatter
-        @html_formatter = HyperlinkHtml.new(self.path, self)
-      end
-
-      # Convert leading comment markers to spaces, but only
-      # if all non-blank lines have them
-
-      if str =~ /^(?>\s*)[^\#]/
-        content = str
-      else
-        content = str.gsub(/^\s*(#+)/)  { $1.tr('#',' ') }
-      end
-
-      res = @markup.convert(content, @html_formatter)
-      if remove_para
-        res.sub!(/^<p>/, '')
-        res.sub!(/<\/p>$/, '')
-      end
-      res
-    end
-
-    # Qualify a stylesheet URL; if if +css_name+ does not begin with '/' or
-    # 'http[s]://', prepend a prefix relative to +path+. Otherwise, return it
-    # unmodified.
-
-    def style_url(path, css_name=nil)
-#      $stderr.puts "style_url( #{path.inspect}, #{css_name.inspect} )"
-      css_name ||= CSS_NAME
-      if %r{^(https?:/)?/} =~ css_name
-        return css_name
-      else
-        return HTMLGenerator.gen_url(path, css_name)
-      end
-    end
-
-    # Build a webcvs URL with the given 'url' argument. URLs with a '%s' in them
-    # get the file's path sprintfed into them; otherwise they're just catenated
-    # together.
-
-    def cvs_url(url, full_path)
-      if /%s/ =~ url
-        return sprintf( url, full_path )
-      else
-        return url + full_path
-      end
-    end
-  end
-
-
-  #####################################################################
-  #
-  # A Context is built by the parser to represent a container: contexts
-  # hold classes, modules, methods, require lists and include lists.
-  # ClassModule and TopLevel are the context objects we process here
-  # 
-  class ContextUser
-
-    include MarkUp
-
-    attr_reader :context
-    
-    def initialize(context, options)
-      @context = context
-      @options = options
-    end
-      
-    # convenience method to build a hyperlink
-    def href(link, cls, name)
-      %{<a href="#{link}" class="#{cls}">#{name}</a>} #"
-    end
-
-    # return a reference to outselves to be used as an href=
-    # the form depends on whether we're all in one file
-    # or in multiple files
-
-    def as_href(from_path)
-      if @options.all_one_file
-        "#" + path
-      else
-        HTMLGenerator.gen_url(from_path, path)
-      end
-    end
-
-    # Create a list of HtmlMethod objects for each method
-    # in the corresponding context object. If the @options.show_all
-    # variable is set (corresponding to the <tt>--all</tt> option,
-    # we include all methods, otherwise just the public ones.
-
-    def collect_methods
-      list = @context.method_list
-      unless @options.show_all
-        list = list.find_all {|m| m.visibility == :public || m.force_documentation }
-      end
-      @methods = list.collect {|m| HtmlMethod.new(m, self, @options) }
-    end
-
-    # Build a summary list of all the methods in this context
-    def build_method_summary_list(path_prefix="")
-      collect_methods unless @methods
-      meths = @methods.sort
-      res = []
-      meths.each do |meth|
-        res << {
-          "name" => CGI.escapeHTML(meth.name),
-          "aref" => "#{path_prefix}\##{meth.aref}" 
-        }
-      end
-      res
-    end
-
-
-    # Build a list of aliases for which we couldn't find a
-    # corresponding method
-    def build_alias_summary_list(section)
-      values = []
-      @context.aliases.each do |al|
-        next unless al.section == section
-        res = {
-          'old_name' => al.old_name,
-          'new_name' => al.new_name,
-        }
-        if al.comment && !al.comment.empty?
-          res['desc'] = markup(al.comment, true)
-        end
-        values << res
-      end
-      values
-    end
-    
-    # Build a list of constants
-    def build_constants_summary_list(section)
-      values = []
-      @context.constants.each do |co|
-        next unless co.section == section
-        res = {
-          'name'  => co.name,
-          'value' => CGI.escapeHTML(co.value)
-        }
-        res['desc'] = markup(co.comment, true) if co.comment && !co.comment.empty?
-        values << res
-      end
-      values
-    end
-    
-    def build_requires_list(context)
-      potentially_referenced_list(context.requires) {|fn| [fn + ".rb"] }
-    end
-
-    def build_include_list(context)
-      potentially_referenced_list(context.includes)
-    end
-
-    # Build a list from an array of <i>Htmlxxx</i> items. Look up each
-    # in the AllReferences hash: if we find a corresponding entry,
-    # we generate a hyperlink to it, otherwise just output the name.
-    # However, some names potentially need massaging. For example,
-    # you may require a Ruby file without the .rb extension,
-    # but the file names we know about may have it. To deal with
-    # this, we pass in a block which performs the massaging,
-    # returning an array of alternative names to match
-
-    def potentially_referenced_list(array)
-      res = []
-      array.each do |i|
-        ref = AllReferences[i.name] 
-#         if !ref
-#           container = @context.parent
-#           while !ref && container
-#             name = container.name + "::" + i.name
-#             ref = AllReferences[name] 
-#             container = container.parent
-#           end
-#         end
-
-        ref = @context.find_symbol(i.name)
-        ref = ref.viewer if ref
-
-        if !ref && block_given?
-          possibles = yield(i.name)
-          while !ref and !possibles.empty?
-            ref = AllReferences[possibles.shift]
-          end
-        end
-        h_name = CGI.escapeHTML(i.name)
-        if ref and ref.document_self
-          path = url(ref.path)
-          res << { "name" => h_name, "aref" => path }
-        else
-          res << { "name" => h_name }
-        end
-      end
-      res
-    end
-
-    # Build an array of arrays of method details. The outer array has up
-    # to six entries, public, private, and protected for both class
-    # methods, the other for instance methods. The inner arrays contain
-    # a hash for each method
-
-    def build_method_detail_list(section)
-      outer = []
-
-      methods = @methods.sort
-      for singleton in [true, false]
-        for vis in [ :public, :protected, :private ] 
-          res = []
-          methods.each do |m|
-            if m.section == section and
-                m.document_self and 
-                m.visibility == vis and 
-                m.singleton == singleton
-              row = {}
-              if m.call_seq
-                row["callseq"] = m.call_seq.gsub(/->/, '&rarr;')
-              else
-                row["name"]        = CGI.escapeHTML(m.name)
-                row["params"]      = m.params
-              end
-              desc = m.description.strip
-              row["m_desc"]      = desc unless desc.empty?
-              row["aref"]        = m.aref
-              row["visibility"]  = m.visibility.to_s
-
-              alias_names = []
-              m.aliases.each do |other|
-                if other.viewer   # won't be if the alias is private
-                  alias_names << {
-                    'name' => other.name,
-                    'aref'  => other.viewer.as_href(path)
-                  } 
-                end
-              end
-              unless alias_names.empty?
-                row["aka"] = alias_names
-              end
-
-              if @options.inline_source
-                code = m.source_code
-                row["sourcecode"] = code if code
-              else
-                code = m.src_url
-                if code
-                  row["codeurl"] = code
-                  row["imgurl"]  = m.img_url
-                end
-              end
-              res << row
-            end
-          end
-          if res.size > 0 
-            outer << {
-              "type"    => vis.to_s.capitalize,
-              "category"    => singleton ? "Class" : "Instance",
-              "methods" => res
-            }
-          end
-        end
-      end
-      outer
-    end
-
-    # Build the structured list of classes and modules contained
-    # in this context. 
-
-    def build_class_list(level, from, section, infile=nil)
-      res = ""
-      prefix = "&nbsp;&nbsp;::" * level;
-
-      from.modules.sort.each do |mod|
-        next unless mod.section == section
-        next if infile && !mod.defined_in?(infile)
-        if mod.document_self
-          res << 
-            prefix <<
-            "Module " <<
-            href(url(mod.viewer.path), "link", mod.full_name) <<
-            "<br />\n" <<
-            build_class_list(level + 1, mod, section, infile)
-        end
-      end
-
-      from.classes.sort.each do |cls|
-        next unless cls.section == section
-        next if infile && !cls.defined_in?(infile)
-        if cls.document_self
-          res      <<
-            prefix << 
-            "Class " <<
-            href(url(cls.viewer.path), "link", cls.full_name) <<
-            "<br />\n" <<
-            build_class_list(level + 1, cls, section, infile)
-        end
-      end
-
-      res
-    end
-    
-    def url(target)
-      HTMLGenerator.gen_url(path, target)
-    end
-
-    def aref_to(target)
-      if @options.all_one_file
-        "#" + target
-      else
-        url(target)
-      end
-    end
-
-    def document_self
-      @context.document_self
-    end
-
-    def diagram_reference(diagram)
-      res = diagram.gsub(/((?:src|href)=")(.*?)"/) {
-        $1 + url($2) + '"'
-      }
-      res
-    end
-
-
-    # Find a symbol in ourselves or our parent
-    def find_symbol(symbol, method=nil)
-      res = @context.find_symbol(symbol, method)
-      if res
-        res = res.viewer
-      end
-      res
-    end
-
-    # create table of contents if we contain sections
-      
-    def add_table_of_sections
-      toc = []
-      @context.sections.each do |section|
-        if section.title
-          toc << {
-            'secname' => section.title,
-            'href'    => section.sequence
-          }
-        end
-      end
-      
-      @values['toc'] = toc unless toc.empty?
-    end
-
-
-  end
-
-  #####################################################################
-  #
-  # Wrap a ClassModule context
-
-  class HtmlClass < ContextUser
-
-    attr_reader :path
-
-    def initialize(context, html_file, prefix, options)
-      super(context, options)
-
-      @html_file = html_file
-      @is_module = context.is_module?
-      @values    = {}
-
-      context.viewer = self
-
-      if options.all_one_file
-        @path = context.full_name
-      else
-        @path = http_url(context.full_name, prefix)
-      end
-
-      collect_methods
-
-      AllReferences.add(name, self)
-    end
-
-    # return the relative file name to store this class in,
-    # which is also its url
-    def http_url(full_name, prefix)
-      path = full_name.dup
-      if path['<<']
-        path.gsub!(/<<\s*(\w*)/) { "from-#$1" }
-      end
-      File.join(prefix, path.split("::")) + ".html"
-    end
-
-
-    def name
-      @context.full_name
-    end
-
-    def parent_name
-      @context.parent.full_name
-    end
-
-    def index_name
-      name
-    end
-
-    def write_on(f)
-      value_hash
-      template = TemplatePage.new(RDoc::Page::BODY,
-                                      RDoc::Page::CLASS_PAGE,
-                                      RDoc::Page::METHOD_LIST)
-      template.write_html_on(f, @values)
-    end
-
-    def value_hash
-      class_attribute_values
-      add_table_of_sections
-
-      @values["charset"] = @options.charset
-      @values["style_url"] = style_url(path, @options.css)
-
-      d = markup(@context.comment)
-      @values["description"] = d unless d.empty?
-
-      ml = build_method_summary_list
-      @values["methods"] = ml unless ml.empty?
-
-      il = build_include_list(@context)
-      @values["includes"] = il unless il.empty?
-
-      @values["sections"] = @context.sections.map do |section|
-
-        secdata = {
-          "sectitle" => section.title,
-          "secsequence" => section.sequence,
-          "seccomment" => markup(section.comment)
-        }
-
-        al = build_alias_summary_list(section)
-        secdata["aliases"] = al unless al.empty?
-        
-        co = build_constants_summary_list(section)
-        secdata["constants"] = co unless co.empty?
-        
-        al = build_attribute_list(section)
-        secdata["attributes"] = al unless al.empty?
-        
-        cl = build_class_list(0, @context, section)
-        secdata["classlist"] = cl unless cl.empty?
-        
-        mdl = build_method_detail_list(section)
-        secdata["method_list"] = mdl unless mdl.empty?
-
-        secdata
-      end
-
-      @values
-    end
-
-    def build_attribute_list(section)
-      atts = @context.attributes.sort
-      res = []
-      atts.each do |att|
-        next unless att.section == section
-        if att.visibility == :public || @options.show_all
-          entry = {
-            "name"   => CGI.escapeHTML(att.name), 
-            "rw"     => att.rw, 
-            "a_desc" => markup(att.comment, true)
-          }
-          unless att.visibility == :public
-            entry["rw"] << "-"
-          end
-          res << entry
-        end
-      end
-      res
-    end
-
-    def class_attribute_values
-      h_name = CGI.escapeHTML(name)
-
-      @values["classmod"]  = @is_module ? "Module" : "Class"
-      @values["title"]     = "#{@values['classmod']}: #{h_name}"
-
-      c = @context
-      c = c.parent while c and !c.diagram
-      if c && c.diagram
-        @values["diagram"] = diagram_reference(c.diagram)
-      end
-
-      @values["full_name"] = h_name
-
-      parent_class = @context.superclass
-
-      if parent_class
-        @values["parent"] = CGI.escapeHTML(parent_class)
-
-        if parent_name
-          lookup = parent_name + "::" + parent_class
-        else
-          lookup = parent_class
-        end
-
-        parent_url = AllReferences[lookup] || AllReferences[parent_class]
-
-        if parent_url and parent_url.document_self
-          @values["par_url"] = aref_to(parent_url.path)
-        end
-      end
-
-      files = []
-      @context.in_files.each do |f|
-        res = {}
-        full_path = CGI.escapeHTML(f.file_absolute_name)
-
-        res["full_path"]     = full_path
-        res["full_path_url"] = aref_to(f.viewer.path) if f.document_self
-
-        if @options.webcvs
-          res["cvsurl"] = cvs_url( @options.webcvs, full_path )
-        end
-
-        files << res
-      end
-
-      @values['infiles'] = files
-    end
-
-    def <=>(other)
-      self.name <=> other.name
-    end
-
-  end
-
-  #####################################################################
-  #
-  # Handles the mapping of a file's information to HTML. In reality,
-  # a file corresponds to a +TopLevel+ object, containing modules,
-  # classes, and top-level methods. In theory it _could_ contain
-  # attributes and aliases, but we ignore these for now.
-
-  class HtmlFile < ContextUser
-
-    attr_reader :path
-    attr_reader :name
-
-    def initialize(context, options, file_dir)
-      super(context, options)
-
-      @values = {}
-
-      if options.all_one_file
-        @path = filename_to_label
-      else
-        @path = http_url(file_dir)
-      end
-
-      @name = @context.file_relative_name
-
-      collect_methods
-      AllReferences.add(name, self)
-      context.viewer = self
-    end
-
-    def http_url(file_dir)
-      File.join(file_dir, @context.file_relative_name.tr('.', '_')) +
-        ".html"
-    end
-
-    def filename_to_label
-      @context.file_relative_name.gsub(/%|\/|\?|\#/) {|s| '%' + ("%x" % s[0]) }
-    end
-
-    def index_name
-      name
-    end
-
-    def parent_name
-      nil
-    end
-
-    def value_hash
-      file_attribute_values
-      add_table_of_sections
-
-      @values["charset"]   = @options.charset
-      @values["href"]      = path
-      @values["style_url"] = style_url(path, @options.css)
-
-      if @context.comment
-        d = markup(@context.comment)
-        @values["description"] = d if d.size > 0
-      end
-
-      ml = build_method_summary_list
-      @values["methods"] = ml unless ml.empty?
-
-      il = build_include_list(@context)
-      @values["includes"] = il unless il.empty?
-
-      rl = build_requires_list(@context)
-      @values["requires"] = rl unless rl.empty?
-
-      if @options.promiscuous
-        file_context = nil
-      else
-        file_context = @context
-      end
-
-
-      @values["sections"] = @context.sections.map do |section|
-
-        secdata = {
-          "sectitle" => section.title,
-          "secsequence" => section.sequence,
-          "seccomment" => markup(section.comment)
-        }
-
-        cl = build_class_list(0, @context, section, file_context)
-        @values["classlist"] = cl unless cl.empty?
-
-        mdl = build_method_detail_list(section)
-        secdata["method_list"] = mdl unless mdl.empty?
-
-        al = build_alias_summary_list(section)
-        secdata["aliases"] = al unless al.empty?
-        
-        co = build_constants_summary_list(section)
-        @values["constants"] = co unless co.empty?
-
-        secdata
-      end
-      
-      @values
-    end
-    
-    def write_on(f)
-      value_hash
-      template = TemplatePage.new(RDoc::Page::BODY,
-                                  RDoc::Page::FILE_PAGE,
-                                  RDoc::Page::METHOD_LIST)
-      template.write_html_on(f, @values)
-    end
-
-    def file_attribute_values
-      full_path = @context.file_absolute_name
-      short_name = File.basename(full_path)
-      
-      @values["title"] = CGI.escapeHTML("File: #{short_name}")
-
-      if @context.diagram
-        @values["diagram"] = diagram_reference(@context.diagram)
-      end
-
-      @values["short_name"]   = CGI.escapeHTML(short_name)
-      @values["full_path"]    = CGI.escapeHTML(full_path)
-      @values["dtm_modified"] = @context.file_stat.mtime.to_s
-
-      if @options.webcvs
-        @values["cvsurl"] = cvs_url( @options.webcvs, @values["full_path"] )
-      end
-    end
-
-    def <=>(other)
-      self.name <=> other.name
-    end
-  end
-
-  #####################################################################
-
-  class HtmlMethod
-    include MarkUp
-
-    attr_reader :context
-    attr_reader :src_url
-    attr_reader :img_url
-    attr_reader :source_code
-
-    @@seq = "M000000"
-
-    @@all_methods = []
-
-    def HtmlMethod::reset
-      @@all_methods = []
-    end
-
-    def initialize(context, html_class, options)
-      @context    = context
-      @html_class = html_class
-      @options    = options
-      @@seq       = @@seq.succ
-      @seq        = @@seq
-      @@all_methods << self
-
-      context.viewer = self
-
-      if (ts = @context.token_stream)
-        @source_code = markup_code(ts)
-        unless @options.inline_source
-          @src_url = create_source_code_file(@source_code)
-          @img_url = HTMLGenerator.gen_url(path, 'source.png')
-        end
-      end
-
-      AllReferences.add(name, self)
-    end
-    
-    # return a reference to outselves to be used as an href=
-    # the form depends on whether we're all in one file
-    # or in multiple files
-
-    def as_href(from_path)
-      if @options.all_one_file
-        "#" + path
-      else
-        HTMLGenerator.gen_url(from_path, path)
-      end
-    end
-
-    def name
-      @context.name
-    end
-
-    def section
-      @context.section
-    end
-
-    def index_name
-      "#{@context.name} (#{@html_class.name})"
-    end
-
-    def parent_name
-      if @context.parent.parent
-        @context.parent.parent.full_name
-      else
-        nil
-      end
-    end
-
-    def aref
-      @seq
-    end
-
-    def path
-      if @options.all_one_file
-        aref
-      else
-        @html_class.path + "#" + aref
-      end
-    end
-
-    def description
-      markup(@context.comment)
-    end
-
-    def visibility
-      @context.visibility
-    end
-
-    def singleton
-      @context.singleton
-    end
-
-    def call_seq
-      cs = @context.call_seq
-      if cs
-        cs.gsub(/\n/, "<br />\n")
-      else
-        nil
-      end
-    end
-
-    def params
-      # params coming from a call-seq in 'C' will start with the
-      # method name
-      p = @context.params
-      if p !~ /^\w/
-        p = @context.params.gsub(/\s*\#.*/, '')
-        p = p.tr("\n", " ").squeeze(" ")
-        p = "(" + p + ")" unless p[0] == ?(
-        
-        if (block = @context.block_params)
-         # If this method has explicit block parameters, remove any
-         # explicit &block
-
-         p.sub!(/,?\s*&\w+/, '')
-
-          block.gsub!(/\s*\#.*/, '')
-          block = block.tr("\n", " ").squeeze(" ")
-          if block[0] == ?(
-            block.sub!(/^\(/, '').sub!(/\)/, '')
-          end
-          p << " {|#{block.strip}| ...}"
-        end
-      end
-      CGI.escapeHTML(p)
-    end
-    
-    def create_source_code_file(code_body)
-      meth_path = @html_class.path.sub(/\.html$/, '.src')
-      File.makedirs(meth_path)
-      file_path = File.join(meth_path, @seq) + ".html"
-
-      template = TemplatePage.new(RDoc::Page::SRC_PAGE)
-      File.open(file_path, "w") do |f|
-        values = {
-          'title'     => CGI.escapeHTML(index_name),
-          'code'      => code_body,
-          'style_url' => style_url(file_path, @options.css),
-          'charset'   => @options.charset
-        }
-        template.write_html_on(f, values)
-      end
-      HTMLGenerator.gen_url(path, file_path)
-    end
-
-    def HtmlMethod.all_methods
-      @@all_methods
-    end
-
-    def <=>(other)
-      @context <=> other.context
-    end
-
-    ##
-    # Given a sequence of source tokens, mark up the source code
-    # to make it look purty.
-    def old_markup_code(tokens)
-      src = ""
-      tokens.each do |t|
-        next unless t
-        #    p t.class
-#        style = STYLE_MAP[t.class]
-        style = case t
-                when RubyToken::TkCONSTANT then "ruby-constant"
-                when RubyToken::TkKW       then "ruby-keyword kw"
-                when RubyToken::TkIVAR     then "ruby-ivar"
-                when RubyToken::TkOp       then "ruby-operator"
-                when RubyToken::TkId       then "ruby-identifier"
-                when RubyToken::TkNode     then "ruby-node"
-                when RubyToken::TkCOMMENT  then "ruby-comment cmt"
-                when RubyToken::TkREGEXP   then "ruby-regexp re"
-                when RubyToken::TkSTRING   then "ruby-value str"
-                when RubyToken::TkVal      then "ruby-value"
-                else
-                    nil
-                end
-
-        text = CGI.escapeHTML(t.text)
-
-        if style
-          src << "<span class=\"#{style}\">#{text}</span>"
-        else
-          src << text
-        end
-      end
-
-      add_line_numbers(src) if Options.instance.include_line_numbers
-      src
-    end
-
-    require 'coderay'
-    CodeRay::Scanners.load_all
-    CodeRay::Encoders.load_all
-    CodeRay::Styles.load_all
-
-    def markup_code(tokens)
-      code = tokens.map { |t| t.text }.join
-      options = {
-        :css => :class,
-        :line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1,
-        :bold_every => :no_bolding,
-      }
-      options[:line_numbers] = nil unless Options.instance.include_line_numbers
-      CodeRay.scan(code, :ruby).div(options)
-    end
-
-    # we rely on the fact that the first line of a source code
-    # listing has 
-    #    # File xxxxx, line dddd
-
-    def add_line_numbers(src)
-      if src =~ /\A.*, line (\d+)/
-        first = $1.to_i - 1
-        last  = first + src.count("\n")
-        size = last.to_s.length
-        real_fmt = "%#{size}d: "
-        fmt = " " * (size+2)
-        src.gsub!(/^/) do
-          res = sprintf(fmt, first) 
-          first += 1
-          fmt = real_fmt
-          res
-        end
-      end
-    end
-
-    def document_self
-      @context.document_self
-    end
-
-    def aliases
-      @context.aliases
-    end
-
-    def find_symbol(symbol, method=nil)
-      res = @context.parent.find_symbol(symbol, method)
-      if res
-        res = res.viewer
-      end
-      res
-    end
-  end
-
-  #####################################################################
-        
-  class HTMLGenerator
-
-    include MarkUp
-
-    ##
-    # convert a target url to one that is relative to a given
-    # path
-    
-    def HTMLGenerator.gen_url(path, target)
-      from          = File.dirname(path)
-      to, to_file   = File.split(target)
-      
-      from = from.split("/")
-      to   = to.split("/")
-      
-      while from.size > 0 and to.size > 0 and from[0] == to[0]
-        from.shift
-        to.shift
-      end
-      
-      from.fill("..")
-      from.concat(to)
-      from << to_file
-      File.join(*from)
-    end
-
-    # Generators may need to return specific subclasses depending
-    # on the options they are passed. Because of this
-    # we create them using a factory
-
-    def HTMLGenerator.for(options)
-      AllReferences::reset
-      HtmlMethod::reset
-
-      if options.all_one_file
-        HTMLGeneratorInOne.new(options)
-      else
-        HTMLGenerator.new(options)
-      end
-    end
-
-    class <<self
-      protected :new
-    end
-
-    # Set up a new HTML generator. Basically all we do here is load
-    # up the correct output temlate
-
-    def initialize(options) #:not-new:
-      @options    = options
-      load_html_template
-    end
-
-
-    ##
-    # Build the initial indices and output objects
-    # based on an array of TopLevel objects containing
-    # the extracted information. 
-
-    def generate(toplevels)
-      @toplevels  = toplevels
-      @files      = []
-      @classes    = []
-
-      write_style_sheet
-      gen_sub_directories()
-      build_indices
-      generate_html
-    end
-
-    private
-
-    ##
-    # Load up the HTML template specified in the options.
-    # If the template name contains a slash, use it literally
-    #
-    def load_html_template
-      template = @options.template
-      unless template =~ %r{/|\\}
-        template = File.join("rdoc/generators/template",
-                             @options.generator.key, template)
-      end
-      require template
-      extend RDoc::Page
-    rescue LoadError
-      $stderr.puts "Could not find HTML template '#{template}'"
-      exit 99
-    end
-
-    ##
-    # Write out the style sheet used by the main frames
-    #
-    
-    def write_style_sheet
-      template = TemplatePage.new(RDoc::Page::STYLE)
-      unless @options.css
-        File.open(CSS_NAME, "w") do |f|
-          values = { "fonts" => RDoc::Page::FONTS }
-          template.write_html_on(f, values)
-        end
-      end
-    end
-
-    ##
-    # See the comments at the top for a description of the
-    # directory structure
-
-    def gen_sub_directories
-      File.makedirs(FILE_DIR, CLASS_DIR)
-    rescue 
-      $stderr.puts $!.message
-      exit 1
-    end
-
-    ##
-    # Generate:
-    #
-    # * a list of HtmlFile objects for each TopLevel object.
-    # * a list of HtmlClass objects for each first level
-    #   class or module in the TopLevel objects
-    # * a complete list of all hyperlinkable terms (file,
-    #   class, module, and method names)
-
-    def build_indices
-
-      @toplevels.each do |toplevel|
-        @files << HtmlFile.new(toplevel, @options, FILE_DIR)
-      end
-
-      RDoc::TopLevel.all_classes_and_modules.each do |cls|
-        build_class_list(cls, @files[0], CLASS_DIR)
-      end
-    end
-
-    def build_class_list(from, html_file, class_dir)
-      @classes << HtmlClass.new(from, html_file, class_dir, @options)
-      from.each_classmodule do |mod|
-        build_class_list(mod, html_file, class_dir)
-      end
-    end
-
-    ##
-    # Generate all the HTML
-    #
-    def generate_html
-      # the individual descriptions for files and classes
-      gen_into(@files)
-      gen_into(@classes)
-      # and the index files
-      gen_file_index
-      gen_class_index
-      gen_method_index
-      gen_main_index
-      
-      # this method is defined in the template file
-      write_extra_pages if defined? write_extra_pages
-    end
-
-    def gen_into(list)
-      list.each do |item|
-        if item.document_self
-          op_file = item.path
-          File.makedirs(File.dirname(op_file))
-          File.open(op_file, "w") { |file| item.write_on(file) }
-        end
-      end
-
-    end
-
-    def gen_file_index
-      gen_an_index(@files, 'Files', 
-                   RDoc::Page::FILE_INDEX, 
-                   "fr_file_index.html")
-    end
-
-    def gen_class_index
-      gen_an_index(@classes, 'Classes',
-                   RDoc::Page::CLASS_INDEX,
-                   "fr_class_index.html")
-    end
-
-    def gen_method_index
-      gen_an_index(HtmlMethod.all_methods, 'Methods', 
-                   RDoc::Page::METHOD_INDEX,
-                   "fr_method_index.html")
-    end
-
-    
-    def gen_an_index(collection, title, template, filename)
-      template = TemplatePage.new(RDoc::Page::FR_INDEX_BODY, template)
-      res = []
-      collection.sort.each do |f|
-        if f.document_self
-          res << { "href" => f.path, "name" => f.index_name }
-        end
-      end
-
-      values = {
-        "entries"    => res,
-        'list_title' => CGI.escapeHTML(title),
-        'index_url'  => main_url,
-        'charset'    => @options.charset,
-        'style_url'  => style_url('', @options.css),
-      }
-
-      File.open(filename, "w") do |f|
-        template.write_html_on(f, values)
-      end
-    end
-
-    # The main index page is mostly a template frameset, but includes
-    # the initial page. If the <tt>--main</tt> option was given,
-    # we use this as our main page, otherwise we use the
-    # first file specified on the command line.
-
-    def gen_main_index
-      template = TemplatePage.new(RDoc::Page::INDEX)
-      File.open("index.html", "w") do |f|
-        values = {
-          "initial_page" => main_url,
-          'title'        => CGI.escapeHTML(@options.title),
-          'charset'      => @options.charset
-        }
-        if @options.inline_source
-          values['inline_source'] = true
-        end
-        template.write_html_on(f, values)
-      end
-    end
-
-    # return the url of the main page
-    def main_url
-      main_page = @options.main_page
-      ref = nil
-      if main_page
-        ref = AllReferences[main_page]
-        if ref
-          ref = ref.path
-        else
-          $stderr.puts "Could not find main page #{main_page}"
-        end
-      end
-
-      unless ref
-        for file in @files
-          if file.document_self
-            ref = file.path 
-            break
-          end
-        end
-      end
-
-      unless ref
-        $stderr.puts "Couldn't find anything to document"
-        $stderr.puts "Perhaps you've used :stopdoc: in all classes"
-        exit(1)
-      end
-
-      ref
-    end
-
-
-  end
-  HTML_CODERAYGenerator = HTMLGenerator
-
-
-  ######################################################################
-
-
-  class HTMLGeneratorInOne < HTMLGenerator
-
-    def initialize(*args)
-      super
-    end
-
-    ##
-    # Build the initial indices and output objects
-    # based on an array of TopLevel objects containing
-    # the extracted information. 
-
-    def generate(info)
-      @toplevels  = info
-      @files      = []
-      @classes    = []
-      @hyperlinks = {}
-
-      build_indices
-      generate_xml
-    end
-
-
-    ##
-    # Generate:
-    #
-    # * a list of HtmlFile objects for each TopLevel object.
-    # * a list of HtmlClass objects for each first level
-    #   class or module in the TopLevel objects
-    # * a complete list of all hyperlinkable terms (file,
-    #   class, module, and method names)
-
-    def build_indices
-
-      @toplevels.each do |toplevel|
-        @files << HtmlFile.new(toplevel, @options, FILE_DIR)
-      end
-
-      RDoc::TopLevel.all_classes_and_modules.each do |cls|
-        build_class_list(cls, @files[0], CLASS_DIR)
-      end
-    end
-
-    def build_class_list(from, html_file, class_dir)
-      @classes << HtmlClass.new(from, html_file, class_dir, @options)
-      from.each_classmodule do |mod|
-        build_class_list(mod, html_file, class_dir)
-      end
-    end
-
-    ##
-    # Generate all the HTML. For the one-file case, we generate
-    # all the information in to one big hash
-    #
-    def generate_xml
-      values = { 
-        'charset' => @options.charset,
-        'files'   => gen_into(@files),
-        'classes' => gen_into(@classes),
-        'title'        => CGI.escapeHTML(@options.title),
-      }
-      
-      # this method is defined in the template file
-      write_extra_pages if defined? write_extra_pages
-
-      template = TemplatePage.new(RDoc::Page::ONE_PAGE)
-
-      if @options.op_name
-        opfile = File.open(@options.op_name, "w")
-      else
-        opfile = $stdout
-      end
-      template.write_html_on(opfile, values)
-    end
-
-    def gen_into(list)
-      res = []
-      list.each do |item|
-        res << item.value_hash
-      end
-      res
-    end
-
-    def gen_file_index
-      gen_an_index(@files, 'Files')
-    end
-
-    def gen_class_index
-      gen_an_index(@classes, 'Classes')
-    end
-
-    def gen_method_index
-      gen_an_index(HtmlMethod.all_methods, 'Methods')
-    end
-
-    
-    def gen_an_index(collection, title)
-      res = []
-      collection.sort.each do |f|
-        if f.document_self
-          res << { "href" => f.path, "name" => f.index_name }
-        end
-      end
-
-      return {
-        "entries" => res,
-        'list_title' => title,
-        'index_url'  => main_url,
-      }
-    end
-
-  end
-end
diff --git a/rake_tasks/bundler.rake b/rake_tasks/bundler.rake
deleted file mode 100644
index 8de149d..0000000
--- a/rake_tasks/bundler.rake
+++ /dev/null
@@ -1,6 +0,0 @@
-begin
-  require 'bundler'
-  Bundler::GemHelper.install_tasks
-rescue LoadError
-  puts 'Please gem install bundler.'
-end
diff --git a/rake_helpers/code_statistics.rb b/rake_tasks/code_statistics.rb
index 0a2016b..0a2016b 100644
--- a/rake_helpers/code_statistics.rb
+++ b/rake_tasks/code_statistics.rb
diff --git a/rake_tasks/diff.rake b/rake_tasks/diff.rake
deleted file mode 100644
index f0af55a..0000000
--- a/rake_tasks/diff.rake
+++ /dev/null
@@ -1,93 +0,0 @@
-# A simple differ using svn. Handles externals.
-class Differ < Hash
-  
-  include Rake::DSL if defined? Rake::DSL
-  
-  def initialize path
-    @path = path
-    super 0
-  end
-  
-  def count key, value
-    self[key] += value
-    value
-  end
-  
-  FORMAT = '  %-30s %8d lines, %3d changes in %2d files'
-  
-  def scan(path = @path)
-    Dir.chdir path do
-      diff_file_name = 'diff'
-      if File.directory? 'diff'
-        diff_file_name = 'diff.diff'
-      end
-      system "svn diff > #{diff_file_name}"
-      if File.size? diff_file_name
-        puts FORMAT %
-          [
-            path,
-            count(:LOC, `wc -l #{diff_file_name}`.to_i),
-            count(:changes, `grep ^@@ #{diff_file_name} | wc -l`.to_i),
-            count(:files, `grep ^Index #{diff_file_name} | wc -l`.to_i),
-          ]
-      else
-        rm diff_file_name
-      end
-    end
-  end
-  
-  def scan_with_externals(path = @path)
-    scan path
-    `svn status`.scan(/^X\s*(.*)/) do |external,|
-      scan external
-    end
-  end
-  
-  def clean(path = @path)
-    Dir.chdir path do
-      rm 'diff' if File.file? 'diff'
-      rm 'diff.diff' if File.file? 'diff.diff'
-    end
-  end
-  
-  def clean_with_externals(path = @path)
-    clean path
-    `svn status`.scan(/^X\s*(.*)/) do |external,|
-      clean external
-    end
-  end
-  
-  def differences?
-    self[:LOC] > 0
-  end
-  
-  def inspect
-    FORMAT %
-      [ 'Total', self[:LOC], self[:changes], self[:files] ]
-  end
-  
-end
-
-namespace :diff do
-  
-  desc 'Make a diff and print a summary'
-  task :summary do
-    differ = Differ.new '.'
-    differ.scan_with_externals
-    if differ.empty?
-      puts 'No differences found.'
-    else
-      p differ
-    end
-  end
-  
-  desc 'Remove all diffs'
-  task :clean do
-    differ = Differ.new '.'
-    differ.clean_with_externals
-  end
-  
-end
-
-desc 'Make a diff and print a summary'
-task :diff => 'diff:summary'
diff --git a/rake_tasks/documentation.rake b/rake_tasks/documentation.rake
index d555022..4f7cef7 100644
--- a/rake_tasks/documentation.rake
+++ b/rake_tasks/documentation.rake
@@ -15,18 +15,9 @@ Rake::RDocTask.new :doc do |rd|
   rd.title = 'CodeRay Documentation'
   
   rd.options << '--line-numbers' << '--tab-width' << '2'
-  # rd.options << '--fmt' << ENV.fetch('format', 'html_coderay')
-  # require 'pathname'
-  # template = File.join ROOT, 'rake_helpers', 'coderay_rdoc_template.rb'
-  # rd.template = Pathname.new(template).expand_path.to_s
   
   rd.main = 'README_INDEX.rdoc'
   rd.rdoc_files.add 'README_INDEX.rdoc'
   rd.rdoc_files.add Dir['lib']
   rd.rdoc_dir = 'doc'
 end if defined? Rake::RDocTask
-
-desc 'Copy the documentation over to the CodeRay website'
-task :copy_doc do
-  cp_r 'doc/.', '../../rails/coderay/public/doc'
-end
diff --git a/rake_tasks/ruby-versions.rake b/rake_tasks/ruby-versions.rake
deleted file mode 100644
index af408ff..0000000
--- a/rake_tasks/ruby-versions.rake
+++ /dev/null
@@ -1,10 +0,0 @@
-task 'ruby:version' do
-  puts
-  if defined? RUBY_DESCRIPTION
-    ruby_version = RUBY_DESCRIPTION
-  else
-    ruby_version = "ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE} patchlevel #{RUBY_PATCHLEVEL}) [#{RUBY_PLATFORM}]"
-  end
-  require './test/lib/term/ansicolor'
-  puts Term::ANSIColor.bold(Term::ANSIColor.green(ruby_version))
-end
-\ No newline at end of file
diff --git a/rake_tasks/statistic.rake b/rake_tasks/statistic.rake
index 99de378..d30e9b1 100644
--- a/rake_tasks/statistic.rake
+++ b/rake_tasks/statistic.rake
@@ -1,6 +1,6 @@
 desc 'Report code statistics (LOC) from the application'
 task :stats do
-  require 'rake_helpers/code_statistics'
+  require './rake_tasks/code_statistics'
   CodeStatistics.new(
     ['Main', 'lib', /coderay.rb$/],
     ['CodeRay', 'lib/coderay/'],
diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake
index a60699d..371214a 100644
--- a/rake_tasks/test.rake
+++ b/rake_tasks/test.rake
@@ -1,7 +1,7 @@
 namespace :test do
   desc 'run all sample tests'
   task :samples do
-    ruby './sample/suite.rb'
+    ruby './test/samples/suite.rb'
   end
   
   desc 'run functional tests'
diff --git a/sample/cache.expected b/sample/cache.expected
deleted file mode 100644
index f815e88..0000000
--- a/sample/cache.expected
+++ /dev/null
@@ -1,2 +0,0 @@
-test &lt;test&gt;
-test <span class="ta">&lt;test&gt;</span>
diff --git a/sample/css.expected b/sample/css.expected
deleted file mode 100644
index 09709ff..0000000
--- a/sample/css.expected
+++ /dev/null
@@ -1,130 +0,0 @@
-.CodeRay {
-  background-color: #f8f8f8;
-  border: 1px solid silver;
-  font-family: 'Courier New', 'Terminal', monospace;
-  color: #000;
-}
-.CodeRay pre { margin: 0px }
-
-div.CodeRay { }
-
-span.CodeRay { white-space: pre; border: 0px; padding: 2px }
-
-table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
-table.CodeRay td { padding: 2px 4px; vertical-align: top }
-
-.CodeRay .line_numbers, .CodeRay .no {
-  background-color: #def;
-  color: gray;
-  text-align: right;
-}
-.CodeRay .line_numbers tt { font-weight: bold }
-.CodeRay .line_numbers .highlighted { color: red }
-.CodeRay .no { padding: 0px 4px }
-.CodeRay .code { width: 100% }
-
-ol.CodeRay { font-size: 10pt }
-ol.CodeRay li { white-space: pre }
-
-.CodeRay .code pre { overflow: auto }
-
-.CodeRay .debug { color:white ! important; background:blue ! important; }
-
-.CodeRay .af { color:#00C }
-.CodeRay .an { color:#007 }
-.CodeRay .at { color:#f08 }
-.CodeRay .av { color:#700 }
-.CodeRay .aw { color:#C00 }
-.CodeRay .bi { color:#509; font-weight:bold }
-.CodeRay .c  { color:#888; }
-
-.CodeRay .ch { color:#04D }
-.CodeRay .ch .k { color:#04D }
-.CodeRay .ch .dl { color:#039 }
-
-.CodeRay .cl { color:#B06; font-weight:bold }
-.CodeRay .cm { color:#A08; font-weight:bold }
-.CodeRay .co { color:#036; font-weight:bold }
-.CodeRay .cr { color:#0A0 }
-.CodeRay .cv { color:#369 }
-.CodeRay .de { color:#B0B; }
-.CodeRay .df { color:#099; font-weight:bold }
-.CodeRay .di { color:#088; font-weight:bold }
-.CodeRay .dl { color:black }
-.CodeRay .do { color:#970 }
-.CodeRay .dt { color:#34b }
-.CodeRay .ds { color:#D42; font-weight:bold }
-.CodeRay .e  { color:#666; font-weight:bold }
-.CodeRay .en { color:#800; font-weight:bold }
-.CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#C00; font-weight:bold }
-.CodeRay .fl { color:#60E; font-weight:bold }
-.CodeRay .fu { color:#06B; font-weight:bold }
-.CodeRay .gv { color:#d70; font-weight:bold }
-.CodeRay .hx { color:#058; font-weight:bold }
-.CodeRay .i  { color:#00D; font-weight:bold }
-.CodeRay .ic { color:#B44; font-weight:bold }
-
-.CodeRay .il { background: #ddd; color: black }
-.CodeRay .il .il { background: #ccc }
-.CodeRay .il .il .il { background: #bbb }
-.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
-.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
-
-.CodeRay .im { color:#f00; }
-.CodeRay .in { color:#B2B; font-weight:bold }
-.CodeRay .iv { color:#33B }
-.CodeRay .la { color:#970; font-weight:bold }
-.CodeRay .lv { color:#963 }
-.CodeRay .oc { color:#40E; font-weight:bold }
-.CodeRay .of { color:#000; font-weight:bold }
-.CodeRay .op { }
-.CodeRay .pc { color:#038; font-weight:bold }
-.CodeRay .pd { color:#369; font-weight:bold }
-.CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight:bold }
-.CodeRay .pt { color:#074; font-weight:bold }
-.CodeRay .r, .kw  { color:#080; font-weight:bold }
-
-.CodeRay .ke { color: #808; }
-.CodeRay .ke .dl { color: #606; }
-.CodeRay .ke .ch { color: #80f; }
-.CodeRay .vl { color: #088; }
-
-.CodeRay .rx { background-color:#fff0ff }
-.CodeRay .rx .k { color:#808 }
-.CodeRay .rx .dl { color:#404 }
-.CodeRay .rx .mod { color:#C2C }
-.CodeRay .rx .fu  { color:#404; font-weight: bold }
-
-.CodeRay .s { background-color:#fff0f0; color: #D20; }
-.CodeRay .s .s { background-color:#ffe0e0 }
-.CodeRay .s .s  .s { background-color:#ffd0d0 }
-.CodeRay .s .k { }
-.CodeRay .s .ch { color: #b0b; }
-.CodeRay .s .dl { color: #710; }
-
-.CodeRay .sh { background-color:#f0fff0; color:#2B2 }
-.CodeRay .sh .k { }
-.CodeRay .sh .dl { color:#161 }
-
-.CodeRay .sy { color:#A60 }
-.CodeRay .sy .k { color:#A60 }
-.CodeRay .sy .dl { color:#630 }
-
-.CodeRay .ta { color:#070 }
-.CodeRay .tf { color:#070; font-weight:bold }
-.CodeRay .ts { color:#D70; font-weight:bold }
-.CodeRay .ty { color:#339; font-weight:bold }
-.CodeRay .v  { color:#036 }
-.CodeRay .xt { color:#444 }
-
-.CodeRay .ins { background: #afa; }
-.CodeRay .del { background: #faa; }
-.CodeRay .chg { color: #aaf; background: #007; }
-.CodeRay .head { color: #f8f; background: #505 }
-
-.CodeRay .ins .ins { color: #080; font-weight:bold }
-.CodeRay .del .del { color: #800; font-weight:bold }
-.CodeRay .chg .chg { color: #66f; }
-.CodeRay .head .head { color: #f4f; }
diff --git a/sample/div.expected b/sample/div.expected
deleted file mode 100644
index f28ede3..0000000
--- a/sample/div.expected
+++ /dev/null
@@ -1,17 +0,0 @@
-<div class="CodeRay">
-  <div class="code"><pre><span style="color:#080;font-weight:bold">for</span> a <span style="color:#080;font-weight:bold">in</span> <span style="color:#00D;font-weight:bold">0</span>..<span style="color:#00D;font-weight:bold">255</span>
-        a = a.chr
-        <span style="color:#080;font-weight:bold">begin</span>
-                x = eval(<span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="">?</span><span style="color:#b0b">\\</span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>a<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>)
-                <span style="color:#080;font-weight:bold">if</span> x == a[<span style="color:#00D;font-weight:bold">0</span>]
-                        <span style="color:#080;font-weight:bold">next</span>
-                <span style="color:#080;font-weight:bold">else</span>
-                        print <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>a<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="">: </span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>x<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>
-                <span style="color:#080;font-weight:bold">end</span>
-        <span style="color:#080;font-weight:bold">rescue</span> <span style="color:#036;font-weight:bold">SyntaxError</span> =&gt; boom
-                print <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>a<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="">: error</span><span style="color:#710">&quot;</span></span>
-        <span style="color:#080;font-weight:bold">end</span>
-        puts
-<span style="color:#080;font-weight:bold">end</span>
-</pre></div>
-</div>
diff --git a/sample/dump.expected b/sample/dump.expected
deleted file mode 100644
index a451686..0000000
--- a/sample/dump.expected
+++ /dev/null
@@ -1,21 +0,0 @@
-YAML: 2358 bytes
-Dump: 1109 bytes
-undumped:
-<div class="CodeRay">
-  <div class="code"><pre>require <span class="s"><span class="dl">'</span><span class="k">coderay</span><span class="dl">'</span></span>
-
-<span class="c"># scan some code</span>
-tokens = <span class="co">CodeRay</span>.scan(<span class="co">File</span>.read(<span class="gv">$0</span>), <span class="sy">:ruby</span>)
-
-<span class="c"># dump using YAML</span>
-yaml = tokens.yaml
-puts <span class="s"><span class="dl">'</span><span class="k">YAML: %4d bytes</span><span class="dl">'</span></span> % yaml.size
-
-<span class="c"># dump using Marshal</span>
-dump = tokens.dump(<span class="i">0</span>)
-puts <span class="s"><span class="dl">'</span><span class="k">Dump: %4d bytes</span><span class="dl">'</span></span> % dump.size
-
-<span class="c"># undump and encode</span>
-puts <span class="s"><span class="dl">'</span><span class="k">undumped:</span><span class="dl">'</span></span>, dump.undump.div(<span class="sy">:css</span> =&gt; <span class="sy">:class</span>)
-</pre></div>
-</div>
diff --git a/sample/dump.rb b/sample/dump.rb
deleted file mode 100644
index cd68dc8..0000000
--- a/sample/dump.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-require 'coderay'
-
-# scan some code
-tokens = CodeRay.scan(File.read($0), :ruby)
-
-# dump using YAML
-yaml = tokens.yaml
-puts 'YAML: %4d bytes' % yaml.size
-
-# dump using Marshal
-dump = tokens.dump(0)
-puts 'Dump: %4d bytes' % dump.size
-
-# undump and encode
-puts 'undumped:', dump.undump.div(:css => :class)
diff --git a/test/executable/suite.rb b/test/executable/suite.rb
index d386f4b..997405c 100644
--- a/test/executable/suite.rb
+++ b/test/executable/suite.rb
@@ -14,7 +14,7 @@ class TestCodeRayExecutable < Test::Unit::TestCase
   
   ROOT_DIR = Pathname.new(File.dirname(__FILE__)) + '..' + '..'
   EXECUTABLE = ROOT_DIR + 'bin' + 'coderay'
-  RUBY_COMMAND = RUBY_VERSION < '2.0.0' ? 'ruby -w' : 'ruby'  # Ruby 2 currently throws warnings for bundler
+  RUBY_COMMAND = 'ruby'
   EXE_COMMAND =
     if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success?
       # use Nailgun
diff --git a/test/functional/examples.rb b/test/functional/examples.rb
index 15f9ca3..985ef87 100755
--- a/test/functional/examples.rb
+++ b/test/functional/examples.rb
@@ -114,7 +114,7 @@ Token Types (7):
     
     # format the tokens
     term = terminal_encoder.encode_tokens(tokens)
-    assert_equal "\e[1;31mimport\e[0m \e[33mthis\e[0m;  \e[37m# The Zen of Python\e[0m", term
+    assert_equal "\e[32mimport\e[0m \e[31mthis\e[0m;  \e[1;30m# The Zen of Python\e[0m", term
     
     # re-using scanner and encoder
     ruby_highlighter = CodeRay::Duo[:ruby, :div]
diff --git a/sample/README b/test/samples/README
index b0ab6e4..b0ab6e4 100644
--- a/sample/README
+++ b/test/samples/README
diff --git a/test/samples/cache.actual b/test/samples/cache.actual
new file mode 100644
index 0000000..c131857
--- /dev/null
+++ b/test/samples/cache.actual
@@ -0,0 +1,2 @@
+test &lt;test&gt;
+test <span class="tag">&lt;test&gt;</span>
diff --git a/test/samples/cache.expected b/test/samples/cache.expected
new file mode 100644
index 0000000..c131857
--- /dev/null
+++ b/test/samples/cache.expected
@@ -0,0 +1,2 @@
+test &lt;test&gt;
+test <span class="tag">&lt;test&gt;</span>
diff --git a/sample/cache.rb b/test/samples/cache.rb
index 0c0b847..0c0b847 100644
--- a/sample/cache.rb
+++ b/test/samples/cache.rb
diff --git a/sample/count.expected b/test/samples/count.expected
index 7f493b6..7f493b6 100644
--- a/sample/count.expected
+++ b/test/samples/count.expected
diff --git a/sample/count.rb b/test/samples/count.rb
index bcb7c2d..bcb7c2d 100644
--- a/sample/count.rb
+++ b/test/samples/count.rb
diff --git a/test/samples/css.actual b/test/samples/css.actual
new file mode 100644
index 0000000..be73a7f
--- /dev/null
+++ b/test/samples/css.actual
@@ -0,0 +1,127 @@
+.CodeRay {
+  background-color: hsl(0,0%,95%);
+  border: 1px solid silver;
+  color: black;
+}
+.CodeRay pre {
+  margin: 0px;
+}
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line-numbers {
+  background-color: hsl(180,65%,90%);
+  color: gray;
+  text-align: right;
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+}
+.CodeRay .line-numbers a {
+  background-color: hsl(180,65%,90%) !important;
+  color: gray !important;
+  text-decoration: none !important;
+}
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
+.CodeRay .line-numbers a:target { color: blue !important; }
+.CodeRay .line-numbers .highlighted { color: red !important; }
+.CodeRay .line-numbers .highlighted a { color: red !important; }
+.CodeRay span.line-numbers { padding: 0px 4px; }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .code { width: 100%; }
+
+.CodeRay .debug { color: white !important; background: blue !important; }
+
+.CodeRay .annotation { color:#007 }
+.CodeRay .attribute-name { color:#b48 }
+.CodeRay .attribute-value { color:#700 }
+.CodeRay .binary { color:#549 }
+.CodeRay .binary .char { color:#325 }
+.CodeRay .binary .delimiter { color:#325 }
+.CodeRay .char { color:#D20 }
+.CodeRay .char .content { color:#D20 }
+.CodeRay .char .delimiter { color:#710 }
+.CodeRay .class { color:#B06; font-weight:bold }
+.CodeRay .class-variable { color:#369 }
+.CodeRay .color { color:#0A0 }
+.CodeRay .comment { color:#777 }
+.CodeRay .comment .char { color:#444 }
+.CodeRay .comment .delimiter { color:#444 }
+.CodeRay .constant { color:#036; font-weight:bold }
+.CodeRay .decorator { color:#B0B }
+.CodeRay .definition { color:#099; font-weight:bold }
+.CodeRay .delimiter { color:black }
+.CodeRay .directive { color:#088; font-weight:bold }
+.CodeRay .docstring { color:#D42; }
+.CodeRay .doctype { color:#34b }
+.CodeRay .done { text-decoration: line-through; color: gray }
+.CodeRay .entity { color:#800; font-weight:bold }
+.CodeRay .error { color:#F00; background-color:#FAA }
+.CodeRay .escape  { color:#666 }
+.CodeRay .exception { color:#C00; font-weight:bold }
+.CodeRay .float { color:#60E }
+.CodeRay .function { color:#06B; font-weight:bold }
+.CodeRay .function .delimiter { color:#024; font-weight:bold }
+.CodeRay .global-variable { color:#d70 }
+.CodeRay .hex { color:#02b }
+.CodeRay .id  { color:#33D; font-weight:bold }
+.CodeRay .include { color:#B44; font-weight:bold }
+.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black }
+.CodeRay .inline-delimiter { font-weight: bold; color: #666 }
+.CodeRay .instance-variable { color:#33B }
+.CodeRay .integer  { color:#00D }
+.CodeRay .imaginary { color:#f00 }
+.CodeRay .important { color:#D00 }
+.CodeRay .key { color: #606 }
+.CodeRay .key .char { color: #60f }
+.CodeRay .key .delimiter { color: #404 }
+.CodeRay .keyword { color:#080; font-weight:bold }
+.CodeRay .label { color:#970; font-weight:bold }
+.CodeRay .local-variable { color:#963 }
+.CodeRay .namespace { color:#707; font-weight:bold }
+.CodeRay .octal { color:#40E }
+.CodeRay .operator { }
+.CodeRay .predefined { color:#369; font-weight:bold }
+.CodeRay .predefined-constant { color:#069 }
+.CodeRay .predefined-type { color:#0a5; font-weight:bold }
+.CodeRay .preprocessor { color:#579 }
+.CodeRay .pseudo-class { color:#00C; font-weight:bold }
+.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); }
+.CodeRay .regexp .content { color:#808 }
+.CodeRay .regexp .delimiter { color:#404 }
+.CodeRay .regexp .modifier { color:#C2C }
+.CodeRay .reserved { color:#080; font-weight:bold }
+.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); }
+.CodeRay .shell .content { color:#2B2 }
+.CodeRay .shell .delimiter { color:#161 }
+.CodeRay .string { background-color:hsla(0,100%,50%,0.05); }
+.CodeRay .string .char { color: #b0b }
+.CodeRay .string .content { color: #D20 }
+.CodeRay .string .delimiter { color: #710 }
+.CodeRay .string .modifier { color: #E40 }
+.CodeRay .symbol { color:#A60 }
+.CodeRay .symbol .content { color:#A60 }
+.CodeRay .symbol .delimiter { color:#630 }
+.CodeRay .tag { color:#070 }
+.CodeRay .type { color:#339; font-weight:bold }
+.CodeRay .value { color: #088 }
+.CodeRay .variable { color:#037 }
+
+.CodeRay .insert { background: hsla(120,100%,50%,0.12) }
+.CodeRay .delete { background: hsla(0,100%,50%,0.12) }
+.CodeRay .change { color: #bbf; background: #007 }
+.CodeRay .head { color: #f8f; background: #505 }
+.CodeRay .head .filename { color: white; }
+
+.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; }
+.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; }
+
+.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold }
+.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold }
+.CodeRay .change .change { color: #88f }
+.CodeRay .head .head { color: #f4f }
diff --git a/test/samples/css.expected b/test/samples/css.expected
new file mode 100644
index 0000000..be73a7f
--- /dev/null
+++ b/test/samples/css.expected
@@ -0,0 +1,127 @@
+.CodeRay {
+  background-color: hsl(0,0%,95%);
+  border: 1px solid silver;
+  color: black;
+}
+.CodeRay pre {
+  margin: 0px;
+}
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line-numbers {
+  background-color: hsl(180,65%,90%);
+  color: gray;
+  text-align: right;
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+}
+.CodeRay .line-numbers a {
+  background-color: hsl(180,65%,90%) !important;
+  color: gray !important;
+  text-decoration: none !important;
+}
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
+.CodeRay .line-numbers a:target { color: blue !important; }
+.CodeRay .line-numbers .highlighted { color: red !important; }
+.CodeRay .line-numbers .highlighted a { color: red !important; }
+.CodeRay span.line-numbers { padding: 0px 4px; }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .code { width: 100%; }
+
+.CodeRay .debug { color: white !important; background: blue !important; }
+
+.CodeRay .annotation { color:#007 }
+.CodeRay .attribute-name { color:#b48 }
+.CodeRay .attribute-value { color:#700 }
+.CodeRay .binary { color:#549 }
+.CodeRay .binary .char { color:#325 }
+.CodeRay .binary .delimiter { color:#325 }
+.CodeRay .char { color:#D20 }
+.CodeRay .char .content { color:#D20 }
+.CodeRay .char .delimiter { color:#710 }
+.CodeRay .class { color:#B06; font-weight:bold }
+.CodeRay .class-variable { color:#369 }
+.CodeRay .color { color:#0A0 }
+.CodeRay .comment { color:#777 }
+.CodeRay .comment .char { color:#444 }
+.CodeRay .comment .delimiter { color:#444 }
+.CodeRay .constant { color:#036; font-weight:bold }
+.CodeRay .decorator { color:#B0B }
+.CodeRay .definition { color:#099; font-weight:bold }
+.CodeRay .delimiter { color:black }
+.CodeRay .directive { color:#088; font-weight:bold }
+.CodeRay .docstring { color:#D42; }
+.CodeRay .doctype { color:#34b }
+.CodeRay .done { text-decoration: line-through; color: gray }
+.CodeRay .entity { color:#800; font-weight:bold }
+.CodeRay .error { color:#F00; background-color:#FAA }
+.CodeRay .escape  { color:#666 }
+.CodeRay .exception { color:#C00; font-weight:bold }
+.CodeRay .float { color:#60E }
+.CodeRay .function { color:#06B; font-weight:bold }
+.CodeRay .function .delimiter { color:#024; font-weight:bold }
+.CodeRay .global-variable { color:#d70 }
+.CodeRay .hex { color:#02b }
+.CodeRay .id  { color:#33D; font-weight:bold }
+.CodeRay .include { color:#B44; font-weight:bold }
+.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black }
+.CodeRay .inline-delimiter { font-weight: bold; color: #666 }
+.CodeRay .instance-variable { color:#33B }
+.CodeRay .integer  { color:#00D }
+.CodeRay .imaginary { color:#f00 }
+.CodeRay .important { color:#D00 }
+.CodeRay .key { color: #606 }
+.CodeRay .key .char { color: #60f }
+.CodeRay .key .delimiter { color: #404 }
+.CodeRay .keyword { color:#080; font-weight:bold }
+.CodeRay .label { color:#970; font-weight:bold }
+.CodeRay .local-variable { color:#963 }
+.CodeRay .namespace { color:#707; font-weight:bold }
+.CodeRay .octal { color:#40E }
+.CodeRay .operator { }
+.CodeRay .predefined { color:#369; font-weight:bold }
+.CodeRay .predefined-constant { color:#069 }
+.CodeRay .predefined-type { color:#0a5; font-weight:bold }
+.CodeRay .preprocessor { color:#579 }
+.CodeRay .pseudo-class { color:#00C; font-weight:bold }
+.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); }
+.CodeRay .regexp .content { color:#808 }
+.CodeRay .regexp .delimiter { color:#404 }
+.CodeRay .regexp .modifier { color:#C2C }
+.CodeRay .reserved { color:#080; font-weight:bold }
+.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); }
+.CodeRay .shell .content { color:#2B2 }
+.CodeRay .shell .delimiter { color:#161 }
+.CodeRay .string { background-color:hsla(0,100%,50%,0.05); }
+.CodeRay .string .char { color: #b0b }
+.CodeRay .string .content { color: #D20 }
+.CodeRay .string .delimiter { color: #710 }
+.CodeRay .string .modifier { color: #E40 }
+.CodeRay .symbol { color:#A60 }
+.CodeRay .symbol .content { color:#A60 }
+.CodeRay .symbol .delimiter { color:#630 }
+.CodeRay .tag { color:#070 }
+.CodeRay .type { color:#339; font-weight:bold }
+.CodeRay .value { color: #088 }
+.CodeRay .variable { color:#037 }
+
+.CodeRay .insert { background: hsla(120,100%,50%,0.12) }
+.CodeRay .delete { background: hsla(0,100%,50%,0.12) }
+.CodeRay .change { color: #bbf; background: #007 }
+.CodeRay .head { color: #f8f; background: #505 }
+.CodeRay .head .filename { color: white; }
+
+.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; }
+.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; }
+
+.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold }
+.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold }
+.CodeRay .change .change { color: #88f }
+.CodeRay .head .head { color: #f4f }
diff --git a/sample/css.rb b/test/samples/css.rb
index 52e4bcc..52e4bcc 100644
--- a/sample/css.rb
+++ b/test/samples/css.rb
diff --git a/test/samples/div.actual b/test/samples/div.actual
new file mode 100644
index 0000000..d1e692a
--- /dev/null
+++ b/test/samples/div.actual
@@ -0,0 +1,17 @@
+<div class="CodeRay">
+  <div class="code"><pre><span style="color:#080;font-weight:bold">for</span> a <span style="color:#080;font-weight:bold">in</span> <span style="color:#00D">0</span>..<span style="color:#00D">255</span>
+        a = a.chr
+        <span style="color:#080;font-weight:bold">begin</span>
+                x = eval(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">?</span><span style="color:#b0b">\\</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>)
+                <span style="color:#080;font-weight:bold">if</span> x == a[<span style="color:#00D">0</span>]
+                        <span style="color:#080;font-weight:bold">next</span>
+                <span style="color:#080;font-weight:bold">else</span>
+                        print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: </span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>x<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>
+                <span style="color:#080;font-weight:bold">end</span>
+        <span style="color:#080;font-weight:bold">rescue</span> <span style="color:#036;font-weight:bold">SyntaxError</span> =&gt; boom
+                print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: error</span><span style="color:#710">&quot;</span></span>
+        <span style="color:#080;font-weight:bold">end</span>
+        puts
+<span style="color:#080;font-weight:bold">end</span>
+</pre></div>
+</div>
diff --git a/test/samples/div.expected b/test/samples/div.expected
new file mode 100644
index 0000000..d1e692a
--- /dev/null
+++ b/test/samples/div.expected
@@ -0,0 +1,17 @@
+<div class="CodeRay">
+  <div class="code"><pre><span style="color:#080;font-weight:bold">for</span> a <span style="color:#080;font-weight:bold">in</span> <span style="color:#00D">0</span>..<span style="color:#00D">255</span>
+        a = a.chr
+        <span style="color:#080;font-weight:bold">begin</span>
+                x = eval(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">?</span><span style="color:#b0b">\\</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>)
+                <span style="color:#080;font-weight:bold">if</span> x == a[<span style="color:#00D">0</span>]
+                        <span style="color:#080;font-weight:bold">next</span>
+                <span style="color:#080;font-weight:bold">else</span>
+                        print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: </span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>x<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>
+                <span style="color:#080;font-weight:bold">end</span>
+        <span style="color:#080;font-weight:bold">rescue</span> <span style="color:#036;font-weight:bold">SyntaxError</span> =&gt; boom
+                print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: error</span><span style="color:#710">&quot;</span></span>
+        <span style="color:#080;font-weight:bold">end</span>
+        puts
+<span style="color:#080;font-weight:bold">end</span>
+</pre></div>
+</div>
diff --git a/sample/div.rb b/test/samples/div.rb
index 27b6f32..27b6f32 100644
--- a/sample/div.rb
+++ b/test/samples/div.rb
diff --git a/test/samples/encoder.actual b/test/samples/encoder.actual
new file mode 100644
index 0000000..8bd83a9
--- /dev/null
+++ b/test/samples/encoder.actual
@@ -0,0 +1,65 @@
+Encoders Demo: puts 17 + 4
+
+Statistic:
+
+Code Statistics
+
+Tokens                   8
+  Non-Whitespace         4
+Bytes Total             12
+
+Token Types (4):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                        8  100.00 %     1.5
+  space                        4   50.00 %     1.0
+  integer                      2   25.00 %     1.5
+  ident                        1   12.50 %     4.0
+  operator                     1   12.50 %     1.0
+
+
+Original text:
+[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}]
+
+YAML:
+--- 
+- - puts
+  - :ident
+- - " "
+  - :space
+- - "17"
+  - :integer
+- - " "
+  - :space
+- - +
+  - :operator
+- - " "
+  - :space
+- - "4"
+  - :integer
+- - |
+    
+
+  - :space
+
+Dump:
+"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257"
+compressed: 79 byte < 1200 byte
+
+Undump:
+
+Code Statistics
+
+Tokens                 800
+  Non-Whitespace       400
+Bytes Total           1200
+
+Token Types (4):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                      800  100.00 %     1.5
+  space                      400   50.00 %     1.0
+  integer                    200   25.00 %     1.5
+  ident                      100   12.50 %     4.0
+  operator                   100   12.50 %     1.0
+
diff --git a/sample/encoder.expected b/test/samples/encoder.expected
index 438032a..8bd83a9 100644
--- a/sample/encoder.expected
+++ b/test/samples/encoder.expected
@@ -43,8 +43,8 @@ YAML:
   - :space
 
 Dump:
-"x\332\355\330\273\n\302@\020\005PQIL4\235\245E\260\265\022\004a\266\021\002B\332\250U\2525\031$\210\233\260\273)\202?o\036\370\370\006\271\325\354\314\345\334\017\330\351,\216h\031\2259'\262!:\227wV&\035\207\223\324]{Um\r\371E\316\312\266\253\023\222o*\231q\373v\267{Z\024\312\362\215u\037\t\267\e\e\n\312\212\265\264\345\357u'f\335\360\373\255/\025\3167n\253\206\374\335!<XXXXXXXXXXXXXXXXXXXXX\330\277\267\016\005\a\223I\245X\307\027Z}\276\352O\303\315\020%\365\265y:47V\263|\034G/\301K\315)"
-compressed: 188 byte < 1200 byte
+"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257"
+compressed: 79 byte < 1200 byte
 
 Undump:
 
diff --git a/sample/encoder.rb b/test/samples/encoder.rb
index 22eaa22..bfcfbfa 100644
--- a/sample/encoder.rb
+++ b/test/samples/encoder.rb
@@ -27,8 +27,9 @@ puts 'YAML:'
 puts yaml
 
 # alternative 3
+require 'zlib'
 BIGSAMPLE = SAMPLE * 100
-dump = CodeRay.scan(BIGSAMPLE, :ruby).dump
+dump = Zlib::Deflate.deflate(CodeRay.scan(BIGSAMPLE, :ruby).debug)
 puts
 puts 'Dump:'
 p dump
@@ -36,4 +37,4 @@ puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size]
 
 puts
 puts 'Undump:'
-puts dump.undump.statistic
+puts CodeRay.scan(Zlib::Inflate.inflate(dump), :debug).statistic
diff --git a/test/samples/global_vars.actual b/test/samples/global_vars.actual
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/samples/global_vars.actual
diff --git a/test/samples/global_vars.diff b/test/samples/global_vars.diff
new file mode 100644
index 0000000..2ed3382
--- /dev/null
+++ b/test/samples/global_vars.diff
@@ -0,0 +1,5 @@
+1,3d0
+< <--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL
+< ODER
+< TEXT = <--$BLA-->.TEST(...) 
+\ No newline at end of file
diff --git a/sample/global_vars.expected b/test/samples/global_vars.expected
index 0dc13c8..0dc13c8 100644
--- a/sample/global_vars.expected
+++ b/test/samples/global_vars.expected
diff --git a/sample/global_vars.rb b/test/samples/global_vars.rb
index 8066d67..8066d67 100644
--- a/sample/global_vars.rb
+++ b/test/samples/global_vars.rb
diff --git a/sample/global_vars2.expected b/test/samples/global_vars2.expected
index 964cf50..964cf50 100644
--- a/sample/global_vars2.expected
+++ b/test/samples/global_vars2.expected
diff --git a/sample/global_vars2.rb b/test/samples/global_vars2.rb
index 7646890..7646890 100644
--- a/sample/global_vars2.rb
+++ b/test/samples/global_vars2.rb
diff --git a/sample/highlight.expected b/test/samples/highlight.expected
index 6a9b278..6a9b278 100644
--- a/sample/highlight.expected
+++ b/test/samples/highlight.expected
diff --git a/sample/highlight.rb b/test/samples/highlight.rb
index 846efa4..846efa4 100644
--- a/sample/highlight.rb
+++ b/test/samples/highlight.rb
diff --git a/sample/html.expected b/test/samples/html.expected
index e98d589..e98d589 100644
--- a/sample/html.expected
+++ b/test/samples/html.expected
diff --git a/sample/html.rb b/test/samples/html.rb
index c18284a..c18284a 100644
--- a/sample/html.rb
+++ b/test/samples/html.rb
diff --git a/sample/html2.expected b/test/samples/html2.expected
index c8ae56a..c8ae56a 100644
--- a/sample/html2.expected
+++ b/test/samples/html2.expected
diff --git a/sample/html2.rb b/test/samples/html2.rb
index 618d168..618d168 100644
--- a/sample/html2.rb
+++ b/test/samples/html2.rb
diff --git a/sample/html_list.expected b/test/samples/html_list.expected
index a4092c8..a4092c8 100644
--- a/sample/html_list.expected
+++ b/test/samples/html_list.expected
diff --git a/sample/html_list.rb b/test/samples/html_list.rb
index fdfa512..fdfa512 100644
--- a/sample/html_list.rb
+++ b/test/samples/html_list.rb
diff --git a/sample/load_encoder.expected b/test/samples/load_encoder.expected
index 1cff356..1cff356 100644
--- a/sample/load_encoder.expected
+++ b/test/samples/load_encoder.expected
diff --git a/sample/load_encoder.rb b/test/samples/load_encoder.rb
index 9594bfa..9594bfa 100644
--- a/sample/load_encoder.rb
+++ b/test/samples/load_encoder.rb
diff --git a/sample/load_scanner.expected b/test/samples/load_scanner.expected
index a2d200d..a2d200d 100644
--- a/sample/load_scanner.expected
+++ b/test/samples/load_scanner.expected
diff --git a/sample/load_scanner.rb b/test/samples/load_scanner.rb
index 23be8a2..23be8a2 100644
--- a/sample/load_scanner.rb
+++ b/test/samples/load_scanner.rb
diff --git a/sample/more.expected b/test/samples/more.expected
index 196904d..196904d 100644
--- a/sample/more.expected
+++ b/test/samples/more.expected
diff --git a/sample/more.rb b/test/samples/more.rb
index 0db7ba4..0db7ba4 100644
--- a/sample/more.rb
+++ b/test/samples/more.rb
diff --git a/sample/scanner.expected b/test/samples/scanner.expected
index 5015168..5015168 100644
--- a/sample/scanner.expected
+++ b/test/samples/scanner.expected
diff --git a/sample/scanner.rb b/test/samples/scanner.rb
index 6a0245e..6a0245e 100644
--- a/sample/scanner.rb
+++ b/test/samples/scanner.rb
diff --git a/sample/server.rb b/test/samples/server.rb
index ccdff32..ccdff32 100644
--- a/sample/server.rb
+++ b/test/samples/server.rb
diff --git a/sample/simple.expected b/test/samples/simple.expected
index b3d7875..b3d7875 100644
--- a/sample/simple.expected
+++ b/test/samples/simple.expected
diff --git a/sample/simple.rb b/test/samples/simple.rb
index a3129b0..a3129b0 100644
--- a/sample/simple.rb
+++ b/test/samples/simple.rb
diff --git a/sample/stream.rb b/test/samples/stream.rb
index 7ed8a22..7ed8a22 100644
--- a/sample/stream.rb
+++ b/test/samples/stream.rb
diff --git a/sample/stream2.expected b/test/samples/stream2.expected
index 83aee98..83aee98 100644
--- a/sample/stream2.expected
+++ b/test/samples/stream2.expected
diff --git a/sample/stream2.rb b/test/samples/stream2.rb
index d43cc9a..d43cc9a 100644
--- a/sample/stream2.rb
+++ b/test/samples/stream2.rb
diff --git a/sample/suite.rb b/test/samples/suite.rb
index fa24114..cfe53c0 100644
--- a/sample/suite.rb
+++ b/test/samples/suite.rb
@@ -1,5 +1,5 @@
 mydir = File.dirname(__FILE__)
-$:.unshift mydir + '/../lib/'
+$:.unshift mydir + '/../../lib/'
 
 $VERBOSE = true
 
@@ -29,7 +29,7 @@ class CodeRaySuite < TestCase
         output = name + '.expected'
         code = File.open(input, 'rb') { |f| break f.read }
         
-        result = `ruby -wI../lib #{input}`
+        result = `ruby -wI../../lib #{input}`
         
         diff = output.sub '.expected', '.diff'
         File.delete diff if File.exist? diff
@@ -39,10 +39,10 @@ class CodeRaySuite < TestCase
           ok = expected == result
           unless ok
             File.open(computed, 'w') { |f| f.write result }
-            `diff #{output} #{computed} > #{diff}` if $DEBUG
+            `diff #{output} #{computed} > #{diff}`
             puts "Test failed; output written to #{diff}."
           end
-          assert(ok, "Output error: #{computed} != #{output}") unless $DEBUG
+          assert(ok, "Output error: #{computed} != #{output}")
         else
           File.open(output, 'w') do |f| f.write result end
           puts "New test: #{output}"
diff --git a/sample/tokens.expected b/test/samples/tokens.expected
index 747904e..747904e 100644
--- a/sample/tokens.expected
+++ b/test/samples/tokens.expected
diff --git a/sample/tokens.rb b/test/samples/tokens.rb
index 91b8abb..91b8abb 100644
--- a/sample/tokens.rb
+++ b/test/samples/tokens.rb
diff --git a/test/unit/tokens.rb b/test/unit/tokens.rb
index 86dc632..73b0fd5 100644
--- a/test/unit/tokens.rb
+++ b/test/unit/tokens.rb
@@ -18,15 +18,6 @@ class TokensTest < Test::Unit::TestCase
     assert_equal tokens.count, 4
   end
   
-  def test_dump_undump
-    tokens = make_tokens
-    tokens2 = nil
-    assert_nothing_raised do
-      tokens2 = tokens.dump.undump
-    end
-    assert_equal tokens, tokens2
-  end
-  
   def test_to_s
     assert_equal 'string()', make_tokens.to_s
   end