Tests:

- improved coderay_suite.rb (random and shuffled tests, max parameter, scannerlang->lang, sorted test cases...)
- changed html output extension to .actual.html to svn:ignore them
- fixed some tests (deleted $Id$ etc.)
- made XHTML testcase work

Scanners:
- fixed HTML, Delphi and Nitro scanners thanks to new tests

Engine:
- Tokens#fix and #fix! added (yet to be tested)
- improved Scanner#raise_inspect a bit

Converted more files to UNIX format (go away, stinkin' \r!)

12 files changed, 1042 insertions, 989 deletions

diff --git a/lib/coderay/helpers/filetype.rb b/lib/coderay/helpers/filetype.rb
index 7a9c489..7341f01 100644
--- a/lib/coderay/helpers/filetype.rb
+++ b/lib/coderay/helpers/filetype.rb
@@ -1,180 +1,180 @@
-# =FileType

-#

-# A simple filetype recognizer

-#

-# Author: murphy (mail to murphy cYcnus de)

-#

-# Version: 0.1 (2005.september.1)

-#

-# == Documentation

-#

-#  # determine the type of the given

-#   lang = FileType[ARGV.first]

-#  

-#   # return :plaintext if the file type is unknown

-#   lang = FileType.fetch ARGV.first, :plaintext

-#  

-#   # try the shebang line, too

-#   lang = FileType.fetch ARGV.first, :plaintext, true

-module FileType

-

-  UnknownFileType = Class.new Exception

-

-  class << self

-

-    # Try to determine the file type of the file.

-    #

-    # +filename+ is a relative or absolute path to a file.

-    #

-    # The file itself is only accessed when +read_shebang+ is set to true.

-    # That means you can get filetypes from files that don't exist.

-    def [] filename, read_shebang = false

-      name = File.basename filename

-      ext = File.extname name

-      ext.sub!(/^\./, '')  # delete the leading dot

-

-      type =

-        TypeFromExt[ext] ||

-        TypeFromExt[ext.downcase] ||

-        TypeFromName[name] ||

-        TypeFromName[name.downcase]

-      type ||= shebang(filename) if read_shebang

-

-      type

-    end

-

-    def shebang filename

-      begin

-        File.open filename, 'r' do |f|

-          first_line = f.gets

-          first_line[TypeFromShebang]

-        end

-      rescue IOError

-        nil

-      end

-    end

-

-    # This works like Hash#fetch.

-    #

-    # If the filetype cannot be found, the +default+ value

-    # is returned.

-    def fetch filename, default = nil, read_shebang = false

-      if default and block_given?

-        warn 'block supersedes default value argument'

-      end

-

-      unless type = self[filename, read_shebang]

-        return yield if block_given?

-        return default if default

-        raise UnknownFileType, 'Could not determine type of %p.' % filename

-      end

-      type

-    end

-

-  end

-

-  TypeFromExt = {

-    'rb' => :ruby,

-    'rbw' => :ruby,

-    'rake' => :ruby,

-    'cpp' => :c,

-    'c' => :c,

-    'h' => :c,

-    'xml' => :xml,

-    'htm' => :html,

-    'html' => :html,

-    'xhtml' => :xhtml,

-    'rhtml' => :rhtml,

-    'yaml' => :yaml,

-    'yml' => :yaml,

-  }

-

-  TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/

-

-  TypeFromName = {

-    'Rakefile' => :ruby,

-    'Rantfile' => :ruby,

-  }

-

-end

-

-if $0 == __FILE__

-  $VERBOSE = true

-  eval DATA.read, nil, $0, __LINE__+4

-end

-

-__END__

-

-require 'test/unit'

-

-class TC_FileType < Test::Unit::TestCase

-

-  def test_fetch

-    assert_raise FileType::UnknownFileType do

-      FileType.fetch ''

-    end

-

-    assert_throws :not_found do

-      FileType.fetch '.' do

-        throw :not_found

-      end

-    end

-

-    assert_equal :default, FileType.fetch('c', :default)

-

-    stderr, fake_stderr = $stderr, Object.new

-    $err = ''

-    def fake_stderr.write x

-      $err << x

-    end

-    $stderr = fake_stderr

-    FileType.fetch('c', :default) { }

-    assert_equal "block supersedes default value argument\n", $err

-    $stderr = stderr

-  end

-

-  def test_ruby

-    assert_equal :ruby, FileType['test.rb']

-    assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']

-    assert_equal :ruby, FileType['/usr/bin/something/Rakefile']

-    assert_equal :ruby, FileType['~/myapp/gem/Rantfile']

-    assert_equal :ruby, FileType['./lib/tasks\repository.rake']

-    assert_not_equal :ruby, FileType['test_rb']

-    assert_not_equal :ruby, FileType['Makefile']

-    assert_not_equal :ruby, FileType['set.rb/set']

-    assert_not_equal :ruby, FileType['~/projects/blabla/rb']

-  end

-

-  def test_c

-    assert_equal :c, FileType['test.c']

-    assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']

-    assert_not_equal :c, FileType['test_c']

-    assert_not_equal :c, FileType['Makefile']

-    assert_not_equal :c, FileType['set.h/set']

-    assert_not_equal :c, FileType['~/projects/blabla/c']

-  end

-

-  def test_html

-    assert_equal :html, FileType['test.htm']

-    assert_equal :xhtml, FileType['test.xhtml']

-    assert_equal :xhtml, FileType['test.html.xhtml']

-    assert_equal :rhtml, FileType['_form.rhtml']

-  end

-

-  def test_yaml

-    assert_equal :yaml, FileType['test.yml']

-    assert_equal :yaml, FileType['test.yaml']

-    assert_equal :yaml, FileType['my.html.yaml']

-    assert_not_equal :yaml, FileType['YAML']

-  end

-

-  def test_shebang

-    dir = './test'

-    if File.directory? dir

-      Dir.chdir dir do

-        assert_equal :c, FileType['test.c']

-      end

-    end

-  end

-

-end

+# =FileType
+#
+# A simple filetype recognizer
+#
+# Author: murphy (mail to murphy cYcnus de)
+#
+# Version: 0.1 (2005.september.1)
+#
+# == Documentation
+#
+#  # determine the type of the given
+#   lang = FileType[ARGV.first]
+#  
+#   # return :plaintext if the file type is unknown
+#   lang = FileType.fetch ARGV.first, :plaintext
+#  
+#   # try the shebang line, too
+#   lang = FileType.fetch ARGV.first, :plaintext, true
+module FileType
+
+  UnknownFileType = Class.new Exception
+
+  class << self
+
+    # Try to determine the file type of the file.
+    #
+    # +filename+ is a relative or absolute path to a file.
+    #
+    # The file itself is only accessed when +read_shebang+ is set to true.
+    # That means you can get filetypes from files that don't exist.
+    def [] filename, read_shebang = false
+      name = File.basename filename
+      ext = File.extname name
+      ext.sub!(/^\./, '')  # delete the leading dot
+
+      type =
+        TypeFromExt[ext] ||
+        TypeFromExt[ext.downcase] ||
+        TypeFromName[name] ||
+        TypeFromName[name.downcase]
+      type ||= shebang(filename) if read_shebang
+
+      type
+    end
+
+    def shebang filename
+      begin
+        File.open filename, 'r' do |f|
+          first_line = f.gets
+          first_line[TypeFromShebang]
+        end
+      rescue IOError
+        nil
+      end
+    end
+
+    # This works like Hash#fetch.
+    #
+    # If the filetype cannot be found, the +default+ value
+    # is returned.
+    def fetch filename, default = nil, read_shebang = false
+      if default and block_given?
+        warn 'block supersedes default value argument'
+      end
+
+      unless type = self[filename, read_shebang]
+        return yield if block_given?
+        return default if default
+        raise UnknownFileType, 'Could not determine type of %p.' % filename
+      end
+      type
+    end
+
+  end
+
+  TypeFromExt = {
+    'rb' => :ruby,
+    'rbw' => :ruby,
+    'rake' => :ruby,
+    'cpp' => :c,
+    'c' => :c,
+    'h' => :c,
+    'xml' => :xml,
+    'htm' => :html,
+    'html' => :html,
+    'xhtml' => :xhtml,
+    'rhtml' => :rhtml,
+    'yaml' => :yaml,
+    'yml' => :yaml,
+  }
+
+  TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
+
+  TypeFromName = {
+    'Rakefile' => :ruby,
+    'Rantfile' => :ruby,
+  }
+
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  eval DATA.read, nil, $0, __LINE__+4
+end
+
+__END__
+
+require 'test/unit'
+
+class TC_FileType < Test::Unit::TestCase
+
+  def test_fetch
+    assert_raise FileType::UnknownFileType do
+      FileType.fetch ''
+    end
+
+    assert_throws :not_found do
+      FileType.fetch '.' do
+        throw :not_found
+      end
+    end
+
+    assert_equal :default, FileType.fetch('c', :default)
+
+    stderr, fake_stderr = $stderr, Object.new
+    $err = ''
+    def fake_stderr.write x
+      $err << x
+    end
+    $stderr = fake_stderr
+    FileType.fetch('c', :default) { }
+    assert_equal "block supersedes default value argument\n", $err
+    $stderr = stderr
+  end
+
+  def test_ruby
+    assert_equal :ruby, FileType['test.rb']
+    assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
+    assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
+    assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
+    assert_equal :ruby, FileType['./lib/tasks\repository.rake']
+    assert_not_equal :ruby, FileType['test_rb']
+    assert_not_equal :ruby, FileType['Makefile']
+    assert_not_equal :ruby, FileType['set.rb/set']
+    assert_not_equal :ruby, FileType['~/projects/blabla/rb']
+  end
+
+  def test_c
+    assert_equal :c, FileType['test.c']
+    assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
+    assert_not_equal :c, FileType['test_c']
+    assert_not_equal :c, FileType['Makefile']
+    assert_not_equal :c, FileType['set.h/set']
+    assert_not_equal :c, FileType['~/projects/blabla/c']
+  end
+
+  def test_html
+    assert_equal :html, FileType['test.htm']
+    assert_equal :xhtml, FileType['test.xhtml']
+    assert_equal :xhtml, FileType['test.html.xhtml']
+    assert_equal :rhtml, FileType['_form.rhtml']
+  end
+
+  def test_yaml
+    assert_equal :yaml, FileType['test.yml']
+    assert_equal :yaml, FileType['test.yaml']
+    assert_equal :yaml, FileType['my.html.yaml']
+    assert_not_equal :yaml, FileType['YAML']
+  end
+
+  def test_shebang
+    dir = './test'
+    if File.directory? dir
+      Dir.chdir dir do
+        assert_equal :c, FileType['test.c']
+      end
+    end
+  end
+
+end
diff --git a/lib/coderay/helpers/gzip_simple.rb b/lib/coderay/helpers/gzip_simple.rb
index 28e7f1e..df4bcba 100644
--- a/lib/coderay/helpers/gzip_simple.rb
+++ b/lib/coderay/helpers/gzip_simple.rb
@@ -1,122 +1,122 @@
-# =GZip Simple

-#

-# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)

-#

-# Author: murphy (mail to murphy cYcnus de)

-#

-# Version: 0.2 (2005.may.28)

-#

-# ==Documentation

-#

-# See +GZip+ module and the +String+ extensions.

-#

-module GZip

-

-  require 'zlib'

-

-  # The default zipping level. 7 zips good and fast.

-  DEFAULT_GZIP_LEVEL = 7

-

-  # Unzips the given string +s+.

-  #

-  # Example:

-  #   require 'gzip_simple'

-  #   print GZip.gunzip(File.read('adresses.gz'))

-  def GZip.gunzip s

-    Zlib::Inflate.inflate s

-  end

-

-  # Zips the given string +s+.

-  #

-  # Example:

-  #   require 'gzip_simple'

-  #   File.open('adresses.gz', 'w') do |file

-  #     file.write GZip.gzip('Mum: 0123 456 789', 9)

-  #   end

-  #

-  # If you provide a +level+, you can control how strong

-  # the string is compressed:

-  # - 0: no compression, only convert to gzip format

-  # - 1: compress fast

-  # - 7: compress more, but still fast (default)

-  # - 8: compress more, slower

-  # - 9: compress best, very slow

-  def GZip.gzip s, level = DEFAULT_GZIP_LEVEL

-    Zlib::Deflate.new(level).deflate s, Zlib::FINISH

-  end

-end

-

-# String extensions to use the GZip module.

-#

-# The methods gzip and gunzip provide an even more simple

-# interface to the ZLib:

-#

-#   # create a big string

-#   x = 'a' * 1000

-#   

-#   # zip it

-#   x_gz = x.gzip

-#   

-#   # test the result

-#   puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]

-#   #-> Zipped 1000 bytes to 19 bytes.

-#   

-#   # unzipping works

-#   p x_gz.gunzip == x  #-> true

-class String

-  # Returns the string, unzipped.

-  # See GZip.gunzip

-  def gunzip

-    GZip.gunzip self

-  end

-  # Replaces the string with its unzipped value.

-  # See GZip.gunzip

-  def gunzip!

-    replace gunzip

-  end

-

-  # Returns the string, zipped.

-  # +level+ is the gzip compression level, see GZip.gzip.

-  def gzip level = GZip::DEFAULT_GZIP_LEVEL

-    GZip.gzip self, level

-  end

-  # Replaces the string with its zipped value.

-  # See GZip.gzip.

-  def gzip!(*args)

-    replace gzip(*args)

-  end

-end

-

-if $0 == __FILE__

-  eval DATA.read, nil, $0, __LINE__+4

-end

-

-__END__

-#CODE

-

-# Testing / Benchmark

-x = 'a' * 1000

-x_gz = x.gzip

-puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]  #-> Zipped 1000 bytes to 19 bytes.

-p x_gz.gunzip == x  #-> true

-

-require 'benchmark'

-

-INFO = 'packed to %0.3f%%'  # :nodoc:

-

-x = Array.new(100000) { rand(255).chr + 'aaaaaaaaa' + rand(255).chr }.join

-Benchmark.bm(10) do |bm|

-  for level in 0..9

-    bm.report "zip #{level}" do

-      $x = x.gzip level

-    end

-    puts INFO % [100.0 * $x.size / x.size]

-  end

-  bm.report 'zip' do

-    $x = x.gzip

-  end

-  puts INFO % [100.0 * $x.size / x.size]

-  bm.report 'unzip' do

-    $x.gunzip

-  end

-end

+# =GZip Simple
+#
+# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
+#
+# Author: murphy (mail to murphy cYcnus de)
+#
+# Version: 0.2 (2005.may.28)
+#
+# ==Documentation
+#
+# See +GZip+ module and the +String+ extensions.
+#
+module GZip
+
+  require 'zlib'
+
+  # The default zipping level. 7 zips good and fast.
+  DEFAULT_GZIP_LEVEL = 7
+
+  # Unzips the given string +s+.
+  #
+  # Example:
+  #   require 'gzip_simple'
+  #   print GZip.gunzip(File.read('adresses.gz'))
+  def GZip.gunzip s
+    Zlib::Inflate.inflate s
+  end
+
+  # Zips the given string +s+.
+  #
+  # Example:
+  #   require 'gzip_simple'
+  #   File.open('adresses.gz', 'w') do |file
+  #     file.write GZip.gzip('Mum: 0123 456 789', 9)
+  #   end
+  #
+  # If you provide a +level+, you can control how strong
+  # the string is compressed:
+  # - 0: no compression, only convert to gzip format
+  # - 1: compress fast
+  # - 7: compress more, but still fast (default)
+  # - 8: compress more, slower
+  # - 9: compress best, very slow
+  def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
+    Zlib::Deflate.new(level).deflate s, Zlib::FINISH
+  end
+end
+
+# String extensions to use the GZip module.
+#
+# The methods gzip and gunzip provide an even more simple
+# interface to the ZLib:
+#
+#   # create a big string
+#   x = 'a' * 1000
+#   
+#   # zip it
+#   x_gz = x.gzip
+#   
+#   # test the result
+#   puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]
+#   #-> Zipped 1000 bytes to 19 bytes.
+#   
+#   # unzipping works
+#   p x_gz.gunzip == x  #-> true
+class String
+  # Returns the string, unzipped.
+  # See GZip.gunzip
+  def gunzip
+    GZip.gunzip self
+  end
+  # Replaces the string with its unzipped value.
+  # See GZip.gunzip
+  def gunzip!
+    replace gunzip
+  end
+
+  # Returns the string, zipped.
+  # +level+ is the gzip compression level, see GZip.gzip.
+  def gzip level = GZip::DEFAULT_GZIP_LEVEL
+    GZip.gzip self, level
+  end
+  # Replaces the string with its zipped value.
+  # See GZip.gzip.
+  def gzip!(*args)
+    replace gzip(*args)
+  end
+end
+
+if $0 == __FILE__
+  eval DATA.read, nil, $0, __LINE__+4
+end
+
+__END__
+#CODE
+
+# Testing / Benchmark
+x = 'a' * 1000
+x_gz = x.gzip
+puts 'Zipped %d bytes to %d bytes.' % [x.size, x_gz.size]  #-> Zipped 1000 bytes to 19 bytes.
+p x_gz.gunzip == x  #-> true
+
+require 'benchmark'
+
+INFO = 'packed to %0.3f%%'  # :nodoc:
+
+x = Array.new(100000) { rand(255).chr + 'aaaaaaaaa' + rand(255).chr }.join
+Benchmark.bm(10) do |bm|
+  for level in 0..9
+    bm.report "zip #{level}" do
+      $x = x.gzip level
+    end
+    puts INFO % [100.0 * $x.size / x.size]
+  end
+  bm.report 'zip' do
+    $x = x.gzip
+  end
+  puts INFO % [100.0 * $x.size / x.size]
+  bm.report 'unzip' do
+    $x.gunzip
+  end
+end
diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index 742717d..aba6ff7 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -1,326 +1,326 @@
-# = PluginHost

-#

-# $Id$

-#

-# A simple subclass plugin system.

-#

-#  Example:

-#    class Generators < PluginHost

-#      plugin_path 'app/generators'

-#    end

-#    

-#    class Generator

-#      extend Plugin

-#      PLUGIN_HOST = Generators

-#    end

-#    

-#    class FancyGenerator < Generator

-#      register_for :fancy

-#    end

-#

-#    Generators[:fancy]  #-> FancyGenerator

-#    # or

-#    require_plugin 'Generators/fancy'

-module PluginHost

-

-  # Raised if Encoders::[] fails because:

-  # * a file could not be found

-  # * the requested Encoder is not registered

-  PluginNotFound = Class.new Exception

-  HostNotFound = Class.new Exception

-

-  PLUGIN_HOSTS = []

-  PLUGIN_HOSTS_BY_ID = {}  # dummy hash

-

-  # Loads all plugins using all_plugin_names and load.

-  def load_all

-    for plugin in all_plugin_names

-      load plugin

-    end

-  end

-

-  # Returns the Plugin for +id+.

-  #

-  # Example:

-  #  yaml_plugin = MyPluginHost[:yaml]

-  def [] id, *args, &blk

-    plugin = validate_id(id)

-    begin

-      plugin = plugin_hash.[] plugin, *args, &blk

-    end while plugin.is_a? Symbol

-    plugin

-  end

-

-  # Alias for +[]+.

-  alias load []

-

-  def require_helper plugin_id, helper_name

-    path = path_to File.join(plugin_id, helper_name)

-    require path

-  end

-

-  class << self

-

-    # Adds the module/class to the PLUGIN_HOSTS list.

-    def extended mod

-      PLUGIN_HOSTS << mod

-    end

-

-    # Warns you that you should not #include this module.

-    def included mod

-      warn "#{name} should not be included. Use extend."

-    end

-

-    # Find the PluginHost for host_id.

-    def host_by_id host_id

-      unless PLUGIN_HOSTS_BY_ID.default_proc

-        ph = Hash.new do |h, a_host_id|

-          for host in PLUGIN_HOSTS

-            h[host.host_id] = host

-          end

-          h.fetch a_host_id, nil

-        end

-        PLUGIN_HOSTS_BY_ID.replace ph

-      end

-      PLUGIN_HOSTS_BY_ID[host_id]

-    end

-

-  end

-

-  # The path where the plugins can be found.

-  def plugin_path *args

-    unless args.empty?

-      @plugin_path = File.expand_path File.join(*args)

-      load_map

-    end

-    @plugin_path

-  end

-

-  # The host's ID.

-  #

-  # If PLUGIN_HOST_ID is not set, it is simply the class name.

-  def host_id

-    if self.const_defined? :PLUGIN_HOST_ID

-      self::PLUGIN_HOST_ID

-    else

-      name

-    end

-  end

-

-  # Map a plugin_id to another.

-  #

-  # Usage: Put this in a file plugin_path/_map.rb.

-  #

-  #  class MyColorHost < PluginHost

-  #    map :navy => :dark_blue,

-  #      :maroon => :brown,

-  #      :luna => :moon

-  #  end

-  def map hash

-    for from, to in hash

-      from = validate_id from

-      to = validate_id to

-      plugin_hash[from] = to unless plugin_hash.has_key? from

-    end

-  end

-

-  # Define the default plugin to use when no plugin is found

-  # for a given id.

-  #

-  # See also map.

-  #

-  #  class MyColorHost < PluginHost

-  #    map :navy => :dark_blue

-  #    default :gray

-  #  end

-  def default id

-    id = validate_id id

-    plugin_hash[nil] = id

-  end

-

-  # Every plugin must register itself for one or more

-  # +ids+ by calling register_for, which calls this method.

-  #

-  # See Plugin#register_for.

-  def register plugin, *ids

-    for id in ids

-      unless id.is_a? Symbol

-        raise ArgumentError,

-          "id must be a Symbol, but it was a #{id.class}"

-      end

-      plugin_hash[validate_id(id)] = plugin

-    end

-  end

-

-  # A Hash of plugion_id => Plugin pairs.

-  def plugin_hash

-    @plugin_hash ||= create_plugin_hash

-  end

-

-  # Returns an array of all .rb files in the plugin path.

-  #

-  # The extension .rb is not included.

-  def all_plugin_names

-    Dir[path_to('*')].select do |file|

-      File.basename(file)[/^(?!_)\w+\.rb$/]

-    end.map do |file|

-      File.basename file, '.rb'

-    end

-  end

-

-  # Makes a map of all loaded plugins.

-  def inspect

-    map = plugin_hash.dup

-    map.each do |id, plugin|

-      map[id] = plugin.to_s[/(?>[\w_]+)$/]

-    end

-    "#{name}[#{host_id}]#{map.inspect}"

-  end

-

-protected

-  # Created a new plugin list and stores it to @plugin_hash.

-  def create_plugin_hash

-    @plugin_hash =

-      Hash.new do |h, plugin_id|

-        id = validate_id(plugin_id)

-        path = path_to id

-        begin

-          require path

-        rescue LoadError => boom

-          if h.has_key? nil  # default plugin

-            h[id] = h[nil]

-          else

-            raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]

-          end

-        else

-          # Plugin should have registered by now

-          unless h.has_key? id

-            raise PluginNotFound,

-              "No #{self.name} plugin for #{id.inspect} found in #{path}."

-          end

-        end

-        h[id]

-      end

-  end

-

-  # Loads the map file (see map).

-  #

-  # This is done automatically when plugin_path is called.

-  def load_map

-    mapfile = path_to '_map'

-    if File.exist? mapfile

-      require mapfile

-    elsif $DEBUG

-      warn 'no _map.rb found for %s' % name

-    end

-  end

-

-  # Returns the Plugin for +id+.

-  # Use it like Hash#fetch.

-  #

-  # Example:

-  #  yaml_plugin = MyPluginHost[:yaml, :default]

-  def fetch id, *args, &blk

-    plugin_hash.fetch validate_id(id), *args, &blk

-  end

-

-  # Returns the expected path to the plugin file for the given id.

-  def path_to plugin_id

-    File.join plugin_path, "#{plugin_id}.rb"

-  end

-

-  # Converts +id+ to a Symbol if it is a String,

-  # or returns +id+ if it already is a Symbol.

-  #

-  # Raises +ArgumentError+ for all other objects, or if the

-  # given String includes non-alphanumeric characters (\W).

-  def validate_id id

-    if id.is_a? Symbol or id.nil?

-      id

-    elsif id.is_a? String

-      if id[/\w+/] == id

-        id.to_sym

-      else

-        raise ArgumentError, "Invalid id: '#{id}' given."

-      end

-    else

-      raise ArgumentError,

-        "String or Symbol expected, but #{id.class} given."

-    end

-  end

-

-end

-

-

-# = Plugin

-#

-#  Plugins have to include this module.

-#

-#  IMPORTANT: use extend for this module.

-#

-#  Example: see PluginHost.

-module Plugin

-

-  def included mod

-    warn "#{name} should not be included. Use extend."

-  end

-

-  # Register this class for the given langs.

-  # Example:

-  #   class MyPlugin < PluginHost::BaseClass

-  #     register_for :my_id

-  #     ...

-  #   end

-  #

-  # See PluginHost.register.

-  def register_for *ids

-    plugin_host.register self, *ids

-  end

-

-  # The host for this Plugin class.

-  def plugin_host host = nil

-    if host and not host.is_a? PluginHost

-      raise ArgumentError,

-        "PluginHost expected, but #{host.class} given."

-    end

-    self.const_set :PLUGIN_HOST, host if host

-    self::PLUGIN_HOST

-  end

-

-  # Require some helper files.

-  #

-  # Example:

-  #

-  #  class MyPlugin < PluginHost::BaseClass

-  #     register_for :my_id

-  #     helper :my_helper

-  #

-  # The above example loads the file myplugin/my_helper.rb relative to the

-  # file in which MyPlugin was defined.

-  def helper *helpers

-    for helper in helpers

-      self::PLUGIN_HOST.require_helper plugin_id, helper.to_s

-    end

-  end

-

-  # Returns the pulgin id used by the engine.

-  def plugin_id

-    name[/[\w_]+$/].downcase

-  end

-

-end

-

-

-# Convenience method for plugin loading.

-# The syntax used is:

-#

-#  require_plugin '<Host ID>/<Plugin ID>'

-#

-# Returns the loaded plugin.

-def require_plugin path

-  host_id, plugin_id = path.split '/', 2

-  host = PluginHost.host_by_id(host_id)

-  raise PluginHost::HostNotFound,

-    "No host for #{host_id.inspect} found." unless host

-  host.load plugin_id

-end

+# = PluginHost
+#
+# $Id$
+#
+# A simple subclass plugin system.
+#
+#  Example:
+#    class Generators < PluginHost
+#      plugin_path 'app/generators'
+#    end
+#    
+#    class Generator
+#      extend Plugin
+#      PLUGIN_HOST = Generators
+#    end
+#    
+#    class FancyGenerator < Generator
+#      register_for :fancy
+#    end
+#
+#    Generators[:fancy]  #-> FancyGenerator
+#    # or
+#    require_plugin 'Generators/fancy'
+module PluginHost
+
+  # Raised if Encoders::[] fails because:
+  # * a file could not be found
+  # * the requested Encoder is not registered
+  PluginNotFound = Class.new Exception
+  HostNotFound = Class.new Exception
+
+  PLUGIN_HOSTS = []
+  PLUGIN_HOSTS_BY_ID = {}  # dummy hash
+
+  # Loads all plugins using all_plugin_names and load.
+  def load_all
+    for plugin in all_plugin_names
+      load plugin
+    end
+  end
+
+  # Returns the Plugin for +id+.
+  #
+  # Example:
+  #  yaml_plugin = MyPluginHost[:yaml]
+  def [] id, *args, &blk
+    plugin = validate_id(id)
+    begin
+      plugin = plugin_hash.[] plugin, *args, &blk
+    end while plugin.is_a? Symbol
+    plugin
+  end
+
+  # Alias for +[]+.
+  alias load []
+
+  def require_helper plugin_id, helper_name
+    path = path_to File.join(plugin_id, helper_name)
+    require path
+  end
+
+  class << self
+
+    # Adds the module/class to the PLUGIN_HOSTS list.
+    def extended mod
+      PLUGIN_HOSTS << mod
+    end
+
+    # Warns you that you should not #include this module.
+    def included mod
+      warn "#{name} should not be included. Use extend."
+    end
+
+    # Find the PluginHost for host_id.
+    def host_by_id host_id
+      unless PLUGIN_HOSTS_BY_ID.default_proc
+        ph = Hash.new do |h, a_host_id|
+          for host in PLUGIN_HOSTS
+            h[host.host_id] = host
+          end
+          h.fetch a_host_id, nil
+        end
+        PLUGIN_HOSTS_BY_ID.replace ph
+      end
+      PLUGIN_HOSTS_BY_ID[host_id]
+    end
+
+  end
+
+  # The path where the plugins can be found.
+  def plugin_path *args
+    unless args.empty?
+      @plugin_path = File.expand_path File.join(*args)
+      load_map
+    end
+    @plugin_path
+  end
+
+  # The host's ID.
+  #
+  # If PLUGIN_HOST_ID is not set, it is simply the class name.
+  def host_id
+    if self.const_defined? :PLUGIN_HOST_ID
+      self::PLUGIN_HOST_ID
+    else
+      name
+    end
+  end
+
+  # Map a plugin_id to another.
+  #
+  # Usage: Put this in a file plugin_path/_map.rb.
+  #
+  #  class MyColorHost < PluginHost
+  #    map :navy => :dark_blue,
+  #      :maroon => :brown,
+  #      :luna => :moon
+  #  end
+  def map hash
+    for from, to in hash
+      from = validate_id from
+      to = validate_id to
+      plugin_hash[from] = to unless plugin_hash.has_key? from
+    end
+  end
+
+  # Define the default plugin to use when no plugin is found
+  # for a given id.
+  #
+  # See also map.
+  #
+  #  class MyColorHost < PluginHost
+  #    map :navy => :dark_blue
+  #    default :gray
+  #  end
+  def default id
+    id = validate_id id
+    plugin_hash[nil] = id
+  end
+
+  # Every plugin must register itself for one or more
+  # +ids+ by calling register_for, which calls this method.
+  #
+  # See Plugin#register_for.
+  def register plugin, *ids
+    for id in ids
+      unless id.is_a? Symbol
+        raise ArgumentError,
+          "id must be a Symbol, but it was a #{id.class}"
+      end
+      plugin_hash[validate_id(id)] = plugin
+    end
+  end
+
+  # A Hash of plugion_id => Plugin pairs.
+  def plugin_hash
+    @plugin_hash ||= create_plugin_hash
+  end
+
+  # Returns an array of all .rb files in the plugin path.
+  #
+  # The extension .rb is not included.
+  def all_plugin_names
+    Dir[path_to('*')].select do |file|
+      File.basename(file)[/^(?!_)\w+\.rb$/]
+    end.map do |file|
+      File.basename file, '.rb'
+    end
+  end
+
+  # Makes a map of all loaded plugins.
+  def inspect
+    map = plugin_hash.dup
+    map.each do |id, plugin|
+      map[id] = plugin.to_s[/(?>[\w_]+)$/]
+    end
+    "#{name}[#{host_id}]#{map.inspect}"
+  end
+
+protected
+  # Created a new plugin list and stores it to @plugin_hash.
+  def create_plugin_hash
+    @plugin_hash =
+      Hash.new do |h, plugin_id|
+        id = validate_id(plugin_id)
+        path = path_to id
+        begin
+          require path
+        rescue LoadError => boom
+          if h.has_key? nil  # default plugin
+            h[id] = h[nil]
+          else
+            raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
+          end
+        else
+          # Plugin should have registered by now
+          unless h.has_key? id
+            raise PluginNotFound,
+              "No #{self.name} plugin for #{id.inspect} found in #{path}."
+          end
+        end
+        h[id]
+      end
+  end
+
+  # Loads the map file (see map).
+  #
+  # This is done automatically when plugin_path is called.
+  def load_map
+    mapfile = path_to '_map'
+    if File.exist? mapfile
+      require mapfile
+    elsif $DEBUG
+      warn 'no _map.rb found for %s' % name
+    end
+  end
+
+  # Returns the Plugin for +id+.
+  # Use it like Hash#fetch.
+  #
+  # Example:
+  #  yaml_plugin = MyPluginHost[:yaml, :default]
+  def fetch id, *args, &blk
+    plugin_hash.fetch validate_id(id), *args, &blk
+  end
+
+  # Returns the expected path to the plugin file for the given id.
+  def path_to plugin_id
+    File.join plugin_path, "#{plugin_id}.rb"
+  end
+
+  # Converts +id+ to a Symbol if it is a String,
+  # or returns +id+ if it already is a Symbol.
+  #
+  # Raises +ArgumentError+ for all other objects, or if the
+  # given String includes non-alphanumeric characters (\W).
+  def validate_id id
+    if id.is_a? Symbol or id.nil?
+      id
+    elsif id.is_a? String
+      if id[/\w+/] == id
+        id.to_sym
+      else
+        raise ArgumentError, "Invalid id: '#{id}' given."
+      end
+    else
+      raise ArgumentError,
+        "String or Symbol expected, but #{id.class} given."
+    end
+  end
+
+end
+
+
+# = Plugin
+#
+#  Plugins have to include this module.
+#
+#  IMPORTANT: use extend for this module.
+#
+#  Example: see PluginHost.
+module Plugin
+
+  def included mod
+    warn "#{name} should not be included. Use extend."
+  end
+
+  # Register this class for the given langs.
+  # Example:
+  #   class MyPlugin < PluginHost::BaseClass
+  #     register_for :my_id
+  #     ...
+  #   end
+  #
+  # See PluginHost.register.
+  def register_for *ids
+    plugin_host.register self, *ids
+  end
+
+  # The host for this Plugin class.
+  def plugin_host host = nil
+    if host and not host.is_a? PluginHost
+      raise ArgumentError,
+        "PluginHost expected, but #{host.class} given."
+    end
+    self.const_set :PLUGIN_HOST, host if host
+    self::PLUGIN_HOST
+  end
+
+  # Require some helper files.
+  #
+  # Example:
+  #
+  #  class MyPlugin < PluginHost::BaseClass
+  #     register_for :my_id
+  #     helper :my_helper
+  #
+  # The above example loads the file myplugin/my_helper.rb relative to the
+  # file in which MyPlugin was defined.
+  def helper *helpers
+    for helper in helpers
+      self::PLUGIN_HOST.require_helper plugin_id, helper.to_s
+    end
+  end
+
+  # Returns the pulgin id used by the engine.
+  def plugin_id
+    name[/[\w_]+$/].downcase
+  end
+
+end
+
+
+# Convenience method for plugin loading.
+# The syntax used is:
+#
+#  require_plugin '<Host ID>/<Plugin ID>'
+#
+# Returns the loaded plugin.
+def require_plugin path
+  host_id, plugin_id = path.split '/', 2
+  host = PluginHost.host_by_id(host_id)
+  raise PluginHost::HostNotFound,
+    "No host for #{host_id.inspect} found." unless host
+  host.load plugin_id
+end
diff --git a/lib/coderay/helpers/word_list.rb b/lib/coderay/helpers/word_list.rb
index dfbfaf2..cb67f7a 100644
--- a/lib/coderay/helpers/word_list.rb
+++ b/lib/coderay/helpers/word_list.rb
@@ -1,106 +1,106 @@
-# = WordList

-#

-# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy cYcnus de>

-#

-# License:: LGPL / ask the author

-# Version:: 1.0 (2006-Feb-3)

-#

-# A WordList is a Hash with some additional features.

-# It is intended to be used for keyword recognition.

-#

-# WordList is highly optimized to be used in Scanners,

-# typically to decide whether a given ident is a keyword.

-#

-# For case insensitive words use CaseIgnoringWordList.

-#

-# Example:

-#

-#  # define word arrays

-#  RESERVED_WORDS = %w[

-#    asm break case continue default do else

-#    ...

-#  ]

-#  

-#  PREDEFINED_TYPES = %w[

-#    int long short char void

-#    ...

-#  ]

-#  

-#  PREDEFINED_CONSTANTS = %w[

-#    EOF NULL ...

-#  ]

-#  

-#  # make a WordList

-#  IDENT_KIND = WordList.new(:ident).

-#    add(RESERVED_WORDS, :reserved).

-#    add(PREDEFINED_TYPES, :pre_type).

-#    add(PREDEFINED_CONSTANTS, :pre_constant)

-#

-#  ...

-#

-#  def scan_tokens tokens, options

-#    ...

-#    

-#    elsif scan(/[A-Za-z_][A-Za-z_0-9]*/)

-#      # use it

-#      kind = IDENT_KIND[match]

-#      ...

-class WordList < Hash

-

-  # Create a WordList for the given +words+.

-  #

-  # This WordList responds to [] with +true+, if the word is

-  # in +words+, and with +false+ otherwise.

-  def self.for words

-    new.add words

-  end

-

-  # Creates a new WordList with +default+ as default value.

-  def initialize default = false, &block

-    super default, &block

-  end

-

-  # Checks if a word is included.

-  def include? word

-    has_key? word

-  end

-

-  # Add words to the list and associate them with +kind+.

-  def add words, kind = true

-    words.each do |word|

-      self[word] = kind

-    end

-    self

-  end

-

-end

-

-

-# A CaseIgnoringWordList is like a WordList, only that

-# keys are compared case-insensitively.

-class CaseIgnoringWordList < WordList

-

-  # Creates a new WordList with +default+ as default value.

-  #

-  # Text case is ignored.

-  def initialize default = false, &block

-    block ||= proc do |h, k|

-      h[k] = h.fetch k.downcase, default

-    end

-    super default

-  end

-

-  # Checks if a word is included.

-  def include? word

-    has_key? word.downcase

-  end

-

-  # Add words to the list and associate them with +kind+.

-  def add words, kind = true

-    words.each do |word|

-      self[word.downcase] = kind

-    end

-    self

-  end

-

-end

+# = WordList
+#
+# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy cYcnus de>
+#
+# License:: LGPL / ask the author
+# Version:: 1.0 (2006-Feb-3)
+#
+# A WordList is a Hash with some additional features.
+# It is intended to be used for keyword recognition.
+#
+# WordList is highly optimized to be used in Scanners,
+# typically to decide whether a given ident is a keyword.
+#
+# For case insensitive words use CaseIgnoringWordList.
+#
+# Example:
+#
+#  # define word arrays
+#  RESERVED_WORDS = %w[
+#    asm break case continue default do else
+#    ...
+#  ]
+#  
+#  PREDEFINED_TYPES = %w[
+#    int long short char void
+#    ...
+#  ]
+#  
+#  PREDEFINED_CONSTANTS = %w[
+#    EOF NULL ...
+#  ]
+#  
+#  # make a WordList
+#  IDENT_KIND = WordList.new(:ident).
+#    add(RESERVED_WORDS, :reserved).
+#    add(PREDEFINED_TYPES, :pre_type).
+#    add(PREDEFINED_CONSTANTS, :pre_constant)
+#
+#  ...
+#
+#  def scan_tokens tokens, options
+#    ...
+#    
+#    elsif scan(/[A-Za-z_][A-Za-z_0-9]*/)
+#      # use it
+#      kind = IDENT_KIND[match]
+#      ...
+class WordList < Hash
+
+  # Create a WordList for the given +words+.
+  #
+  # This WordList responds to [] with +true+, if the word is
+  # in +words+, and with +false+ otherwise.
+  def self.for words
+    new.add words
+  end
+
+  # Creates a new WordList with +default+ as default value.
+  def initialize default = false, &block
+    super default, &block
+  end
+
+  # Checks if a word is included.
+  def include? word
+    has_key? word
+  end
+
+  # Add words to the list and associate them with +kind+.
+  def add words, kind = true
+    words.each do |word|
+      self[word] = kind
+    end
+    self
+  end
+
+end
+
+
+# A CaseIgnoringWordList is like a WordList, only that
+# keys are compared case-insensitively.
+class CaseIgnoringWordList < WordList
+
+  # Creates a new WordList with +default+ as default value.
+  #
+  # Text case is ignored.
+  def initialize default = false, &block
+    block ||= proc do |h, k|
+      h[k] = h.fetch k.downcase, default
+    end
+    super default
+  end
+
+  # Checks if a word is included.
+  def include? word
+    has_key? word.downcase
+  end
+
+  # Add words to the list and associate them with +kind+.
+  def add words, kind = true
+    words.each do |word|
+      self[word.downcase] = kind
+    end
+    self
+  end
+
+end
diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 83f73f2..62327c7 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -191,7 +191,7 @@ module CodeRay
       end
 
       # Scanner error with additional status information
-      def raise_inspect msg, tokens, state = nil, ambit = 30
+      def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
         raise ScanError, <<-EOE % [
 
 
diff --git a/lib/coderay/scanners/delphi.rb b/lib/coderay/scanners/delphi.rb
index c141874..fb37f67 100644
--- a/lib/coderay/scanners/delphi.rb
+++ b/lib/coderay/scanners/delphi.rb
@@ -101,6 +101,7 @@ module Scanners
             state = :initial
             next
           elsif scan(/\n/)
+            kind = :error
             state = :initial
           else
             raise "else case \' reached; %p not handled." % peek(1), tokens
@@ -114,7 +115,7 @@ module Scanners
         match ||= matched
         if $DEBUG and not kind
           raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
+            [[match, kind], line], tokens, state
         end
         raise_inspect 'Empty token', tokens unless match
 
diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 181e5d3..5f647d3 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -107,6 +107,7 @@ module Scanners
               kind = :tag
               state = :initial
             elsif scan(/./)
+              kind = :error
               state = :attribute
             end
 
@@ -137,6 +138,8 @@ module Scanners
               next
             elsif scan(/#{ENTITY}/ox)
               kind = :entity
+            elsif scan(/&/)
+              kind = :content
             elsif scan(/[\n>]/)
               tokens << [:close, :string]
               kind = :error
diff --git a/lib/coderay/scanners/nitro_xhtml.rb b/lib/coderay/scanners/nitro_xhtml.rb
index baef162..19354d9 100644
--- a/lib/coderay/scanners/nitro_xhtml.rb
+++ b/lib/coderay/scanners/nitro_xhtml.rb
@@ -113,6 +113,9 @@ module Scanners
 
         elsif entity = scan(/#{NITRO_ENTITY}/o)
           tokens << [entity, :entity]
+        
+        elsif scan(/%/)
+          tokens << [matched, :error]
 
         else
           raise_inspect 'else-case reached!', tokens
diff --git a/lib/coderay/styles/_map.rb b/lib/coderay/styles/_map.rb
index 4420470..52035fe 100644
--- a/lib/coderay/styles/_map.rb
+++ b/lib/coderay/styles/_map.rb
@@ -1,7 +1,7 @@
-module CodeRay

-module Styles

-

-  default :cycnus

-

-end

-end

+module CodeRay
+module Styles
+
+  default :cycnus
+
+end
+end
diff --git a/lib/coderay/styles/cycnus.rb b/lib/coderay/styles/cycnus.rb
index c65ab91..8464ba3 100644
--- a/lib/coderay/styles/cycnus.rb
+++ b/lib/coderay/styles/cycnus.rb
@@ -1,125 +1,125 @@
-module CodeRay

-module Styles

-

-  class Cycnus < Style

-

-    register_for :cycnus

-

-    code_background = '#f8f8f8'

-    numbers_background = '#def'

-    border_color = 'silver'

-    normal_color = '#100'

-

-    CSS_MAIN_STYLES = <<-MAIN

-.CodeRay {

-  background-color: #{code_background};

-  border: 1px solid #{border_color};

-  font-family: 'Courier New', 'Terminal', monospace;

-  color: #{normal_color};

-}

-.CodeRay pre { margin: 0px }

-

-div.CodeRay { }

-

-span.CodeRay { white-space: pre; border: 0px; padding: 2px }

-

-table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }

-table.CodeRay td { padding: 2px 4px; vertical-align: top }

-

-.CodeRay .line_numbers, .CodeRay .no {

-  background-color: #{numbers_background};

-  color: gray;

-  text-align: right;

-}

-.CodeRay .line_numbers tt { font-weight: bold }

-.CodeRay .no { padding: 0px 4px }

-.CodeRay .code { width: 100% }

-

-ol.CodeRay { font-size: 10pt }

-ol.CodeRay li { white-space: pre }

-

-.CodeRay .code pre { overflow: auto }

-    MAIN

-

-    TOKEN_COLORS = <<-'TOKENS'

-.af { color:#00C }

-.an { color:#007 }

-.av { color:#700 }

-.aw { color:#C00 }

-.bi { color:#509; font-weight:bold }

-.c  { color:#888 }

-

-.ch { color:#04D }

-.ch .k { color:#04D }

-.ch .dl { color:#039 }

-

-.cl { color:#B06; font-weight:bold }

-.co { color:#036; font-weight:bold }

-.cr { color:#0A0 }

-.cv { color:#369 }

-.df { color:#099; font-weight:bold }

-.di { color:#088; font-weight:bold }

-.dl { color:black }

-.do { color:#970 }

-.ds { color:#D42; font-weight:bold }

-.e  { color:#666; font-weight:bold }

-.en { color:#800; font-weight:bold }

-.er { color:#F00; background-color:#FAA }

-.ex { color:#F00; font-weight:bold }

-.fl { color:#60E; font-weight:bold }

-.fu { color:#06B; font-weight:bold }

-.gv { color:#d70; font-weight:bold }

-.hx { color:#058; font-weight:bold }

-.i  { color:#00D; font-weight:bold }

-.ic { color:#B44; font-weight:bold }

-

-.il { background: #eee }

-.il .il { background: #ddd }

-.il .il .il { background: #ccc }

-.il .dl { font-weight: bold ! important; color: #888 ! important }

-

-.in { color:#B2B; font-weight:bold }

-.iv { color:#33B }

-.la { color:#970; font-weight:bold }

-.lv { color:#963 }

-.oc { color:#40E; font-weight:bold }

-.on { color:#000; font-weight:bold }

-.op { }

-.pc { color:#038; font-weight:bold }

-.pd { color:#369; font-weight:bold }

-.pp { color:#579 }

-.pt { color:#339; font-weight:bold }

-.r  { color:#080; font-weight:bold }

-

-.rx { background-color:#fff0ff }

-.rx .k { color:#808 }

-.rx .dl { color:#404 }

-.rx .mod { color:#C2C }

-.rx .fu  { color:#404; font-weight: bold }

-

-.s  { background-color:#fff0f0 }

-.s  .s { background-color:#ffe0e0 }

-.s  .s  .s { background-color:#ffd0d0 }

-.s  .k { color:#D20 }

-.s  .dl { color:#710 }

-

-.sh { background-color:#f0fff0 }

-.sh .k { color:#2B2 }

-.sh .dl { color:#161 }

-

-.sy { color:#A60 }

-.sy .k { color:#A60 }

-.sy .dl { color:#630 }

-

-.ta { color:#070 }

-.tf { color:#070; font-weight:bold }

-.ts { color:#D70; font-weight:bold }

-.ty { color:#339; font-weight:bold }

-.v  { color:#036 }

-.xt { color:#444 }

-    TOKENS

-

-  end

-

-end

-end

+module CodeRay
+module Styles
+
+  class Cycnus < Style
+
+    register_for :cycnus
+
+    code_background = '#f8f8f8'
+    numbers_background = '#def'
+    border_color = 'silver'
+    normal_color = '#100'
+
+    CSS_MAIN_STYLES = <<-MAIN
+.CodeRay {
+  background-color: #{code_background};
+  border: 1px solid #{border_color};
+  font-family: 'Courier New', 'Terminal', monospace;
+  color: #{normal_color};
+}
+.CodeRay pre { margin: 0px }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
+table.CodeRay td { padding: 2px 4px; vertical-align: top }
+
+.CodeRay .line_numbers, .CodeRay .no {
+  background-color: #{numbers_background};
+  color: gray;
+  text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .no { padding: 0px 4px }
+.CodeRay .code { width: 100% }
+
+ol.CodeRay { font-size: 10pt }
+ol.CodeRay li { white-space: pre }
+
+.CodeRay .code pre { overflow: auto }
+    MAIN
+
+    TOKEN_COLORS = <<-'TOKENS'
+.af { color:#00C }
+.an { color:#007 }
+.av { color:#700 }
+.aw { color:#C00 }
+.bi { color:#509; font-weight:bold }
+.c  { color:#888 }
+
+.ch { color:#04D }
+.ch .k { color:#04D }
+.ch .dl { color:#039 }
+
+.cl { color:#B06; font-weight:bold }
+.co { color:#036; font-weight:bold }
+.cr { color:#0A0 }
+.cv { color:#369 }
+.df { color:#099; font-weight:bold }
+.di { color:#088; font-weight:bold }
+.dl { color:black }
+.do { color:#970 }
+.ds { color:#D42; font-weight:bold }
+.e  { color:#666; font-weight:bold }
+.en { color:#800; font-weight:bold }
+.er { color:#F00; background-color:#FAA }
+.ex { color:#F00; font-weight:bold }
+.fl { color:#60E; font-weight:bold }
+.fu { color:#06B; font-weight:bold }
+.gv { color:#d70; font-weight:bold }
+.hx { color:#058; font-weight:bold }
+.i  { color:#00D; font-weight:bold }
+.ic { color:#B44; font-weight:bold }
+
+.il { background: #eee }
+.il .il { background: #ddd }
+.il .il .il { background: #ccc }
+.il .dl { font-weight: bold ! important; color: #888 ! important }
+
+.in { color:#B2B; font-weight:bold }
+.iv { color:#33B }
+.la { color:#970; font-weight:bold }
+.lv { color:#963 }
+.oc { color:#40E; font-weight:bold }
+.on { color:#000; font-weight:bold }
+.op { }
+.pc { color:#038; font-weight:bold }
+.pd { color:#369; font-weight:bold }
+.pp { color:#579 }
+.pt { color:#339; font-weight:bold }
+.r  { color:#080; font-weight:bold }
+
+.rx { background-color:#fff0ff }
+.rx .k { color:#808 }
+.rx .dl { color:#404 }
+.rx .mod { color:#C2C }
+.rx .fu  { color:#404; font-weight: bold }
+
+.s  { background-color:#fff0f0 }
+.s  .s { background-color:#ffe0e0 }
+.s  .s  .s { background-color:#ffd0d0 }
+.s  .k { color:#D20 }
+.s  .dl { color:#710 }
+
+.sh { background-color:#f0fff0 }
+.sh .k { color:#2B2 }
+.sh .dl { color:#161 }
+
+.sy { color:#A60 }
+.sy .k { color:#A60 }
+.sy .dl { color:#630 }
+
+.ta { color:#070 }
+.tf { color:#070; font-weight:bold }
+.ts { color:#D70; font-weight:bold }
+.ty { color:#339; font-weight:bold }
+.v  { color:#036 }
+.xt { color:#444 }
+    TOKENS
+
+  end
+
+end
+end
diff --git a/lib/coderay/styles/murphy.rb b/lib/coderay/styles/murphy.rb
index f2fa798..1079f62 100644
--- a/lib/coderay/styles/murphy.rb
+++ b/lib/coderay/styles/murphy.rb
@@ -1,119 +1,119 @@
-module CodeRay

-module Styles

-

-  class Murphy < Style

-

-    register_for :murphy

-

-    code_background = '#001129'

-    numbers_background = code_background

-    border_color = 'silver'

-    normal_color = '#C0C0C0'

-

-    CSS_MAIN_STYLES = <<-MAIN

-.CodeRay {

-  background-color: #{code_background};

-  border: 1px solid #{border_color};

-  font-family: 'Courier New', 'Terminal', monospace;

-  color: #{normal_color};

-}

-.CodeRay pre { margin: 0px; }

-

-div.CodeRay { }

-

-span.CodeRay { white-space: pre; border: 0px; padding: 2px; }

-

-table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }

-table.CodeRay td { padding: 2px 4px; vertical-align: top; }

-

-.CodeRay .line_numbers, .CodeRay .no {

-  background-color: #{numbers_background};

-  color: gray;

-  text-align: right;

-}

-.CodeRay .line_numbers tt { font-weight: bold; }

-.CodeRay .no { padding: 0px 4px; }

-.CodeRay .code { width: 100%; }

-

-ol.CodeRay { font-size: 10pt; }

-ol.CodeRay li { white-space: pre; }

-

-.CodeRay .code pre { overflow: auto; }

-    MAIN

-

-    TOKEN_COLORS = <<-'TOKENS'

-.af { color:#00C; }

-.an { color:#007; }

-.av { color:#700; }

-.aw { color:#C00; }

-.bi { color:#509; font-weight:bold; }

-.c  { color:#666; }

-

-.ch { color:#88F; }

-.ch .k { color:#04D; }

-.ch .dl { color:#039; }

-

-.cl { color:#e9e; font-weight:bold; }

-.co { color:#5ED; font-weight:bold; }

-.cr { color:#0A0; }

-.cv { color:#ccf; }

-.df { color:#099; font-weight:bold; }

-.di { color:#088; font-weight:bold; }

-.dl { color:black; }

-.do { color:#970; }

-.ds { color:#D42; font-weight:bold; }

-.e  { color:#666; font-weight:bold; }

-.er { color:#F00; background-color:#FAA; }

-.ex { color:#F00; font-weight:bold; }

-.fl { color:#60E; font-weight:bold; }

-.fu { color:#5ed; font-weight:bold; }

-.gv { color:#f84; }

-.hx { color:#058; font-weight:bold; }

-.i  { color:#66f; font-weight:bold; }

-.ic { color:#B44; font-weight:bold; }

-.il { }

-.in { color:#B2B; font-weight:bold; }

-.iv { color:#aaf; }

-.la { color:#970; font-weight:bold; }

-.lv { color:#963; }

-.oc { color:#40E; font-weight:bold; }

-.on { color:#000; font-weight:bold; }

-.op { }

-.pc { color:#08f; font-weight:bold; }

-.pd { color:#369; font-weight:bold; }

-.pp { color:#579; }

-.pt { color:#66f; font-weight:bold; }

-.r  { color:#5de; font-weight:bold; }

-

-.rx { background-color:#221133; }

-.rx .k { color:#f8f; }

-.rx .dl { color:#f0f; }

-.rx .mod { color:#f0b; }

-.rx .fu  { color:#404; font-weight: bold; }

-

-.s  { background-color:#331122; }

-.s  .s { background-color:#ffe0e0; }

-.s  .s  .s { background-color:#ffd0d0; }

-.s  .k { color:#F88; }

-.s  .dl { color:#f55; }

-

-.sh { background-color:#f0fff0; }

-.sh .k { color:#2B2; }

-.sh .dl { color:#161; }

-

-.sy { color:#Fc8; }

-.sy .k { color:#Fc8; }

-.sy .dl { color:#F84; }

-

-.ta { color:#070; }

-.tf { color:#070; font-weight:bold; }

-.ts { color:#D70; font-weight:bold; }

-.ty { color:#339; font-weight:bold; }

-.v  { color:#036; }

-.xt { color:#444; }

-    TOKENS

-

-  end

-

-end

-end

+module CodeRay
+module Styles
+
+  class Murphy < Style
+
+    register_for :murphy
+
+    code_background = '#001129'
+    numbers_background = code_background
+    border_color = 'silver'
+    normal_color = '#C0C0C0'
+
+    CSS_MAIN_STYLES = <<-MAIN
+.CodeRay {
+  background-color: #{code_background};
+  border: 1px solid #{border_color};
+  font-family: 'Courier New', 'Terminal', monospace;
+  color: #{normal_color};
+}
+.CodeRay pre { margin: 0px; }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line_numbers, .CodeRay .no {
+  background-color: #{numbers_background};
+  color: gray;
+  text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold; }
+.CodeRay .no { padding: 0px 4px; }
+.CodeRay .code { width: 100%; }
+
+ol.CodeRay { font-size: 10pt; }
+ol.CodeRay li { white-space: pre; }
+
+.CodeRay .code pre { overflow: auto; }
+    MAIN
+
+    TOKEN_COLORS = <<-'TOKENS'
+.af { color:#00C; }
+.an { color:#007; }
+.av { color:#700; }
+.aw { color:#C00; }
+.bi { color:#509; font-weight:bold; }
+.c  { color:#666; }
+
+.ch { color:#88F; }
+.ch .k { color:#04D; }
+.ch .dl { color:#039; }
+
+.cl { color:#e9e; font-weight:bold; }
+.co { color:#5ED; font-weight:bold; }
+.cr { color:#0A0; }
+.cv { color:#ccf; }
+.df { color:#099; font-weight:bold; }
+.di { color:#088; font-weight:bold; }
+.dl { color:black; }
+.do { color:#970; }
+.ds { color:#D42; font-weight:bold; }
+.e  { color:#666; font-weight:bold; }
+.er { color:#F00; background-color:#FAA; }
+.ex { color:#F00; font-weight:bold; }
+.fl { color:#60E; font-weight:bold; }
+.fu { color:#5ed; font-weight:bold; }
+.gv { color:#f84; }
+.hx { color:#058; font-weight:bold; }
+.i  { color:#66f; font-weight:bold; }
+.ic { color:#B44; font-weight:bold; }
+.il { }
+.in { color:#B2B; font-weight:bold; }
+.iv { color:#aaf; }
+.la { color:#970; font-weight:bold; }
+.lv { color:#963; }
+.oc { color:#40E; font-weight:bold; }
+.on { color:#000; font-weight:bold; }
+.op { }
+.pc { color:#08f; font-weight:bold; }
+.pd { color:#369; font-weight:bold; }
+.pp { color:#579; }
+.pt { color:#66f; font-weight:bold; }
+.r  { color:#5de; font-weight:bold; }
+
+.rx { background-color:#221133; }
+.rx .k { color:#f8f; }
+.rx .dl { color:#f0f; }
+.rx .mod { color:#f0b; }
+.rx .fu  { color:#404; font-weight: bold; }
+
+.s  { background-color:#331122; }
+.s  .s { background-color:#ffe0e0; }
+.s  .s  .s { background-color:#ffd0d0; }
+.s  .k { color:#F88; }
+.s  .dl { color:#f55; }
+
+.sh { background-color:#f0fff0; }
+.sh .k { color:#2B2; }
+.sh .dl { color:#161; }
+
+.sy { color:#Fc8; }
+.sy .k { color:#Fc8; }
+.sy .dl { color:#F84; }
+
+.ta { color:#070; }
+.tf { color:#070; font-weight:bold; }
+.ts { color:#D70; font-weight:bold; }
+.ty { color:#339; font-weight:bold; }
+.v  { color:#036; }
+.xt { color:#444; }
+    TOKENS
+
+  end
+
+end
+end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index c8c62e0..b0ce70e 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -170,7 +170,7 @@ module CodeRay
       print ' Tokens#optimize: before: %d - ' % size if $DEBUG
       last_kind = last_text = nil
       new = self.class.new
-      each do |text, kind|
+      for text, kind in self
         if text.is_a? String
           if kind == last_kind
             last_text << text
@@ -195,6 +195,51 @@ module CodeRay
     def optimize!
       replace optimize
     end
+    
+    # Ensure that all :open tokens have a correspondent :close one.
+    #
+    # TODO: Test this!
+    def fix
+      # Check token nesting using a stack of kinds.
+      opened = []
+      for token, kind in self
+        if token == :open
+          opened.push kind
+        elsif token == :close
+          expected = opened.pop
+          if kind != expected
+            # Unexpected :close; decide what to do based on the kind:
+            # - token was opened earlier: also close tokens in between
+            # - token was never opened: delete the :close (skip with next)
+            next unless opened.rindex expected
+            tokens << [:close, kind] until (kind = opened.pop) == expected
+          end
+        end
+        tokens << [token, kind]
+      end
+      # Close remaining opened tokens
+      tokens << [:close, kind] while kind = opened.pop
+      tokens
+    end
+    
+    def fix!
+      replace fix
+    end
+    
+    # Makes sure that:
+    # - newlines are single tokens
+    #   (which means all other token are single-line)
+    # - there are no open tokens at the end the line
+    #
+    # This makes it simple for encoders that work line-oriented,
+    # like HTML with list-style numeration.
+    def split_into_lines
+      raise NotImplementedError
+    end
+
+    def split_into_lines!
+      replace split_into_lines
+    end
 
     # Dumps the object into a String that can be saved
     # in files or databases.
@@ -304,7 +349,8 @@ module CodeRay
 
     # This method is not implemented due to speed reasons. Use Tokens.
     def text_size
-      raise NotImplementedError, 'This method is not implemented due to speed reasons.'
+      raise NotImplementedError,
+        'This method is not implemented due to speed reasons.'
     end
 
     # A TokenStream cannot be dumped. Use Tokens.