Merge branch 'master' into lua-scanner

Conflicts:
	lib/coderay/token_kinds.rb

23 files changed, 558 insertions, 248 deletions

diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 95d6138..c03d3fb 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -24,11 +24,12 @@ module Encoders
     end
     
     def text_token text, kind
+      raise 'empty token' if $CODERAY_DEBUG && text.empty?
       if kind == :space
         @out << text
       else
         # TODO: Escape (
-        text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
+        text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/)
         @out << kind.to_s << '(' << text << ')'
       end
     end
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 635a4d8..b897f5e 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -126,22 +126,21 @@ module Encoders
     
   protected
     
-    HTML_ESCAPE = {  #:nodoc:
-      '&' => '&amp;',
-      '"' => '&quot;',
-      '>' => '&gt;',
-      '<' => '&lt;',
-    }
+    def self.make_html_escape_hash
+      {
+        '&' => '&amp;',
+        '"' => '&quot;',
+        '>' => '&gt;',
+        '<' => '&lt;',
+        # "\t" => will be set to ' ' * options[:tab_width] during setup
+      }.tap do |hash|
+        # Escape ASCII control codes except \x9 == \t and \xA == \n.
+        (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
+      end
+    end
     
-    # This was to prevent illegal HTML.
-    # Strange chars should still be avoided in codes.
-    evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
-    evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
-    #ansi_chars = Array(0x7f..0xff)
-    #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
-    # \x9 (\t) and \xA (\n) not included
-    #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
-    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
+    HTML_ESCAPE = make_html_escape_hash
+    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
     
     TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
       h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
@@ -172,59 +171,22 @@ module Encoders
     def setup options
       super
       
+      check_options! options
+      
       if options[:wrap] || options[:line_numbers]
         @real_out = @out
         @out = ''
       end
       
-      options[:break_lines] = true if options[:line_numbers] == :inline
-      
       @break_lines = (options[:break_lines] == true)
       
-      @HTML_ESCAPE = HTML_ESCAPE.dup
-      @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
       
       @opened = []
       @last_opened = nil
       @css = CSS.new options[:style]
       
-      hint = options[:hint]
-      if hint && ![:debug, :info, :info_long].include?(hint)
-        raise ArgumentError, "Unknown value %p for :hint; \
-          expected :info, :info_long, :debug, false, or nil." % hint
-      end
-      
-      css_classes = TokenKinds
-      case options[:css]
-      when :class
-        @span_for_kind = Hash.new do |h, k|
-          if k.is_a? ::Symbol
-            kind = k_dup = k
-          else
-            kind = k.first
-            k_dup = k.dup
-          end
-          if kind != :space && (hint || css_class = css_classes[kind])
-            title = HTML.token_path_to_hint hint, k if hint
-            css_class ||= css_classes[kind]
-            h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
-          else
-            h[k_dup] = nil
-          end
-        end
-      when :style
-        @span_for_kind = Hash.new do |h, k|
-          kind = k.is_a?(Symbol) ? k : k.first
-          h[k.is_a?(Symbol) ? k : k.dup] =
-            if kind != :space && (hint || css_classes[kind])
-              title = HTML.token_path_to_hint hint, k if hint
-              style = @css.get_style Array(k).map { |c| css_classes[c] }
-              "<span#{title}#{" style=\"#{style}\"" if style}>"
-            end
-        end
-      else
-        raise ArgumentError, "Unknown value %p for :css." % options[:css]
-      end
+      @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
       
       @set_last_opened = options[:hint] || options[:css] == :style
     end
@@ -255,20 +217,10 @@ module Encoders
   public
     
     def text_token text, kind
-      if text =~ /#{HTML_ESCAPE_PATTERN}/o
-        text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
-      end
+      style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
       
-      style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
-      
-      if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0
-        close = '</span>' * c
-        reopen = ''
-        @opened.each_with_index do |k, index|
-          reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>')
-        end
-        text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}")
-      end
+      text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
+      text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
       
       if style
         @out << style << text << '</span>'
@@ -279,25 +231,19 @@ module Encoders
     
     # token groups, eg. strings
     def begin_group kind
-      @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
+      @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
       @opened << kind
       @last_opened = kind if @set_last_opened
     end
     
     def end_group kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a token (%p) ' \
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
-      end
-      if @opened.pop
-        @out << '</span>'
-        @last_opened = @opened.last if @last_opened
-      end
+      check_group_nesting 'token group', kind if $CODERAY_DEBUG
+      close_span
     end
     
     # whole lines to be highlighted, eg. a deleted line in a diff
     def begin_line kind
-      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
+      if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
         if style['class="']
           @out << style.sub('class="', 'class="line ')
         else
@@ -311,16 +257,71 @@ module Encoders
     end
     
     def end_line kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a line (%p) ' \
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+      check_group_nesting 'line', kind if $CODERAY_DEBUG
+      close_span
+    end
+    
+  protected
+    
+    def check_options! options
+      unless [false, nil, :debug, :info, :info_long].include? options[:hint]
+        raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
+      end
+      
+      unless [:class, :style].include? options[:css]
+        raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
+      end
+      
+      options[:break_lines] = true if options[:line_numbers] == :inline
+    end
+    
+    def css_class_for_kinds kinds
+      TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
+    end
+    
+    def style_for_kinds kinds
+      css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
+      @css.get_style_for_css_classes css_classes
+    end
+    
+    def make_span_for_kinds method, hint
+      Hash.new do |h, kinds|
+        h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
+          css_class = css_class_for_kinds(kinds)
+          title     = HTML.token_path_to_hint hint, kinds if hint
+          
+          if css_class || title
+            if method == :style
+              style = style_for_kinds(kinds)
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
+            else
+              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
+            end
+          end
+        end
+      end
+    end
+    
+    def check_group_nesting name, kind
+      if @opened.empty? || @opened.last != kind
+        warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
       end
+    end
+    
+    def break_lines text, style
+      reopen = ''
+      @opened.each_with_index do |k, index|
+        reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
+      end
+      text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
+    end
+    
+    def close_span
       if @opened.pop
         @out << '</span>'
         @last_opened = @opened.last if @last_opened
       end
     end
-    
   end
   
 end
diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb
index 6de4b46..164d7f8 100644
--- a/lib/coderay/encoders/html/css.rb
+++ b/lib/coderay/encoders/html/css.rb
@@ -11,7 +11,7 @@ module Encoders
       end
 
       def initialize style = :default
-        @classes = Hash.new
+        @styles = Hash.new
         style = CSS.load_stylesheet style
         @stylesheet = [
           style::CSS_MAIN_STYLES,
@@ -20,12 +20,12 @@ module Encoders
         parse style::TOKEN_COLORS
       end
 
-      def get_style styles
-        cl = @classes[styles.first]
+      def get_style_for_css_classes css_classes
+        cl = @styles[css_classes.first]
         return '' unless cl
         style = ''
-        1.upto styles.size do |offset|
-          break if style = cl[styles[offset .. -1]]
+        1.upto css_classes.size do |offset|
+          break if style = cl[css_classes[offset .. -1]]
         end
         # warn 'Style not found: %p' % [styles] if style.empty?
         return style
@@ -52,8 +52,8 @@ module Encoders
           for selector in selectors.split(',')
             classes = selector.scan(/[-\w]+/)
             cl = classes.pop
-            @classes[cl] ||= Hash.new
-            @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
+            @styles[cl] ||= Hash.new
+            @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
           end
         end
       end
diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index a0ceb3c..9894b91 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -19,73 +19,73 @@ module CodeRay
       register_for :terminal
       
       TOKEN_COLORS = {
-        :annotation => '35',
-        :attribute_name => '33',
-        :attribute_value => '31',
-        :binary => '1;35',
+        :annotation => "\e[35m",
+        :attribute_name => "\e[33m",
+        :attribute_value => "\e[31m",
+        :binary => "\e[1;35m",
         :char => {
-          :self => '36', :delimiter => '1;34'
+          :self => "\e[36m", :delimiter => "\e[1;34m"
         },
-        :class => '1;35',
-        :class_variable => '36',
-        :color => '32',
-        :comment => '37',
-        :complex => '1;34',
-        :constant => ['1;34', '4'],
-        :decoration => '35',
-        :definition => '1;32',
-        :directive => ['32', '4'],
-        :doc => '46',
-        :doctype => '1;30',
-        :doc_string => ['31', '4'],
-        :entity => '33',
-        :error => ['1;33', '41'],
-        :exception => '1;31',
-        :float => '1;35',
-        :function => '1;34',
-        :global_variable => '42',
-        :hex => '1;36',
-        :include => '33',
-        :integer => '1;34',
-        :key => '35',
-        :label => '1;15',
-        :local_variable => '33',
-        :octal => '1;35',
-        :operator_name => '1;29',
-        :predefined_constant => '1;36',
-        :predefined_type => '1;30',
-        :predefined => ['4', '1;34'],
-        :preprocessor => '36',
-        :pseudo_class => '1;34',
+        :class => "\e[1;35m",
+        :class_variable => "\e[36m",
+        :color => "\e[32m",
+        :comment => "\e[37m",
+        :complex => "\e[1;34m",
+        :constant => "\e[1;34m\e[4m",
+        :decoration => "\e[35m",
+        :definition => "\e[1;32m",
+        :directive => "\e[32m\e[4m",
+        :doc => "\e[46m",
+        :doctype => "\e[1;30m",
+        :docstring => "\e[31m\e[4m",
+        :entity => "\e[33m",
+        :error => "\e[1;33m\e[41m",
+        :exception => "\e[1;31m",
+        :float => "\e[1;35m",
+        :function => "\e[1;34m",
+        :global_variable => "\e[42m",
+        :hex => "\e[1;36m",
+        :include => "\e[33m",
+        :integer => "\e[1;34m",
+        :key => "\e[35m",
+        :label => "\e[1;15m",
+        :local_variable => "\e[33m",
+        :octal => "\e[1;35m",
+        :operator_name => "\e[1;29m",
+        :predefined_constant => "\e[1;36m",
+        :predefined_type => "\e[1;30m",
+        :predefined => "\e[4m\e[1;34m",
+        :preprocessor => "\e[36m",
+        :pseudo_class => "\e[1;34m",
         :regexp => {
-          :self => '31',
-          :content => '31',
-          :delimiter => '1;29',
-          :modifier => '35',
+          :self => "\e[31m",
+          :content => "\e[31m",
+          :delimiter => "\e[1;29m",
+          :modifier => "\e[35m",
         },
-        :reserved => '1;31',
+        :reserved => "\e[1;31m",
         :shell => {
-          :self => '42',
-          :content => '1;29',
-          :delimiter => '37',
+          :self => "\e[42m",
+          :content => "\e[1;29m",
+          :delimiter => "\e[37m",
         },
         :string => {
-          :self => '32',
-          :modifier => '1;32',
-          :escape => '1;36',
-          :delimiter => '1;32',
-          :char => '1;36',
+          :self => "\e[32m",
+          :modifier => "\e[1;32m",
+          :escape => "\e[1;36m",
+          :delimiter => "\e[1;32m",
+          :char => "\e[1;36m",
         },
-        :symbol => '1;32',
-        :tag => '1;34',
-        :type => '1;34',
-        :value => '36',
-        :variable => '1;34',
+        :symbol => "\e[1;32m",
+        :tag => "\e[1;34m",
+        :type => "\e[1;34m",
+        :value => "\e[36m",
+        :variable => "\e[1;34m",
         
-        :insert => '42',
-        :delete => '41',
-        :change => '44',
-        :head => '45'
+        :insert => "\e[42m",
+        :delete => "\e[41m",
+        :change => "\e[44m",
+        :head => "\e[45m"
       }
       TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
@@ -114,10 +114,10 @@ module CodeRay
             end
           end
           
-          @out << ansi_colorize(color)
-          @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color))
-          @out << ansi_clear
-          @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self]
+          @out << color
+          @out << text.gsub("\n", "\e[0m\n" + color)
+          @out << "\e[0m"
+          @out << @subcolors[:self] if @subcolors
         else
           @out << text
         end
@@ -134,7 +134,7 @@ module CodeRay
           # nothing to close
         else
           @opened.pop
-          @out << ansi_clear
+          @out << "\e[0m"
           @out << open_token(@opened.last)
         end
       end
@@ -146,7 +146,7 @@ module CodeRay
           @opened.pop
           # whole lines to be highlighted,
           # eg. added/modified/deleted lines in a diff
-          @out << "\t" * 100 + ansi_clear
+          @out << (@line_filler ||= "\t" * 100 + "\e[0m")
           @out << open_token(@opened.last)
         end
       end
@@ -157,23 +157,16 @@ module CodeRay
         if color = TOKEN_COLORS[kind]
           if Hash === color
             @subcolors = color
-            ansi_colorize(color[:self]) if color[:self]
+            color[:self]
           else
             @subcolors = {}
-            ansi_colorize(color)
+            color
           end
         else
           @subcolors = nil
           ''
         end
       end
-      
-      def ansi_colorize(color)
-        Array(color).map { |c| "\e[#{c}m" }.join
-      end
-      def ansi_clear
-        ansi_colorize(0)
-      end
     end
   end
-end
\ No newline at end of file
+end
diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb
index 637001b..a5d83ff 100644
--- a/lib/coderay/helpers/file_type.rb
+++ b/lib/coderay/helpers/file_type.rb
@@ -99,6 +99,7 @@ module CodeRay
       'mab'      => :ruby,
       'pas'      => :delphi,
       'patch'    => :diff,
+      'phtml'    => :php,
       'php'      => :php,
       'php3'     => :php,
       'php4'     => :php,
@@ -116,10 +117,10 @@ module CodeRay
       'rpdf'     => :ruby,
       'ru'       => :ruby,
       'rxml'     => :ruby,
-      # 'sch'      => :scheme,
+      'sass'     => :sass,
       'sql'      => :sql,
-      # 'ss'       => :scheme,
       'tmproj'   => :xml,
+      'xaml'     => :xml,
       'xhtml'    => :html,
       'xml'      => :xml,
       'yaml'     => :yaml,
diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index 137c1ab..d14c5a9 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -131,7 +131,7 @@ module CodeRay
     
     # A Hash of plugion_id => Plugin pairs.
     def plugin_hash
-      @plugin_hash ||= make_plugin_hash
+      @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map }
     end
     
     # Returns an array of all .rb files in the plugin path.
@@ -158,7 +158,6 @@ module CodeRay
     # This is done automatically when plugin_path is called.
     def load_plugin_map
       mapfile = path_to '_map'
-      @plugin_map_loaded = true
       if File.exist? mapfile
         require mapfile
         true
@@ -171,23 +170,16 @@ module CodeRay
     
     # Return a plugin hash that automatically loads plugins.
     def make_plugin_hash
-      @plugin_map_loaded ||= false
       Hash.new do |h, plugin_id|
         id = validate_id(plugin_id)
         path = path_to id
         begin
           require path
         rescue LoadError => boom
-          if @plugin_map_loaded
-            if h.has_key?(:default)
-              warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]]
-              h[:default]
-            else
-              raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
-            end
+          if h.has_key?(:default)
+            h[:default]
           else
-            load_plugin_map
-            h[plugin_id]
+            raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
           end
         else
           # Plugin should have registered by now
@@ -271,7 +263,6 @@ module CodeRay
     end
     
     def aliases
-      plugin_host.load_plugin_map
       plugin_host.plugin_hash.inject [] do |aliases, (key, _)|
         aliases << key if plugin_host[key] == self
         aliases
diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 907cf00..b3f7e17 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -182,16 +182,9 @@ module CodeRay
       # Scan the code and returns all tokens in a Tokens object.
       def tokenize source = nil, options = {}
         options = @options.merge(options)
-        @tokens = options[:tokens] || @tokens || Tokens.new
-        @tokens.scanner = self if @tokens.respond_to? :scanner=
-        case source
-        when Array
-          self.string = self.class.normalize(source.join)
-        when nil
-          reset
-        else
-          self.string = self.class.normalize(source)
-        end
+        
+        set_tokens_from_options options
+        set_string_from_source source
         
         begin
           scan_tokens @tokens, options
@@ -261,6 +254,22 @@ module CodeRay
       def setup  # :doc:
       end
       
+      def set_string_from_source source
+        case source
+        when Array
+          self.string = self.class.normalize(source.join)
+        when nil
+          reset
+        else
+          self.string = self.class.normalize(source)
+        end
+      end
+      
+      def set_tokens_from_options options
+        @tokens = options[:tokens] || @tokens || Tokens.new
+        @tokens.scanner = self if @tokens.respond_to? :scanner=
+      end
+      
       # This is the central method, and commonly the only one a
       # subclass implements.
       #
@@ -277,19 +286,15 @@ module CodeRay
         @binary_string = nil if defined? @binary_string
       end
       
-      # Scanner error with additional status information
-      def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller
-        raise ScanError, <<-EOE % [
+      SCAN_ERROR_MESSAGE = <<-MESSAGE
 
 
-***ERROR in %s: %s (after %d tokens)
+***ERROR in %s: %s (after %s tokens)
 
 tokens:
 %s
 
-current line: %d  column: %d  pos: %d
-matched: %p  state: %p
-bol? = %p,  eos? = %p
+%s
 
 surrounding code:
 %p  ~~  %p
@@ -297,16 +302,43 @@ surrounding code:
 
 ***ERROR***
 
-        EOE
-          File.basename(caller[0]),
-          msg,
-          tokens.respond_to?(:size) ? tokens.size : 0,
-          tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '',
-          line, column, pos,
-          matched, state, bol?, eos?,
+      MESSAGE
+      
+      def raise_inspect_arguments message, tokens, state, ambit
+        return File.basename(caller[0]),
+          message,
+          tokens_size(tokens),
+          tokens_last(tokens, 10).map(&:inspect).join("\n"),
+          scanner_state_info(state),
           binary_string[pos - ambit, ambit],
-          binary_string[pos, ambit],
-        ], backtrace
+          binary_string[pos, ambit]
+      end
+      
+      SCANNER_STATE_INFO = <<-INFO
+current line: %d  column: %d  pos: %d
+matched: %p  state: %p
+bol?: %p,  eos?: %p
+      INFO
+      
+      def scanner_state_info state
+        SCANNER_STATE_INFO % [
+          line, column, pos,
+          matched, state || 'No state given!',
+          bol?, eos?,
+        ]
+      end
+      
+      # Scanner error with additional status information
+      def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller
+        raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace
+      end
+      
+      def tokens_size tokens
+        tokens.size if tokens.respond_to?(:size)
+      end
+      
+      def tokens_last tokens, n
+        tokens.respond_to?(:last) ? tokens.last(n) : []
       end
       
       # Shorthand for scan_until(/\z/).
diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb
index 8d24b99..84b6e8e 100644
--- a/lib/coderay/scanners/c.rb
+++ b/lib/coderay/scanners/c.rb
@@ -148,7 +148,7 @@ module Scanners
             encoder.text_token match, :char
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group :string
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
             label_expected = false
           else
diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb
index 9da62f4..e61f56f 100644
--- a/lib/coderay/scanners/cpp.rb
+++ b/lib/coderay/scanners/cpp.rb
@@ -160,7 +160,7 @@ module Scanners
             encoder.text_token match, :char
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group :string
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
             label_expected = false
           else
diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index 7b731ef..732f9c5 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -7,27 +7,25 @@ module Scanners
     
     KINDS_NOT_LOC = [
       :comment,
-      :class, :pseudo_class, :type,
-      :constant, :directive,
+      :class, :pseudo_class, :tag,
+      :id, :directive,
       :key, :value, :operator, :color, :float, :string,
-      :error, :important,
+      :error, :important, :type,
     ]  # :nodoc:
     
     module RE  # :nodoc:
       Hex = /[0-9a-fA-F]/
-      Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
-      Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
-      NMChar = /[-_a-zA-Z0-9]|#{Escape}/
-      NMStart = /[_a-zA-Z]|#{Escape}/
-      NL = /\r\n|\r|\n|\f/
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
+      Unicode = /\\#{Hex}{1,6}\b/ # differs from standard because it allows uppercase hex too
+      Escape = /#{Unicode}|\\[^\n0-9a-fA-F]/
+      NMChar = /[-_a-zA-Z0-9]/
+      NMStart = /[_a-zA-Z]/
+      String1 = /"(?:[^\n\\"]+|\\\n|#{Escape})*"?/  # TODO: buggy regexp
+      String2 = /'(?:[^\n\\']+|\\\n|#{Escape})*'?/  # TODO: buggy regexp
       String = /#{String1}|#{String2}/
       
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
-      Color = /#{HexColor}/
       
-      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
@@ -35,16 +33,15 @@ module Scanners
       
       reldimensions = %w[em ex px]
       absdimensions = %w[in cm mm pt pc]
-      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s]))
+      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s dpi dppx deg]))
       
       Dimension = /#{Num}#{Unit}/
       
-      Comment = %r! /\* (?: .*? \*/ | .* ) !mx
-      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
+      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n]|\\\))*\)?/
       
-      Id = /##{Name}/
+      Id = /(?!#{HexColor}\b(?!-))##{Name}/
       Class = /\.#{Name}/
-      PseudoClass = /:#{Name}/
+      PseudoClass = /::?#{Ident}/
       AttributeSelector = /\[[^\]]*\]?/
     end
     
@@ -52,7 +49,7 @@ module Scanners
     
     def setup
       @state = :initial
-      @value_expected = nil
+      @value_expected = false
     end
     
     def scan_tokens encoder, options
@@ -67,13 +64,13 @@ module Scanners
         elsif case states.last
           when :initial, :media
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
               next
             elsif match = scan(RE::Id)
-              encoder.text_token match, :constant
+              encoder.text_token match, :id
               next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
@@ -158,7 +155,7 @@ module Scanners
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
           
-        elsif match = scan(/#{RE::Color}/o)
+        elsif match = scan(/#{RE::HexColor}/o)
           encoder.text_token match, :color
           
         elsif match = scan(/! *important/)
@@ -170,7 +167,7 @@ module Scanners
         elsif match = scan(RE::AtKeyword)
           encoder.text_token match, :directive
           
-        elsif match = scan(/ [+>:;,.=()\/] /x)
+        elsif match = scan(/ [+>~:;,.=()\/] /x)
           if match == ':'
             value_expected = true
           elsif match == ';'
diff --git a/lib/coderay/scanners/diff.rb b/lib/coderay/scanners/diff.rb
index 38efaf4..af0f755 100644
--- a/lib/coderay/scanners/diff.rb
+++ b/lib/coderay/scanners/diff.rb
@@ -45,7 +45,7 @@ module Scanners
           if match = scan(/--- |\+\+\+ |=+|_+/)
             encoder.begin_line line_kind = :head
             encoder.text_token match, :head
-            if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
+            if match = scan(/[^\x00\n]+?(?=$|[\t\n]|  \(revision)/)
               encoder.text_token match, :filename
               if options[:highlight_code] && match != '/dev/null'
                 file_type = CodeRay::FileType.fetch(match, :text)
diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb
index c1490ac..b282864 100644
--- a/lib/coderay/scanners/java.rb
+++ b/lib/coderay/scanners/java.rb
@@ -147,7 +147,7 @@ module Scanners
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group state
             state = :initial
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
           else
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb
index 43ecb18..9eb0a0a 100644
--- a/lib/coderay/scanners/java_script.rb
+++ b/lib/coderay/scanners/java_script.rb
@@ -54,10 +54,17 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
       
-      state = :initial
-      string_delimiter = nil
+      state, string_delimiter = options[:state] || @state
+      if string_delimiter
+        encoder.begin_group state
+      end
+      
       value_expected = true
       key_expected = false
       function_expected = false
@@ -72,9 +79,10 @@ module Scanners
             value_expected = true if !value_expected && match.index(?\n)
             encoder.text_token match, :space
             
-          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .*() ) !mx)
             value_expected = true
             encoder.text_token match, :comment
+            state = :open_multi_line_comment if self[1]
             
           elsif check(/\.?\d/)
             key_expected = value_expected = false
@@ -175,20 +183,36 @@ module Scanners
             encoder.text_token match, :content
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group state
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
+            string_delimiter = nil
             key_expected = value_expected = false
             state = :initial
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
+            raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
           end
           
+        when :open_multi_line_comment
+          if match = scan(%r! .*? \*/ !mx)
+            state = :initial
+          else
+            match = scan(%r! .+ !mx)
+          end
+          value_expected = true
+          encoder.text_token match, :comment if match
+          
         else
-          raise_inspect 'Unknown state', encoder
+          #:nocov:
+          raise_inspect 'Unknown state: %p' % [state], encoder
+          #:nocov:
           
         end
         
       end
       
+      if options[:keep_state]
+        @state = state, string_delimiter
+      end
+      
       if [:string, :regexp].include? state
         encoder.end_group state
       end
diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 0c90c34..4e0f462 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -70,7 +70,7 @@ module Scanners
             encoder.text_token match, :content
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group state
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
           else
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index cbdbbdb..a9492ab 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -133,7 +133,7 @@ module Scanners
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group string_type
             string_type = nil
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
           else
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state
diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index c5cf1e2..c282f31 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -96,7 +96,7 @@ module Scanners
                                       /#{patterns::METHOD_NAME}/o)
               
               kind = patterns::IDENT_KIND[match]
-              if kind == :ident && value_expected != :colon_expected && scan(/:(?!:)/)
+              if value_expected != :colon_expected && scan(/:(?!:)/)
                 value_expected = true
                 encoder.text_token match, :key
                 encoder.text_token ':',   :operator
diff --git a/lib/coderay/scanners/sass.rb b/lib/coderay/scanners/sass.rb
new file mode 100644
index 0000000..167051d
--- /dev/null
+++ b/lib/coderay/scanners/sass.rb
@@ -0,0 +1,227 @@
+module CodeRay
+module Scanners
+  
+  # A scanner for Sass.
+  class Sass < CSS
+    
+    register_for :sass
+    file_extension 'sass'
+    
+    STRING_CONTENT_PATTERN = {
+      "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
+      '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
+    }
+    
+  protected
+    
+    def setup
+      @state = :initial
+    end
+    
+    def scan_tokens encoder, options
+      states = Array(options[:state] || @state)
+      string_delimiter = nil
+      
+      until eos?
+        
+        if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/))
+          encoder.text_token self[1], :space if self[1]
+          encoder.begin_group :comment
+          encoder.text_token self[2], :delimiter
+          encoder.text_token self[3], :content if self[3]
+          if match = scan(/(?:\n+#{self[1]} .*)+/)
+            encoder.text_token match, :content
+          end
+          encoder.end_group :comment
+        elsif match = scan(/\n|[^\n\S]+\n?/)
+          encoder.text_token match, :space
+          if match.index(/\n/)
+            value_expected = false
+            states.pop if states.last == :include
+          end
+        
+        elsif states.last == :sass_inline && (match = scan(/\}/))
+          encoder.text_token match, :inline_delimiter
+          encoder.end_group :inline
+          states.pop
+        
+        elsif case states.last
+          when :initial, :media, :sass_inline
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
+              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag)
+              next
+            elsif !value_expected && (match = scan(/\*/))
+              encoder.text_token match, :tag
+              next
+            elsif match = scan(RE::Class)
+              encoder.text_token match, :class
+              next
+            elsif match = scan(RE::Id)
+              encoder.text_token match, :id
+              next
+            elsif match = scan(RE::PseudoClass)
+              encoder.text_token match, :pseudo_class
+              next
+            elsif match = scan(RE::AttributeSelector)
+              # TODO: Improve highlighting inside of attribute selectors.
+              encoder.text_token match[0,1], :operator
+              encoder.text_token match[1..-2], :attribute_name if match.size > 2
+              encoder.text_token match[-1,1], :operator if match[-1] == ?]
+              next
+            elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o)
+              encoder.text_token match, :function
+              next
+            elsif match = scan(/@import\b/)
+              encoder.text_token match, :directive
+              states << :include
+              next
+            elsif match = scan(/@media\b/)
+              encoder.text_token match, :directive
+              # states.push :media_before_name
+              next
+            end
+          
+          when :block
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
+              if value_expected
+                encoder.text_token match, :value
+              else
+                encoder.text_token match, :key
+              end
+              next
+            end
+            
+          when :string
+            if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+              encoder.text_token match, :content
+            elsif match = scan(/['"]/)
+              encoder.text_token match, :delimiter
+              encoder.end_group :string
+              string_delimiter = nil
+              states.pop
+            elsif match = scan(/#\{/)
+              encoder.begin_group :inline
+              encoder.text_token match, :inline_delimiter
+              states.push :sass_inline
+            elsif match = scan(/ \\ | $ /x)
+              encoder.end_group :string
+              encoder.text_token match, :error unless match.empty?
+              states.pop
+            else
+              raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
+            end
+          
+          when :include
+            if match = scan(/[^\s'",]+/)
+              encoder.text_token match, :include
+              next
+            end
+          
+          else
+            #:nocov:
+            raise_inspect 'Unknown state', encoder
+            #:nocov:
+            
+          end
+          
+        elsif match = scan(/\$#{RE::Ident}/o)
+          encoder.text_token match, :variable
+          next
+        
+        elsif match = scan(/&/)
+          encoder.text_token match, :local_variable
+          
+        elsif match = scan(/\+#{RE::Ident}/o)
+          encoder.text_token match, :include
+          value_expected = true
+          
+        elsif match = scan(/\/\*(?:.*?\*\/|.*)|\/\/.*/)
+          encoder.text_token match, :comment
+          
+        elsif match = scan(/#\{/)
+          encoder.begin_group :inline
+          encoder.text_token match, :inline_delimiter
+          states.push :sass_inline
+          
+        elsif match = scan(/\{/)
+          value_expected = false
+          encoder.text_token match, :operator
+          states.push :block
+          
+        elsif match = scan(/\}/)
+          value_expected = false
+          encoder.text_token match, :operator
+          if states.last == :block || states.last == :media
+            states.pop
+          end
+          
+        elsif match = scan(/['"]/)
+          encoder.begin_group :string
+          string_delimiter = match
+          encoder.text_token match, :delimiter
+          if states.include? :sass_inline
+            content = scan_until(/(?=#{string_delimiter}|\}|\z)/)
+            encoder.text_token content, :content unless content.empty?
+            encoder.text_token string_delimiter, :delimiter if scan(/#{string_delimiter}/)
+            encoder.end_group :string
+          else
+            states.push :string
+          end
+          
+        elsif match = scan(/#{RE::Function}/o)
+          encoder.begin_group :function
+          start = match[/^[-\w]+\(/]
+          encoder.text_token start, :delimiter
+          if match[-1] == ?)
+            encoder.text_token match[start.size..-2], :content
+            encoder.text_token ')', :delimiter
+          else
+            encoder.text_token match[start.size..-1], :content
+          end
+          encoder.end_group :function
+          
+        elsif match = scan(/[a-z][-a-z_]*(?=\()/o)
+          encoder.text_token match, :predefined
+          
+        elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+          encoder.text_token match, :float
+          
+        elsif match = scan(/#{RE::HexColor}/o)
+          encoder.text_token match, :color
+          
+        elsif match = scan(/! *(?:important|optional)/)
+          encoder.text_token match, :important
+          
+        elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
+          encoder.text_token match, :color
+          
+        elsif match = scan(/@else if\b|#{RE::AtKeyword}/)
+          encoder.text_token match, :directive
+          value_expected = true
+          
+        elsif match = scan(/ == | != | [-+*\/>~:;,.=()] /x)
+          if match == ':'
+            value_expected = true
+          elsif match == ';'
+            value_expected = false
+          end
+          encoder.text_token match, :operator
+          
+        else
+          encoder.text_token getch, :error
+          
+        end
+        
+      end
+      
+      if options[:keep_state]
+        @state = states
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/scanners/sql.rb b/lib/coderay/scanners/sql.rb
index bcbffd5..b757278 100644
--- a/lib/coderay/scanners/sql.rb
+++ b/lib/coderay/scanners/sql.rb
@@ -148,7 +148,7 @@ module CodeRay module Scanners
               encoder.text_token string_content, :content
               string_content = ''
             end
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
           else
             raise "else case \" reached; %p not handled." % peek(1), encoder
diff --git a/lib/coderay/scanners/taskpaper.rb b/lib/coderay/scanners/taskpaper.rb
new file mode 100644
index 0000000..42670bc
--- /dev/null
+++ b/lib/coderay/scanners/taskpaper.rb
@@ -0,0 +1,36 @@
+module CodeRay
+module Scanners
+  
+  class Taskpaper < Scanner
+    
+    register_for :taskpaper
+    file_extension 'taskpaper'
+    
+  protected
+    
+    def scan_tokens encoder, options
+      until eos?
+        if match = scan(/\S.*:.*$/)                  # project
+          encoder.text_token(match, :namespace)
+        elsif match = scan(/-.+@done.*/)             # completed task
+          encoder.text_token(match, :done)
+        elsif match = scan(/-(?:[^@\n]+|@(?!due))*/) # task
+          encoder.text_token(match, :plain)
+        elsif match = scan(/@due.*/)                 # comment
+          encoder.text_token(match, :important)
+        elsif match = scan(/.+/)                     # comment
+          encoder.text_token(match, :comment)
+        elsif match = scan(/\s+/)                    # space
+          encoder.text_token(match, :space)
+        else                                         # other
+          encoder.text_token getch, :error
+        end
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/styles/alpha.rb b/lib/coderay/styles/alpha.rb
index a60725b..7b2381d 100644
--- a/lib/coderay/styles/alpha.rb
+++ b/lib/coderay/styles/alpha.rb
@@ -39,6 +39,9 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
   color: gray !important;
   text-decoration: none !important;
 }
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
 .CodeRay .line-numbers a:target { color: blue !important; }
 .CodeRay .line-numbers .highlighted { color: red !important; }
 .CodeRay .line-numbers .highlighted a { color: red !important; }
@@ -78,14 +81,17 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .exception { color:#C00; font-weight:bold }
 .float { color:#60E }
 .function { color:#06B; font-weight:bold }
+.function .delimiter { color:#024; font-weight:bold }
 .global-variable { color:#d70 }
 .hex { color:#02b }
+.id  { color:#33D; font-weight:bold }
 .imaginary { color:#f00 }
 .include { color:#B44; font-weight:bold }
 .inline { background-color: hsla(0,0%,0%,0.07); color: black }
 .inline-delimiter { font-weight: bold; color: #666 }
 .instance-variable { color:#33B }
 .integer  { color:#00D }
+.important { color:#D00 }
 .key .char { color: #60f }
 .key .delimiter { color: #404 }
 .key { color: #606 }
@@ -137,6 +143,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .delete .delete { color: #c00; background:transparent; font-weight:bold }
 .change .change { color: #88f }
 .head .head { color: #f4f }
+
+.done { text-decoration: line-through; color: gray }
     TOKENS
 
   end
diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb
index a0ad5e4..923c859 100755
--- a/lib/coderay/token_kinds.rb
+++ b/lib/coderay/token_kinds.rb
@@ -29,7 +29,8 @@ module CodeRay
     :directive           => 'directive',
     :doc                 => 'doc',
     :doctype             => 'doctype',
-    :doc_string          => 'doc-string',
+    :docstring           => 'doc-string',
+    :done                => 'done',
     :entity              => 'entity',
     :error               => 'error',
     :escape              => 'escape',
@@ -39,6 +40,7 @@ module CodeRay
     :function            => 'function',
     :global_variable     => 'global-variable',
     :hex                 => 'hex',
+    :id                  => 'id',
     :imaginary           => 'imaginary',
     :important           => 'important',
     :include             => 'include',
@@ -84,8 +86,4 @@ module CodeRay
   )
 
   TokenKinds[:method]    = TokenKinds[:function]
-  TokenKinds[:escape]    = TokenKinds[:delimiter]
-  TokenKinds[:docstring] = TokenKinds[:comment]
-
-  TokenKinds.freeze
 end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index c747017..6957d69 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -93,6 +93,7 @@ module CodeRay
     # This method is used by @Scanner#tokenize@ when called with an Array
     # of source strings. The Diff encoder uses it for inline highlighting.
     def split_into_parts *sizes
+      return Array.new(sizes.size) { Tokens.new } if size == 2 && first == ''
       parts = []
       opened = []
       content = nil
diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb
index bfb5f24..4b4f085 100644
--- a/lib/coderay/version.rb
+++ b/lib/coderay/version.rb
@@ -1,3 +1,3 @@
 module CodeRay
-  VERSION = '1.0.9'
+  VERSION = '1.1.0'
 end