Major rewrite of encoders to support IO output; fixed some minor scanner bugs; cleanups; dropped NitroXHTML scanner; improved tests

14 files changed, 88 insertions, 44 deletions

diff --git a/lib/coderay/encoders/_map.rb b/lib/coderay/encoders/_map.rb
index 24ada0a..e5cbdeb 100644
--- a/lib/coderay/encoders/_map.rb
+++ b/lib/coderay/encoders/_map.rb
@@ -3,7 +3,6 @@ module Encoders
   
   map \
     :loc             => :lines_of_code,
-    :html            => :page,
     :plain           => :text,
     :plaintext       => :text,
     :remove_comments => :comment_filter,
diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb
index 52d4bdd..98a427e 100644
--- a/lib/coderay/encoders/count.rb
+++ b/lib/coderay/encoders/count.rb
@@ -11,17 +11,23 @@ module Encoders
   protected
     
     def setup options
-      @out = 0
+      super
+      
+      @count = 0
+    end
+    
+    def finish options
+      output @count
     end
     
   public
     
     def text_token text, kind
-      @out += 1
+      @count += 1
     end
     
     def begin_group kind
-      @out += 1
+      @count += 1
     end
     alias end_group begin_group
     alias begin_line begin_group
diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 08f1309..95d6138 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -27,7 +27,7 @@ module Encoders
       if kind == :space
         @out << text
       else
-        # FIXME: Escape (
+        # TODO: Escape (
         text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
         @out << kind.to_s << '(' << text << ')'
       end
diff --git a/lib/coderay/encoders/filter.rb b/lib/coderay/encoders/filter.rb
index a71e43e..e7f34d6 100644
--- a/lib/coderay/encoders/filter.rb
+++ b/lib/coderay/encoders/filter.rb
@@ -21,29 +21,35 @@ module Encoders
     
   protected
     def setup options
-      @out = options[:tokens] || Tokens.new
+      super
+      
+      @tokens = options[:tokens] || Tokens.new
+    end
+    
+    def finish options
+      output @tokens
     end
     
   public
     
     def text_token text, kind  # :nodoc:
-      @out.text_token text, kind
+      @tokens.text_token text, kind
     end
     
     def begin_group kind  # :nodoc:
-      @out.begin_group kind
+      @tokens.begin_group kind
     end
     
     def begin_line kind  # :nodoc:
-      @out.begin_line kind
+      @tokens.begin_line kind
     end
     
     def end_group kind  # :nodoc:
-      @out.end_group kind
+      @tokens.end_group kind
     end
     
     def end_line kind  # :nodoc:
-      @out.end_line kind
+      @tokens.end_line kind
     end
     
   end
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index ec73d2c..abbafad 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -164,6 +164,11 @@ module Encoders
     def setup options
       super
       
+      if options[:wrap] || options[:line_numbers]
+        @real_out = @out
+        @out = ''
+      end
+      
       @HTML_ESCAPE = HTML_ESCAPE.dup
       @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
       
@@ -214,7 +219,7 @@ module Encoders
     
     def finish options
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened]
+        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
         @out << '</span>' while @opened.pop
         @last_opened = nil
       end
@@ -227,6 +232,11 @@ module Encoders
       @out.wrap! options[:wrap]
       @out.apply_title! options[:title]
       
+      if defined?(@real_out) && @real_out
+        @real_out << @out
+        @out = @real_out
+      end
+      
       super
     end
     
diff --git a/lib/coderay/encoders/html/css.rb b/lib/coderay/encoders/html/css.rb
index c459222..6de4b46 100644
--- a/lib/coderay/encoders/html/css.rb
+++ b/lib/coderay/encoders/html/css.rb
@@ -44,7 +44,7 @@ module Encoders
         ( [^\}]+ )?          # $2 = style
         \s* \} \s*
       |
-        ( . )                # $3 = error
+        ( [^\n]+ )           # $3 = error
       /mx
       def parse stylesheet
         stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error|
diff --git a/lib/coderay/encoders/html/output.rb b/lib/coderay/encoders/html/output.rb
index 004351b..4f65878 100644
--- a/lib/coderay/encoders/html/output.rb
+++ b/lib/coderay/encoders/html/output.rb
@@ -55,7 +55,6 @@ module Encoders
       end
       
       def apply_title! title
-        # FIXME: This may change the output!
         self.sub!(/(<title>)(<\/title>)/) { $1 + title + $2 }
         self
       end
diff --git a/lib/coderay/encoders/json.rb b/lib/coderay/encoders/json.rb
index ccad554..0a95397 100644
--- a/lib/coderay/encoders/json.rb
+++ b/lib/coderay/encoders/json.rb
@@ -36,32 +36,45 @@ module Encoders
     
   protected
     def setup options
-      @out = []
+      super
+      
+      @first = true
+      @out << '['
     end
     
     def finish options
-      @out.to_json
+      @out << ']'
+    end
+    
+    def append data
+      if @first
+        @first = false
+      else
+        @out << ','
+      end
+      
+      @out << data.to_json
     end
     
   public
     def text_token text, kind
-      @out << { :type => 'text', :text => text, :kind => kind }
+      append :type => 'text', :text => text, :kind => kind
     end
     
     def begin_group kind
-      @out << { :type => 'block', :action => 'open', :kind => kind }
+      append :type => 'block', :action => 'open', :kind => kind
     end
     
     def end_group kind
-      @out << { :type => 'block', :action => 'close', :kind => kind }
+      append :type => 'block', :action => 'close', :kind => kind
     end
     
     def begin_line kind
-      @out << { :type => 'block', :action => 'begin_line', :kind => kind }
+      append :type => 'block', :action => 'begin_line', :kind => kind
     end
     
     def end_line kind
-      @out << { :type => 'block', :action => 'end_line', :kind => kind }
+      append :type => 'block', :action => 'end_line', :kind => kind
     end
     
   end
diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb
index c49cade..5f8422f 100644
--- a/lib/coderay/encoders/lines_of_code.rb
+++ b/lib/coderay/encoders/lines_of_code.rb
@@ -31,11 +31,12 @@ module Encoders
       end
       
       options[:exclude] = kinds_not_loc
+      
       super options
     end
     
     def finish options
-      @out.to_s.scan(NON_EMPTY_LINE).size
+      output @tokens.text.scan(NON_EMPTY_LINE).size
     end
     
   end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index e52f28f..2315d9e 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -15,15 +15,12 @@ module Encoders
   protected
     
     def setup options
+      super
+      
       @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
       @real_token_count = 0
     end
     
-    def generate tokens, options
-      @tokens = tokens
-      super
-    end
-    
     STATS = <<-STATS  # :nodoc:
 
 Code Statistics
@@ -51,11 +48,13 @@ Token Types (%d):
       types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
         TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
       end.join
-      STATS % [
+      @out << STATS % [
         all_count, @real_token_count, all_size,
         @type_stats.delete_if { |k, v| k.is_a? String }.size,
         types_stats
       ]
+      
+      super
     end
     
   public
diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index 84e2215..15c66f9 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -24,19 +24,22 @@ module Encoders
     
     def text_token text, kind
       super
-      @out << @sep if @sep
+      
+      if @first
+        @first = false
+      else
+        @out << @sep
+      end if @sep
     end
     
   protected
     def setup options
       super
+      
+      @first = true
       @sep = options[:separator]
     end
     
-    def finish options
-      super.chomp @sep
-    end
-    
   end
   
 end
diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb
index 1ecf6ae..4773ea3 100644
--- a/lib/coderay/encoders/token_kind_filter.rb
+++ b/lib/coderay/encoders/token_kind_filter.rb
@@ -34,6 +34,7 @@ module Encoders
   protected
     def setup options
       super
+      
       @group_excluded = false
       @exclude = options[:exclude]
       @exclude = Array(@exclude) unless @exclude == :all
diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb
index f2e7c02..3d306a6 100644
--- a/lib/coderay/encoders/xml.rb
+++ b/lib/coderay/encoders/xml.rb
@@ -10,7 +10,7 @@ module Encoders
     
     FILE_EXTENSION = 'xml'
     
-    require 'rexml/document'
+    autoload :REXML, 'rexml/document'
     
     DEFAULT_OPTIONS = {
       :tab_width => 8,
@@ -20,6 +20,8 @@ module Encoders
     
   protected
     def setup options
+      super
+      
       @doc = REXML::Document.new
       @doc << REXML::XMLDecl.new
       @tab_width = options[:tab_width]
@@ -27,9 +29,9 @@ module Encoders
     end
     
     def finish options
-      @out = ''
       @doc.write @out, options[:pretty], options[:transitive], true
-      @out
+      
+      super
     end
     
   public
diff --git a/lib/coderay/encoders/yaml.rb b/lib/coderay/encoders/yaml.rb
index c12f8d0..1eb2523 100644
--- a/lib/coderay/encoders/yaml.rb
+++ b/lib/coderay/encoders/yaml.rb
@@ -6,39 +6,44 @@ module Encoders
   # Slow.
   class YAML < Encoder
     
+    autoload :YAML, 'yaml'
+    
     register_for :yaml
     
     FILE_EXTENSION = 'yaml'
     
   protected
     def setup options
-      require 'yaml'
-      @out = []
+      super
+      
+      @data = []
     end
     
     def finish options
-      @out.to_a.to_yaml
+      YAML.dump @data, @out
+      
+      super
     end
     
   public
     def text_token text, kind
-      @out << [text, kind]
+      @data << [text, kind]
     end
     
     def begin_group kind
-      @out << [:begin_group, kind]
+      @data << [:begin_group, kind]
     end
     
     def end_group kind
-      @out << [:end_group, kind]
+      @data << [:end_group, kind]
     end
     
     def begin_line kind
-      @out << [:begin_line, kind]
+      @data << [:begin_line, kind]
     end
     
     def end_line kind
-      @out << [:end_line, kind]
+      @data << [:end_line, kind]
     end
     
   end