Direct Streaming! See #142 and Changes.textile.

11 files changed, 338 insertions, 193 deletions

diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb
index 2e60a89..451a7f8 100644
--- a/lib/coderay/encoders/count.rb
+++ b/lib/coderay/encoders/count.rb
@@ -1,25 +1,55 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
   # Returns the number of tokens.
   # 
-  # Text and block tokens (:open etc.) are counted.
+  # Text and block tokens are counted.
   class Count < Encoder
-
+    
     include Streamable
     register_for :count
-
+    
   protected
-
+    
     def setup options
       @out = 0
     end
-
-    def token text, kind
+    
+    def text_token text, kind
+      @out += 1
+    end
+    
+    def begin_group kind
       @out += 1
     end
+    alias end_group begin_group
+    alias begin_line begin_group
+    alias end_line begin_group
     
   end
-
+  
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class CountTest < Test::Unit::TestCase
+  
+  def test_count
+    tokens = CodeRay.scan <<-RUBY.strip, :ruby
+#!/usr/bin/env ruby
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal 9, tokens.encode_with(:count)
+  end
+  
+end
\ No newline at end of file
diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 4c680d3..89e430f 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -19,31 +19,43 @@ module Encoders
     register_for :debug
 
     FILE_EXTENSION = 'raydebug'
+    
+    def initialize options = {}
+      super
+      @opened = []
+    end
 
-  protected
+  public
+  
     def text_token text, kind
       if kind == :space
-        text
+        @out << text
       else
         text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
-        "#{kind}(#{text})"
+        @out << kind.to_s << '(' << text << ')'
       end
     end
 
-    def open_token kind
-      "#{kind}<"
+    def begin_group kind
+      @opened << kind
+      @out << kind.to_s << '<'
     end
 
-    def close_token kind
-      '>'
+    def end_group kind
+      if @opened.last != kind
+        puts @out
+        raise "we are inside #{@opened.inspect}, not #{kind}"
+      end
+      @opened.pop
+      @out << '>'
     end
 
     def begin_line kind
-      "#{kind}["
+      @out << kind.to_s << '['
     end
 
     def end_line kind
-      ']'
+      @out << ']'
     end
 
   end
@@ -74,16 +86,16 @@ class DebugEncoderTest < Test::Unit::TestCase
   TEST_INPUT = CodeRay::Tokens[
     ['10', :integer],
     ['(\\)', :operator],
-    [:open, :string],
+    [:begin_group, :string],
     ['test', :content],
-    [:close, :string],
+    [:end_group, :string],
     [:begin_line, :test],
     ["\n", :space],
     ["\n  \t", :space],
     ["   \n", :space],
     ["[]", :method],
     [:end_line, :test],
-  ]
+  ].flatten
   TEST_OUTPUT = <<-'DEBUG'.chomp
 integer(10)operator((\\\))string<content(test)>test[
 
diff --git a/lib/coderay/encoders/filter.rb b/lib/coderay/encoders/filter.rb
index c1991cf..6b78ad3 100644
--- a/lib/coderay/encoders/filter.rb
+++ b/lib/coderay/encoders/filter.rb
@@ -16,15 +16,27 @@ module Encoders
     end
     
     def text_token text, kind
-      [text, kind] if include_text_token? text, kind
+      @out.text_token text, kind if include_text_token? text, kind
     end
     
     def include_text_token? text, kind
       true
     end
     
-    def block_token action, kind
-      [action, kind] if include_block_token? action, kind
+    def begin_group kind
+      @out.begin_group kind if include_block_token? :begin_group, kind
+    end
+    
+    def end_group kind
+      @out.end_group kind if include_block_token? :end_group, kind
+    end
+    
+    def begin_line kind
+      @out.begin_line kind if include_block_token? :begin_line, kind
+    end
+    
+    def end_line kind
+      @out.end_line kind if include_block_token? :end_line, kind
     end
     
     def include_block_token? action, kind
@@ -59,7 +71,7 @@ class FilterTest < Test::Unit::TestCase
   def test_filtering_text_tokens
     tokens = CodeRay::Tokens.new
     10.times do |i|
-      tokens << [i.to_s, :index]
+      tokens.text_token i.to_s, :index
     end
     assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
     assert_equal tokens, tokens.filter
@@ -68,9 +80,9 @@ class FilterTest < Test::Unit::TestCase
   def test_filtering_block_tokens
     tokens = CodeRay::Tokens.new
     10.times do |i|
-      tokens << [:open, :index]
-      tokens << [i.to_s, :content]
-      tokens << [:close, :index]
+      tokens.begin_group :index
+      tokens.text_token i.to_s, :content
+      tokens.end_group :index
     end
     assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
     assert_equal tokens, tokens.filter
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index dcdffa1..807fb42 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -83,7 +83,7 @@ module Encoders
   #
   # === :hint
   # Include some information into the output using the title attribute.
-  # Can be :info (show token type on mouse-over), :info_long (with full path)
+  # Can be :info (show token kind on mouse-over), :info_long (with full path)
   # or :debug (via inspect).
   #
   # Default: false
@@ -153,12 +153,18 @@ module Encoders
     #
     # +hint+ may be :info, :info_long or :debug.
     def self.token_path_to_hint hint, kinds
+      # FIXME: TRANSPARENT_TOKEN_KINDS?
+      # if TRANSPARENT_TOKEN_KINDS.include? kinds.first
+      #   kinds = kinds[1..-1]
+      # else
+      #   kinds = kinds[1..-1] + kinds.first
+      # end
       title =
         case hint
         when :info
           TOKEN_KIND_TO_INFO[kinds.first]
         when :info_long
-          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+          kinds.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
         when :debug
           kinds.inspect
         end
@@ -167,13 +173,13 @@ module Encoders
 
     def setup options
       super
-
+      
       @HTML_ESCAPE = HTML_ESCAPE.dup
       @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
-
+      
       @opened = [nil]
       @css = CSS.new options[:style]
-
+      
       hint = options[:hint]
       if hint and not [:debug, :info, :info_long].include? hint
         raise ArgumentError, "Unknown value %p for :hint; \
@@ -184,45 +190,33 @@ module Encoders
 
       when :class
         @css_style = Hash.new do |h, k|
-          c = CodeRay::Tokens::AbbreviationForKind[k.first]
-          if c == :NO_HIGHLIGHT and not hint
-            h[k.dup] = false
-          else
-            title = if hint
-              HTML.token_path_to_hint(hint, k[1..-1] << k.first)
-            else
-              ''
-            end
-            if c == :NO_HIGHLIGHT
-              h[k.dup] = '<span%s>' % [title]
-            else
-              h[k.dup] = '<span%s class="%s">' % [title, c]
+          c = Tokens::AbbreviationForKind[k.first]
+          h[k.dup] = 
+            if c != :NO_HIGHLIGHT or hint
+              if hint
+                title = HTML.token_path_to_hint hint, k
+              end
+              if c == :NO_HIGHLIGHT
+                '<span%s>' % [title]
+              else
+                '<span%s class="%s">' % [title, c]
+              end
             end
-          end
         end
 
       when :style
         @css_style = Hash.new do |h, k|
-          if k.is_a? ::Array
-            styles = k.dup
-          else
-            styles = [k]
-          end
-          type = styles.first
-          classes = styles.map { |c| Tokens::AbbreviationForKind[c] }
-          if classes.first == :NO_HIGHLIGHT and not hint
-            h[k] = false
-          else
-            styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
-            title = HTML.token_path_to_hint hint, styles
-            style = @css[*classes]
-            h[k] =
+          classes = k.map { |c| Tokens::AbbreviationForKind[c] }
+          h[k.dup] =
+            if classes.first != :NO_HIGHLIGHT or hint
+              if hint
+                title = HTML.token_path_to_hint hint, k
+              end
+              style = @css[*classes]
               if style
                 '<span%s style="%s">' % [title, style]
-              else
-                false
               end
-          end
+            end
         end
 
       else
@@ -233,80 +227,81 @@ module Encoders
 
     def finish options
       not_needed = @opened.shift
-      @out << '</span>' * @opened.size
       unless @opened.empty?
         warn '%d tokens still open: %p' % [@opened.size, @opened]
+        @out << '</span>' * @opened.size
       end
-
+      
       @out.extend Output
       @out.css = @css
       @out.numerize! options[:line_numbers], options
       @out.wrap! options[:wrap]
       @out.apply_title! options[:title]
-
+      
       super
     end
-
-    def token text, type
-      case text
-      
-      when nil
-        # raise 'Token with nil as text was given: %p' % [[text, type]] 
-      
-      when String
-        if text =~ /#{HTML_ESCAPE_PATTERN}/o
-          text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
-        end
-        @opened[0] = type
-        if text != "\n" && style = @css_style[@opened]
-          @out << style << text << '</span>'
+    
+  public
+    
+    def text_token text, kind
+      if text =~ /#{HTML_ESCAPE_PATTERN}/o
+        text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
+      end
+      @opened[0] = kind
+      @out <<
+        if style = @css_style[@opened]
+          style + text + '</span>'
         else
-          @out << text
-        end
-        
-      
-      # token groups, eg. strings
-      when :open
-        @opened[0] = type
-        @out << (@css_style[@opened] || '<span>')
-        @opened << type
-      when :close
-        if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
-          warn 'Malformed token stream: Trying to close a token (%p) ' \
-            'that is not open. Open are: %p.' % [type, @opened[1..-1]]
+          text
         end
+    end
+    
+    # token groups, eg. strings
+    def begin_group kind
+      @opened[0] = kind
+      @opened << kind
+      @out << (@css_style[@opened] || '<span>')
+    end
+    
+    def end_group kind
+      if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != kind)
+        warn 'Malformed token stream: Trying to close a token (%p) ' \
+          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+      end
+      @out << 
         if @opened.empty?
-          # nothing to close
+          '' # nothing to close
         else
-          @out << '</span>'
           @opened.pop
+          '</span>'
         end
-      
-      # whole lines to be highlighted, eg. a deleted line in a diff
-      when :begin_line
-        @opened[0] = type
-        if style = @css_style[@opened]
-          @out << style.sub('<span', '<div')
+    end
+    
+    # whole lines to be highlighted, eg. a deleted line in a diff
+    def begin_line kind
+      @opened[0] = kind
+      style = @css_style[@opened]
+      @opened << kind
+      @out <<
+        if style
+          style.sub '<span', '<div'
         else
-          @out << '<div>'
-        end
-        @opened << type
-      when :end_line
-        if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
-          warn 'Malformed token stream: Trying to close a line (%p) ' \
-            'that is not open. Open are: %p.' % [type, @opened[1..-1]]
+          '<div>'
         end
+    end
+    
+    def end_line kind
+      if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != kind)
+        warn 'Malformed token stream: Trying to close a line (%p) ' \
+          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+      end
+      @out <<
         if @opened.empty?
-          # nothing to close
+          ''  # nothing to close
         else
-          @out << '</div>'
           @opened.pop
+          '</div>'
         end
-      
-      else
-        raise 'unknown token kind: %p' % [text]
-        
-      end
     end
 
   end
diff --git a/lib/coderay/encoders/json.rb b/lib/coderay/encoders/json.rb
index 78f0ec0..bb09809 100644
--- a/lib/coderay/encoders/json.rb
+++ b/lib/coderay/encoders/json.rb
@@ -33,11 +33,23 @@ module Encoders
     end
     
     def text_token text, kind
-      { :type => 'text', :text => text, :kind => kind }
+      @out << { :type => 'text', :text => text, :kind => kind }
     end
     
-    def block_token action, kind
-      { :type => 'block', :action => action, :kind => kind }
+    def begin_group kind
+      @out << { :type => 'block', :action => 'open', :kind => kind }
+    end
+    
+    def end_group kind
+      @out << { :type => 'block', :action => 'close', :kind => kind }
+    end
+    
+    def begin_line kind
+      @out << { :type => 'block', :action => 'begin_line', :kind => kind }
+    end
+    
+    def end_line kind
+      @out << { :type => 'block', :action => 'end_line', :kind => kind }
     end
     
     def finish options
diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb
index c6ed4de..6b36aef 100644
--- a/lib/coderay/encoders/lines_of_code.rb
+++ b/lib/coderay/encoders/lines_of_code.rb
@@ -79,9 +79,9 @@ puts "Hello world!"
   
   def test_filtering_block_tokens
     tokens = CodeRay::Tokens.new
-    tokens << ["Hello\n", :world]
-    tokens << ["Hello\n", :space]
-    tokens << ["Hello\n", :comment]
+    tokens.concat ["Hello\n", :world]
+    tokens.concat ["Hello\n", :space]
+    tokens.concat ["Hello\n", :comment]
     assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
     assert_equal 2, tokens.lines_of_code
     assert_equal 2, tokens.loc
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index 1b38938..d267b21 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -1,3 +1,4 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
 
@@ -34,9 +35,25 @@ module Encoders
     end
 
     # TODO Hierarchy handling
-    def block_token action, kind
+    def begin_group kind
+      block_token 'begin_group'
+    end
+
+    def end_group kind
+      block_token 'end_group'
+    end
+
+    def begin_line kind
+      block_token 'begin_line'
+    end
+
+    def end_line kind
+      block_token 'end_line'
+    end
+    
+    def block_token action
       @type_stats['TOTAL'].count += 1
-      @type_stats['open/close'].count += 1
+      @type_stats[action].count += 1
     end
 
     STATS = <<-STATS  # :nodoc:
@@ -77,3 +94,67 @@ Token Types (%d):
 
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class StatisticEncoderTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder
+    stats = nil
+    assert_nothing_raised do
+      stats = CodeRay.encoder :statistic
+    end
+    assert_kind_of CodeRay::Encoders::Encoder, stats
+  end
+  
+  TEST_INPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n", :space],
+    ["\n  \t", :space],
+    ["   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  TEST_OUTPUT = <<-'DEBUG'
+
+Code Statistics
+
+Tokens                  11
+  Non-Whitespace         4
+Bytes Total             20
+
+Token Types (5):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                       11  100.00 %     1.8
+  space                        3   27.27 %     3.0
+  begin_group                  1    9.09 %     0.0
+  begin_line                   1    9.09 %     0.0
+  content                      1    9.09 %     4.0
+  end_group                    1    9.09 %     0.0
+  end_line                     1    9.09 %     0.0
+  integer                      1    9.09 %     2.0
+  method                       1    9.09 %     2.0
+  operator                     1    9.09 %     3.0
+
+  DEBUG
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT)
+    assert_equal TEST_OUTPUT, TEST_INPUT.statistic
+  end
+  
+end
\ No newline at end of file
diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index 7224218..3a774a0 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -92,41 +92,72 @@ module CodeRay
       TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
       TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
-      TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+      TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] =
+        TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] =
+        TOKEN_COLORS[:delimiter]
 
     protected
 
       def setup(options)
         super
         @opened = []
+        @subcolors = nil
       end
-
-      def finish(options)
-        super
-      end
-    
-      def text_token text, type
-        if color = (@subcolors || TOKEN_COLORS)[type]
+      
+    public
+      
+      def text_token text, kind
+        if color = (@subcolors || TOKEN_COLORS)[kind]
           if Hash === color
             if color[:self]
               color = color[:self]
             else
-              return text
+              @out << text
+              return
             end
           end
-
-          out = ansi_colorize(color)
-          out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color))
-          out << ansi_clear
-          out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self]
-          out
+          
+          @out << ansi_colorize(color)
+          @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color))
+          @out << ansi_clear
+          @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self]
         else
-          text
+          @out << text
         end
       end
       
-      def open_token type
-        if color = TOKEN_COLORS[type]
+      def begin_group kind
+        @opened << kind
+        @out << open_token(kind)
+      end
+      alias begin_line begin_group
+      
+      def end_group kind
+        if @opened.empty?
+          # nothing to close
+        else
+          @opened.pop
+          @out << ansi_clear
+          @out << open_token(@opened.last)
+        end
+      end
+      
+      def end_line kind
+        if @opened.empty?
+          # nothing to close
+        else
+          @opened.pop
+          # whole lines to be highlighted,
+          # eg. added/modified/deleted lines in a diff
+          @out << "\t" * 100 + ansi_clear
+          @out << open_token(@opened.last)
+        end
+      end
+      
+    private
+      
+      def open_token kind
+        if color = TOKEN_COLORS[kind]
           if Hash === color
             @subcolors = color
             ansi_colorize(color[:self]) if color[:self]
@@ -140,34 +171,6 @@ module CodeRay
         end
       end
       
-      def block_token action, type
-        case action
-          
-        when :open, :begin_line
-          @opened << type
-          open_token type
-        when :close, :end_line
-          if @opened.empty?
-            # nothing to close
-          else
-            @opened.pop
-            if action == :end_line
-              # whole lines to be highlighted,
-              # eg. added/modified/deleted lines in a diff
-              "\t" * 100 + ansi_clear
-            else
-              ansi_clear
-            end +
-              open_token(@opened.last)
-          end
-          
-        else
-          raise 'unknown token kind: %p' % [text]
-        end
-      end
-      
-    private
-      
       def ansi_colorize(color)
         Array(color).map { |c| "\e[#{c}m" }.join
       end
diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index 26fef84..ecbf624 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -23,16 +23,16 @@ module Encoders
       :separator => ''
     }
 
+    def text_token text, kind
+      @out << text + @sep
+    end
+
   protected
     def setup options
       super
       @sep = options[:separator]
     end
 
-    def text_token text, kind
-      text + @sep
-    end
-
     def finish options
       super.chomp @sep
     end
diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb
index 4b2f582..fd3df44 100644
--- a/lib/coderay/encoders/token_kind_filter.rb
+++ b/lib/coderay/encoders/token_kind_filter.rb
@@ -76,28 +76,28 @@ class TokenKindFilterTest < Test::Unit::TestCase
   def test_filtering_text_tokens
     tokens = CodeRay::Tokens.new
     for i in 1..10
-      tokens << [i.to_s, :index]
-      tokens << [' ', :space] if i < 10
+      tokens.text_token i.to_s, :index
+      tokens.text_token ' ', :space if i < 10
     end
-    assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).size
-    assert_equal 10, tokens.token_kind_filter(:exclude => :space).size
-    assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).size
-    assert_equal 9, tokens.token_kind_filter(:include => :space).size
-    assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).size
-    assert_equal 0, tokens.token_kind_filter(:exclude => :all).size
+    assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).count
+    assert_equal 10, tokens.token_kind_filter(:exclude => :space).count
+    assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).count
+    assert_equal 9, tokens.token_kind_filter(:include => :space).count
+    assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).count
+    assert_equal 0, tokens.token_kind_filter(:exclude => :all).count
   end
   
   def test_filtering_block_tokens
     tokens = CodeRay::Tokens.new
     10.times do |i|
-      tokens << [:open, :index]
-      tokens << [i.to_s, :content]
-      tokens << [:close, :index]
+      tokens.begin_group :index
+      tokens.text_token i.to_s, :content
+      tokens.end_group :index
     end
-    assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).size
-    assert_equal 20, tokens.token_kind_filter(:include => :blubb).size
-    assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).size
-    assert_equal 30, tokens.token_kind_filter(:exclude => :index).size
+    assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).count
+    assert_equal 20, tokens.token_kind_filter(:include => :blubb).count
+    assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).count
+    assert_equal 30, tokens.token_kind_filter(:exclude => :index).count
   end
   
 end
diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb
index f32c967..0006d75 100644
--- a/lib/coderay/encoders/xml.rb
+++ b/lib/coderay/encoders/xml.rb
@@ -53,19 +53,19 @@ module Encoders
         end
       end
     end
-
-    def open_token kind
+    
+    def begin_group kind
       @node = @node.add_element kind.to_s
     end
-
-    def close_token kind
+    
+    def end_group kind
       if @node == @root
         raise 'no token to close!'
       end
       @node = @node.parent
     end
-
+    
   end
-
+  
 end
 end