Added a Filter class; filters inherit from it and yield tokens.

You can write something like this now:

@CodeRay.scan('puts "Hello, world"!  # secret', :ruby).token_filter(:exclude => [:comment]).div@

4 files changed, 34 insertions, 17 deletions

diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index dd97067..352be63 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -1,5 +1,3 @@
-require "stringio"
-
 module CodeRay
 
   # This module holds the Encoder class and its subclasses.
@@ -132,7 +130,7 @@ module CodeRay
       # By default, it calls text_token or block_token, depending on
       # whether +text+ is a String.
       def token text, kind
-        out =
+        encoded_token =
           if text.is_a? ::String
             text_token text, kind
           elsif text.is_a? ::Symbol
@@ -140,12 +138,15 @@ module CodeRay
           else
             raise 'Unknown token text type: %p' % text
           end
-        @out << out if defined?(@out) && @out
+        append_encoded_token_to_output encoded_token
+      end
+      
+      def append_encoded_token_to_output encoded_token
+        @out << encoded_token if encoded_token && defined?(@out) && @out
       end
       
       # Called for each text token ([text, kind]), where text is a String.
       def text_token text, kind
-        ''
       end
       
       # Called for each block (non-text) token ([action, kind]), where action is a Symbol.
@@ -161,7 +162,7 @@ module CodeRay
           end_line kind
         else
           raise 'unknown block action: %p' % action
-        end.to_s
+        end
       end
       
       # Called for each block token at the start of the block ([:open, kind]).
diff --git a/lib/coderay/encoders/comment_filter.rb b/lib/coderay/encoders/comment_filter.rb
index 837f282..e30c664 100644
--- a/lib/coderay/encoders/comment_filter.rb
+++ b/lib/coderay/encoders/comment_filter.rb
@@ -4,11 +4,13 @@ module Encoders
   load :token_filter
   
   class CommentFilter < TokenFilter
-
+    
     register_for :comment_filter
-
+    
     DEFAULT_OPTIONS = TokenFilter::DEFAULT_OPTIONS.merge \
       :exclude => [:comment]
-
+    
+  end
+  
 end
 end
diff --git a/lib/coderay/encoders/filter.rb b/lib/coderay/encoders/filter.rb
new file mode 100644
index 0000000..cb784f6
--- /dev/null
+++ b/lib/coderay/encoders/filter.rb
@@ -0,0 +1,16 @@
+module CodeRay
+module Encoders
+  
+  class Filter < Encoder
+    
+    register_for :filter
+    
+  protected
+    def setup options
+      @out = Tokens.new
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/encoders/token_filter.rb b/lib/coderay/encoders/token_filter.rb
index 4bd77c7..665587b 100644
--- a/lib/coderay/encoders/token_filter.rb
+++ b/lib/coderay/encoders/token_filter.rb
@@ -1,7 +1,9 @@
 module CodeRay
 module Encoders
-
-  class TokenFilter < Encoder
+  
+  load :filter
+  
+  class TokenFilter < Filter
 
     include Streamable
     register_for :token_filter
@@ -19,12 +21,8 @@ module Encoders
     end
     
     def text_token text, kind
-      if @exclude.include?(kind) ||
-        @include != :all && !@include.include?(kind)
-        ''
-      else
-        text
-      end
+      [text, kind] if !@exclude.include?(kind) &&
+        (@include == :all || @include.include?(kind))
     end
 
   end