Added a Filter class; filters inherit from it and yield tokens.

You can write something like this now:

@CodeRay.scan('puts "Hello, world"!  # secret', :ruby).token_filter(:exclude => [:comment]).div@

3 files changed, 27 insertions, 11 deletions

diff --git a/lib/coderay/encoders/comment_filter.rb b/lib/coderay/encoders/comment_filter.rb
index 837f282..e30c664 100644
--- a/lib/coderay/encoders/comment_filter.rb
+++ b/lib/coderay/encoders/comment_filter.rb
@@ -4,11 +4,13 @@ module Encoders
   load :token_filter
   
   class CommentFilter < TokenFilter
-
+    
     register_for :comment_filter
-
+    
     DEFAULT_OPTIONS = TokenFilter::DEFAULT_OPTIONS.merge \
       :exclude => [:comment]
-
+    
+  end
+  
 end
 end
diff --git a/lib/coderay/encoders/filter.rb b/lib/coderay/encoders/filter.rb
new file mode 100644
index 0000000..cb784f6
--- /dev/null
+++ b/lib/coderay/encoders/filter.rb
@@ -0,0 +1,16 @@
+module CodeRay
+module Encoders
+  
+  class Filter < Encoder
+    
+    register_for :filter
+    
+  protected
+    def setup options
+      @out = Tokens.new
+    end
+    
+  end
+  
+end
+end
diff --git a/lib/coderay/encoders/token_filter.rb b/lib/coderay/encoders/token_filter.rb
index 4bd77c7..665587b 100644
--- a/lib/coderay/encoders/token_filter.rb
+++ b/lib/coderay/encoders/token_filter.rb
@@ -1,7 +1,9 @@
 module CodeRay
 module Encoders
-
-  class TokenFilter < Encoder
+  
+  load :filter
+  
+  class TokenFilter < Filter
 
     include Streamable
     register_for :token_filter
@@ -19,12 +21,8 @@ module Encoders
     end
     
     def text_token text, kind
-      if @exclude.include?(kind) ||
-        @include != :all && !@include.include?(kind)
-        ''
-      else
-        text
-      end
+      [text, kind] if !@exclude.include?(kind) &&
+        (@include == :all || @include.include?(kind))
     end
 
   end