* Improved implementation of Encoder *token callbacks.

* Documentation for these methods.
* Added two new encoders: CommentFilter < TokenFilter.
* Simplified Text encoder.
* Code cleanup in HTML encoder subclasses.

7 files changed, 78 insertions, 15 deletions

diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index 07733a1..dd97067 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -142,10 +142,13 @@ module CodeRay
           end
         @out << out if defined?(@out) && @out
       end
-
+      
+      # Called for each text token ([text, kind]), where text is a String.
       def text_token text, kind
+        ''
       end
-
+      
+      # Called for each block (non-text) token ([action, kind]), where action is a Symbol.
       def block_token action, kind
         case action
         when :open
@@ -158,7 +161,23 @@ module CodeRay
           end_line kind
         else
           raise 'unknown block action: %p' % action
-        end
+        end.to_s
+      end
+      
+      # Called for each block token at the start of the block ([:open, kind]).
+      def open_token kind
+      end
+      
+      # Called for each block token end of the block ([:close, kind]).
+      def close_token kind
+      end
+      
+      # Called for each line token block at the start of the line ([:begin_line, kind]).
+      def begin_line kind
+      end
+      
+      # Called for each line token block at the end of the line ([:end_line, kind]).
+      def end_line kind
       end
 
       # Called with merged options after encoding starts.
diff --git a/lib/coderay/encoders/comment_filter.rb b/lib/coderay/encoders/comment_filter.rb
new file mode 100644
index 0000000..837f282
--- /dev/null
+++ b/lib/coderay/encoders/comment_filter.rb
@@ -0,0 +1,14 @@
+module CodeRay
+module Encoders
+  
+  load :token_filter
+  
+  class CommentFilter < TokenFilter
+
+    register_for :comment_filter
+
+    DEFAULT_OPTIONS = TokenFilter::DEFAULT_OPTIONS.merge \
+      :exclude => [:comment]
+
+end
+end
diff --git a/lib/coderay/encoders/div.rb b/lib/coderay/encoders/div.rb
index 3d55415..4120172 100644
--- a/lib/coderay/encoders/div.rb
+++ b/lib/coderay/encoders/div.rb
@@ -9,10 +9,9 @@ module Encoders
 
     register_for :div
 
-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
       :css => :style,
-      :wrap => :div,
-    })
+      :wrap => :div
 
   end
 
diff --git a/lib/coderay/encoders/page.rb b/lib/coderay/encoders/page.rb
index c08f094..1b69cce 100644
--- a/lib/coderay/encoders/page.rb
+++ b/lib/coderay/encoders/page.rb
@@ -9,11 +9,10 @@ module Encoders
 
     register_for :page
 
-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
       :css => :class,
       :wrap => :page,
       :line_numbers => :table
-    })
 
   end
 
diff --git a/lib/coderay/encoders/span.rb b/lib/coderay/encoders/span.rb
index 988afec..319f6fd 100644
--- a/lib/coderay/encoders/span.rb
+++ b/lib/coderay/encoders/span.rb
@@ -9,10 +9,9 @@ module Encoders
 
     register_for :span
 
-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
       :css => :style,
-      :wrap => :span,
-    })
+      :wrap => :span
 
   end
 
diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index 14282ac..161ee67 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -14,16 +14,16 @@ module Encoders
 
   protected
     def setup options
-      @out = ''
+      super
       @sep = options[:separator]
     end
 
-    def token text, kind
-      @out << text + @sep if text.is_a? ::String
+    def text_token text, kind
+      text + @sep
     end
 
     def finish options
-      @out.chomp @sep
+      super.chomp @sep
     end
 
   end
diff --git a/lib/coderay/encoders/token_filter.rb b/lib/coderay/encoders/token_filter.rb
new file mode 100644
index 0000000..4bd77c7
--- /dev/null
+++ b/lib/coderay/encoders/token_filter.rb
@@ -0,0 +1,33 @@
+module CodeRay
+module Encoders
+
+  class TokenFilter < Encoder
+
+    include Streamable
+    register_for :token_filter
+
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+
+  protected
+    def setup options
+      super
+      @exclude = options[:exclude]
+      @include = options[:include]
+    end
+    
+    def text_token text, kind
+      if @exclude.include?(kind) ||
+        @include != :all && !@include.include?(kind)
+        ''
+      else
+        text
+      end
+    end
+
+  end
+
+end
+end