5 files changed, 135 insertions, 106 deletions

diff --git a/lib/coderay/encoders/comment_filter.rb b/lib/coderay/encoders/comment_filter.rb
index 4d3fb54..819c619 100644
--- a/lib/coderay/encoders/comment_filter.rb
+++ b/lib/coderay/encoders/comment_filter.rb
@@ -2,9 +2,18 @@
 module CodeRay
 module Encoders
   
-  load :token_class_filter
+  load :token_kind_filter
   
-  class CommentFilter < TokenClassFilter
+  # A simple Filter that removes all tokens of the :comment kind.
+  # 
+  # Alias: +remove_comments+
+  # 
+  # Usage:
+  #  CodeRay.scan('print # foo', :ruby).remove_comments.text
+  #  #-> "print "
+  # 
+  # See also: TokenKindFilter, LinesOfCode
+  class CommentFilter < TokenKindFilter
     
     register_for :comment_filter
     
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index ffbc6ca..f1fe7ce 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -27,6 +27,7 @@ module Encoders
   #
   # === :tab_width
   # Convert \t characters to +n+ spaces (a number.)
+  # 
   # Default: 8
   #
   # === :css
@@ -125,7 +126,7 @@ module Encoders
     #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
     HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
 
-    TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
+    TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
       h[kind] =
         case kind
         when :pre_constant
@@ -133,24 +134,24 @@ module Encoders
         else
           kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
         end
-    }
+    end
 
     TRANSPARENT_TOKEN_KINDS = [
       :delimiter, :modifier, :content, :escape, :inline_delimiter,
     ].to_set
 
-    # Generate a hint about the given +classes+ in a +hint+ style.
+    # Generate a hint about the given +kinds+ in a +hint+ style.
     #
     # +hint+ may be :info, :info_long or :debug.
-    def self.token_path_to_hint hint, classes
+    def self.token_path_to_hint hint, kinds
       title =
         case hint
         when :info
-          TOKEN_KIND_TO_INFO[classes.first]
+          TOKEN_KIND_TO_INFO[kinds.first]
         when :info_long
-          classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
         when :debug
-          classes.inspect
+          kinds.inspect
         end
       title ? " title=\"#{title}\"" : ''
     end
@@ -174,7 +175,7 @@ module Encoders
 
       when :class
         @css_style = Hash.new do |h, k|
-          c = CodeRay::Tokens::ClassOfKind[k.first]
+          c = CodeRay::Tokens::AbbreviationForKind[k.first]
           if c == :NO_HIGHLIGHT and not hint
             h[k.dup] = false
           else
@@ -199,7 +200,7 @@ module Encoders
             styles = [k]
           end
           type = styles.first
-          classes = styles.map { |c| Tokens::ClassOfKind[c] }
+          classes = styles.map { |c| Tokens::AbbreviationForKind[c] }
           if classes.first == :NO_HIGHLIGHT and not hint
             h[k] = false
           else
@@ -261,13 +262,13 @@ module Encoders
         @out << (@css_style[@opened] || '<span>')
         @opened << type
       when :close
+        if $CODERAY_DEBUG and @opened.last != type
+          warn 'Malformed token stream: Trying to close a token (%p) ' \
+            'that is not open. Open are: %p.' % [type, @opened[1..-1]]
+        end
         if @opened.empty?
           # nothing to close
         else
-          if $DEBUG and (@opened.size == 1 or @opened.last != type)
-            raise 'Malformed token stream: Trying to close a token (%p) \
-              that is not open. Open are: %p.' % [type, @opened[1..-1]]
-          end
           @out << '</span>'
           @opened.pop
         end
@@ -282,13 +283,13 @@ module Encoders
         end
         @opened << type
       when :end_line
+        if $CODERAY_DEBUG and @opened.last != type
+          warn 'Malformed token stream: Trying to close a line (%p) ' \
+            'that is not open. Open are: %p.' % [type, @opened[1..-1]]
+        end
         if @opened.empty?
           # nothing to close
         else
-          if $DEBUG and (@opened.size == 1 or @opened.last != type)
-            raise 'Malformed token stream: Trying to close a line (%p) \
-              that is not open. Open are: %p.' % [type, @opened[1..-1]]
-          end
           @out << '</div>'
           @opened.pop
         end
diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb
index b24145e..60918c4 100644
--- a/lib/coderay/encoders/lines_of_code.rb
+++ b/lib/coderay/encoders/lines_of_code.rb
@@ -4,7 +4,7 @@ module Encoders
   
   # Counts the LoC (Lines of Code). Returns an Integer >= 0.
   # 
-  # Alias: :loc
+  # Alias: +loc+
   # 
   # Everything that is not comment, markup, doctype/shebang, or an empty line,
   # is considered to be code.
@@ -25,10 +25,10 @@ module Encoders
       if scanner = tokens.scanner
         kinds_not_loc = scanner.class::KINDS_NOT_LOC
       else
-        warn ArgumentError, 'Tokens have no scanner.' if $DEBUG
+        warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE
         kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
       end
-      code = tokens.token_class_filter :exclude => kinds_not_loc
+      code = tokens.token_kind_filter :exclude => kinds_not_loc
       @loc = code.text.scan(NON_EMPTY_LINE).size
     end
     
diff --git a/lib/coderay/encoders/token_class_filter.rb b/lib/coderay/encoders/token_class_filter.rb
deleted file mode 100644
index a9e8673..0000000
--- a/lib/coderay/encoders/token_class_filter.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-  
-  load :filter
-  
-  class TokenClassFilter < Filter
-
-    include Streamable
-    register_for :token_class_filter
-
-    DEFAULT_OPTIONS = {
-      :exclude => [],
-      :include => :all
-    }
-
-  protected
-    def setup options
-      super
-      @exclude = options[:exclude]
-      @exclude = Array(@exclude) unless @exclude == :all
-      @include = options[:include]
-      @include = Array(@include) unless @include == :all
-    end
-    
-    def include_text_token? text, kind
-       (@include == :all || @include.include?(kind)) &&
-      !(@exclude == :all || @exclude.include?(kind))
-    end
-    
-  end
-
-end
-end
-
-if $0 == __FILE__
-  $VERBOSE = true
-  $: << File.join(File.dirname(__FILE__), '..')
-  eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class TokenClassFilterTest < Test::Unit::TestCase
-  
-  def test_creation
-    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
-    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
-    filter = nil
-    assert_nothing_raised do
-      filter = CodeRay.encoder :token_class_filter
-    end
-    assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
-  end
-  
-  def test_filtering_text_tokens
-    tokens = CodeRay::Tokens.new
-    for i in 1..10
-      tokens << [i.to_s, :index]
-      tokens << [' ', :space] if i < 10
-    end
-    assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
-    assert_equal 10, tokens.token_class_filter(:exclude => :space).size
-    assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
-    assert_equal 9, tokens.token_class_filter(:include => :space).size
-    assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
-    assert_equal 0, tokens.token_class_filter(:exclude => :all).size
-  end
-  
-  def test_filtering_block_tokens
-    tokens = CodeRay::Tokens.new
-    10.times do |i|
-      tokens << [:open, :index]
-      tokens << [i.to_s, :content]
-      tokens << [:close, :index]
-    end
-    assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
-    assert_equal 20, tokens.token_class_filter(:include => :blubb).size
-    assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
-    assert_equal 30, tokens.token_class_filter(:exclude => :index).size
-  end
-  
-end
diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb
new file mode 100644
index 0000000..4b2f582
--- /dev/null
+++ b/lib/coderay/encoders/token_kind_filter.rb
@@ -0,0 +1,103 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+  
+  load :filter
+  
+  # A Filter that selects tokens based on their token kind.
+  # 
+  # == Options
+  # 
+  # === :exclude
+  # 
+  # One or many symbols (in an Array) which shall be excluded.
+  # 
+  # Default: []
+  # 
+  # === :include
+  # 
+  # One or many symbols (in an array) which shall be included.
+  # 
+  # Default: :all, which means all tokens are included.
+  # 
+  # Exclusion wins over inclusion.
+  # 
+  # See also: CommentFilter
+  class TokenKindFilter < Filter
+
+    include Streamable
+    register_for :token_kind_filter
+
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+
+  protected
+    def setup options
+      super
+      @exclude = options[:exclude]
+      @exclude = Array(@exclude) unless @exclude == :all
+      @include = options[:include]
+      @include = Array(@include) unless @include == :all
+    end
+    
+    def include_text_token? text, kind
+       (@include == :all || @include.include?(kind)) &&
+      !(@exclude == :all || @exclude.include?(kind))
+    end
+    
+  end
+
+end
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokenKindFilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :token_kind_filter
+    end
+    assert_instance_of CodeRay::Encoders::TokenKindFilter, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    for i in 1..10
+      tokens << [i.to_s, :index]
+      tokens << [' ', :space] if i < 10
+    end
+    assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).size
+    assert_equal 10, tokens.token_kind_filter(:exclude => :space).size
+    assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).size
+    assert_equal 9, tokens.token_kind_filter(:include => :space).size
+    assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).size
+    assert_equal 0, tokens.token_kind_filter(:exclude => :all).size
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens << [:open, :index]
+      tokens << [i.to_s, :content]
+      tokens << [:close, :index]
+    end
+    assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).size
+    assert_equal 20, tokens.token_kind_filter(:include => :blubb).size
+    assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).size
+    assert_equal 30, tokens.token_kind_filter(:exclude => :index).size
+  end
+  
+end