Renamed token_classes to token_kinds (closes #122).

* *RENAMED* from ClassOfKind. The term "token class" is no longer used in CodeRay. Instead, tokens have _kinds_.
* *RENAMED* TokenKindFilter to TokenClassFilter.
* *ADDED* documentation on encoders affected by the term change.
* *REMOVED* token kinds :attribute_name_fat, :attribute_value_fat, :operator_fat, :tag_fat, and :xml_text.
* *ADDED* token kind :filename.

10 files changed, 178 insertions, 142 deletions

diff --git a/Changes.textile b/Changes.textile
index deddb27..df6ac8e 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -15,12 +15,23 @@ h3. @coderay@ executable
 * *FIXED*: Warn about generated files.
 * *FIXED*: Ensure linebreak after the output (was problematic for LoC counter).
 
+h3. @Tokens::AbbreviationForKind@
+ 
+* *RENAMED* from @ClassOfKind@. The term "token class" is no longer used in CodeRay. Instead, tokens have _kinds_.
+  See "#122":http://redmine.rubychan.de/issues/122.
+* *REMOVED* token kinds @:attribute_name_fat@, @:attribute_value_fat@, @:operator_fat@, @:tag_fat@, and @:xml_text@.
+* *ADDED* token kind @:filename@.
+
 h3. @Encoders::Terminal@
  
 * *RENAMED* from @Encoders::Term@, added @:term@ alias.
 * *CLEANUP*: Use @#setup@'s @super@, don't use @:procedure@ token class.
 * *CHANGED*: @#token@'s second parameter is no longer optional.
 
+h3. @TokenKindFilter@
+ 
+* *RENAMED* from @TokenClassFilter@.
+
 h3. @Scanners@
  
 * *FIXED*: @scan_tokens@ and @setup@ methods are protected now.
diff --git a/lib/coderay.rb b/lib/coderay.rb
index df29a5a..3636714 100644
--- a/lib/coderay.rb
+++ b/lib/coderay.rb
@@ -126,22 +126,23 @@
 # If you want to re-use scanners and encoders (because that is faster), see
 # CodeRay::Duo for the most convenient (and recommended) interface.
 module CodeRay
-
+  
+  $CODERAY_DEBUG ||= false
+  
   # Version: Major.Minor.Teeny[.Revision]
   # Major: 0 for pre-stable, 1 for stable
   # Minor: feature milestone
   # Teeny: development state, 0 for pre-release
   # Revision: Subversion Revision number (generated on rake gem:make)
-  VERSION = '0.9.1'
+  VERSION = '1.0.0'
 
   require 'coderay/tokens'
-  require 'coderay/token_classes'
+  require 'coderay/token_kinds'
   require 'coderay/scanner'
   require 'coderay/encoder'
   require 'coderay/duo'
   require 'coderay/style'
 
-
   class << self
 
     # Scans the given +code+ (a String) with the Scanner for +lang+.
@@ -295,16 +296,20 @@ module CodeRay
   # This Exception is raised when you try to stream with something that is not
   # capable of streaming.
   class NotStreamableError < Exception
+    
+    # +obj+ is the object that is not streamable.
     def initialize obj
       @obj = obj
     end
-
-    def to_s
+    
+    def to_s  # :nodoc:
       '%s is not Streamable!' % @obj.class
     end
+    
   end
 
-  # A dummy module that is included by subclasses of CodeRay::Scanner an CodeRay::Encoder
+  # A dummy module that is included by subclasses of
+  # CodeRay::Scanners::Scanner and CodeRay::Encoders::Encoder
   # to show that they are able to handle streams.
   module Streamable
   end
diff --git a/lib/coderay/encoders/comment_filter.rb b/lib/coderay/encoders/comment_filter.rb
index 4d3fb54..819c619 100644
--- a/lib/coderay/encoders/comment_filter.rb
+++ b/lib/coderay/encoders/comment_filter.rb
@@ -2,9 +2,18 @@
 module CodeRay
 module Encoders
   
-  load :token_class_filter
+  load :token_kind_filter
   
-  class CommentFilter < TokenClassFilter
+  # A simple Filter that removes all tokens of the :comment kind.
+  # 
+  # Alias: +remove_comments+
+  # 
+  # Usage:
+  #  CodeRay.scan('print # foo', :ruby).remove_comments.text
+  #  #-> "print "
+  # 
+  # See also: TokenKindFilter, LinesOfCode
+  class CommentFilter < TokenKindFilter
     
     register_for :comment_filter
     
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index ffbc6ca..f1fe7ce 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -27,6 +27,7 @@ module Encoders
   #
   # === :tab_width
   # Convert \t characters to +n+ spaces (a number.)
+  # 
   # Default: 8
   #
   # === :css
@@ -125,7 +126,7 @@ module Encoders
     #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
     HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
 
-    TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
+    TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
       h[kind] =
         case kind
         when :pre_constant
@@ -133,24 +134,24 @@ module Encoders
         else
           kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
         end
-    }
+    end
 
     TRANSPARENT_TOKEN_KINDS = [
       :delimiter, :modifier, :content, :escape, :inline_delimiter,
     ].to_set
 
-    # Generate a hint about the given +classes+ in a +hint+ style.
+    # Generate a hint about the given +kinds+ in a +hint+ style.
     #
     # +hint+ may be :info, :info_long or :debug.
-    def self.token_path_to_hint hint, classes
+    def self.token_path_to_hint hint, kinds
       title =
         case hint
         when :info
-          TOKEN_KIND_TO_INFO[classes.first]
+          TOKEN_KIND_TO_INFO[kinds.first]
         when :info_long
-          classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
         when :debug
-          classes.inspect
+          kinds.inspect
         end
       title ? " title=\"#{title}\"" : ''
     end
@@ -174,7 +175,7 @@ module Encoders
 
       when :class
         @css_style = Hash.new do |h, k|
-          c = CodeRay::Tokens::ClassOfKind[k.first]
+          c = CodeRay::Tokens::AbbreviationForKind[k.first]
           if c == :NO_HIGHLIGHT and not hint
             h[k.dup] = false
           else
@@ -199,7 +200,7 @@ module Encoders
             styles = [k]
           end
           type = styles.first
-          classes = styles.map { |c| Tokens::ClassOfKind[c] }
+          classes = styles.map { |c| Tokens::AbbreviationForKind[c] }
           if classes.first == :NO_HIGHLIGHT and not hint
             h[k] = false
           else
@@ -261,13 +262,13 @@ module Encoders
         @out << (@css_style[@opened] || '<span>')
         @opened << type
       when :close
+        if $CODERAY_DEBUG and @opened.last != type
+          warn 'Malformed token stream: Trying to close a token (%p) ' \
+            'that is not open. Open are: %p.' % [type, @opened[1..-1]]
+        end
         if @opened.empty?
           # nothing to close
         else
-          if $DEBUG and (@opened.size == 1 or @opened.last != type)
-            raise 'Malformed token stream: Trying to close a token (%p) \
-              that is not open. Open are: %p.' % [type, @opened[1..-1]]
-          end
           @out << '</span>'
           @opened.pop
         end
@@ -282,13 +283,13 @@ module Encoders
         end
         @opened << type
       when :end_line
+        if $CODERAY_DEBUG and @opened.last != type
+          warn 'Malformed token stream: Trying to close a line (%p) ' \
+            'that is not open. Open are: %p.' % [type, @opened[1..-1]]
+        end
         if @opened.empty?
           # nothing to close
         else
-          if $DEBUG and (@opened.size == 1 or @opened.last != type)
-            raise 'Malformed token stream: Trying to close a line (%p) \
-              that is not open. Open are: %p.' % [type, @opened[1..-1]]
-          end
           @out << '</div>'
           @opened.pop
         end
diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb
index b24145e..60918c4 100644
--- a/lib/coderay/encoders/lines_of_code.rb
+++ b/lib/coderay/encoders/lines_of_code.rb
@@ -4,7 +4,7 @@ module Encoders
   
   # Counts the LoC (Lines of Code). Returns an Integer >= 0.
   # 
-  # Alias: :loc
+  # Alias: +loc+
   # 
   # Everything that is not comment, markup, doctype/shebang, or an empty line,
   # is considered to be code.
@@ -25,10 +25,10 @@ module Encoders
       if scanner = tokens.scanner
         kinds_not_loc = scanner.class::KINDS_NOT_LOC
       else
-        warn ArgumentError, 'Tokens have no scanner.' if $DEBUG
+        warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE
         kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
       end
-      code = tokens.token_class_filter :exclude => kinds_not_loc
+      code = tokens.token_kind_filter :exclude => kinds_not_loc
       @loc = code.text.scan(NON_EMPTY_LINE).size
     end
     
diff --git a/lib/coderay/encoders/token_class_filter.rb b/lib/coderay/encoders/token_class_filter.rb
deleted file mode 100644
index a9e8673..0000000
--- a/lib/coderay/encoders/token_class_filter.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
-module CodeRay
-module Encoders
-  
-  load :filter
-  
-  class TokenClassFilter < Filter
-
-    include Streamable
-    register_for :token_class_filter
-
-    DEFAULT_OPTIONS = {
-      :exclude => [],
-      :include => :all
-    }
-
-  protected
-    def setup options
-      super
-      @exclude = options[:exclude]
-      @exclude = Array(@exclude) unless @exclude == :all
-      @include = options[:include]
-      @include = Array(@include) unless @include == :all
-    end
-    
-    def include_text_token? text, kind
-       (@include == :all || @include.include?(kind)) &&
-      !(@exclude == :all || @exclude.include?(kind))
-    end
-    
-  end
-
-end
-end
-
-if $0 == __FILE__
-  $VERBOSE = true
-  $: << File.join(File.dirname(__FILE__), '..')
-  eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class TokenClassFilterTest < Test::Unit::TestCase
-  
-  def test_creation
-    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
-    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
-    filter = nil
-    assert_nothing_raised do
-      filter = CodeRay.encoder :token_class_filter
-    end
-    assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
-  end
-  
-  def test_filtering_text_tokens
-    tokens = CodeRay::Tokens.new
-    for i in 1..10
-      tokens << [i.to_s, :index]
-      tokens << [' ', :space] if i < 10
-    end
-    assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
-    assert_equal 10, tokens.token_class_filter(:exclude => :space).size
-    assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
-    assert_equal 9, tokens.token_class_filter(:include => :space).size
-    assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
-    assert_equal 0, tokens.token_class_filter(:exclude => :all).size
-  end
-  
-  def test_filtering_block_tokens
-    tokens = CodeRay::Tokens.new
-    10.times do |i|
-      tokens << [:open, :index]
-      tokens << [i.to_s, :content]
-      tokens << [:close, :index]
-    end
-    assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
-    assert_equal 20, tokens.token_class_filter(:include => :blubb).size
-    assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
-    assert_equal 30, tokens.token_class_filter(:exclude => :index).size
-  end
-  
-end
diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb
new file mode 100644
index 0000000..4b2f582
--- /dev/null
+++ b/lib/coderay/encoders/token_kind_filter.rb
@@ -0,0 +1,103 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+  
+  load :filter
+  
+  # A Filter that selects tokens based on their token kind.
+  # 
+  # == Options
+  # 
+  # === :exclude
+  # 
+  # One or many symbols (in an Array) which shall be excluded.
+  # 
+  # Default: []
+  # 
+  # === :include
+  # 
+  # One or many symbols (in an array) which shall be included.
+  # 
+  # Default: :all, which means all tokens are included.
+  # 
+  # Exclusion wins over inclusion.
+  # 
+  # See also: CommentFilter
+  class TokenKindFilter < Filter
+
+    include Streamable
+    register_for :token_kind_filter
+
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+
+  protected
+    def setup options
+      super
+      @exclude = options[:exclude]
+      @exclude = Array(@exclude) unless @exclude == :all
+      @include = options[:include]
+      @include = Array(@include) unless @include == :all
+    end
+    
+    def include_text_token? text, kind
+       (@include == :all || @include.include?(kind)) &&
+      !(@exclude == :all || @exclude.include?(kind))
+    end
+    
+  end
+
+end
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokenKindFilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :token_kind_filter
+    end
+    assert_instance_of CodeRay::Encoders::TokenKindFilter, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    for i in 1..10
+      tokens << [i.to_s, :index]
+      tokens << [' ', :space] if i < 10
+    end
+    assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).size
+    assert_equal 10, tokens.token_kind_filter(:exclude => :space).size
+    assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).size
+    assert_equal 9, tokens.token_kind_filter(:include => :space).size
+    assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).size
+    assert_equal 0, tokens.token_kind_filter(:exclude => :all).size
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens << [:open, :index]
+      tokens << [i.to_s, :content]
+      tokens << [:close, :index]
+    end
+    assert_equal 20, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).size
+    assert_equal 20, tokens.token_kind_filter(:include => :blubb).size
+    assert_equal 30, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).size
+    assert_equal 30, tokens.token_kind_filter(:exclude => :index).size
+  end
+  
+end
diff --git a/lib/coderay/styles/cycnus.rb b/lib/coderay/styles/cycnus.rb
index d2a4353..8196c9f 100644
--- a/lib/coderay/styles/cycnus.rb
+++ b/lib/coderay/styles/cycnus.rb
@@ -1,6 +1,7 @@
 module CodeRay
 module Styles
-
+  
+  # A colorful theme that is also the default for CodeRay output.
   class Cycnus < Style
 
     register_for :cycnus
@@ -10,7 +11,7 @@ module Styles
     border_color = 'silver'
     normal_color = '#000'
 
-    CSS_MAIN_STYLES = <<-MAIN
+    CSS_MAIN_STYLES = <<-MAIN  # :nodoc:
 .CodeRay {
   background-color: #{code_background};
   border: 1px solid #{border_color};
@@ -45,11 +46,9 @@ ol.CodeRay li { white-space: pre }
     TOKEN_COLORS = <<-'TOKENS'
 .debug { color:white ! important; background:blue ! important; }
 
-.af { color:#00C }
 .an { color:#007 }
 .at { color:#f08 }
 .av { color:#700 }
-.aw { color:#C00 }
 .bi { color:#509; font-weight:bold }
 .c  { color:#888; }
 
@@ -92,7 +91,6 @@ ol.CodeRay li { white-space: pre }
 .la { color:#970; font-weight:bold }
 .lv { color:#963 }
 .oc { color:#40E; font-weight:bold }
-.of { color:#000; font-weight:bold }
 .op { }
 .pc { color:#038; font-weight:bold }
 .pd { color:#369; font-weight:bold }
@@ -128,7 +126,6 @@ ol.CodeRay li { white-space: pre }
 .sy .dl { color:#630 }
 
 .ta { color:#070 }
-.tf { color:#070; font-weight:bold }
 .ts { color:#D70; font-weight:bold }
 .ty { color:#339; font-weight:bold }
 .v  { color:#036 }
@@ -138,6 +135,7 @@ ol.CodeRay li { white-space: pre }
 .del { background: #faa; }
 .chg { color: #aaf; background: #007; }
 .head { color: #f8f; background: #505 }
+.head .filename { color: white; }
 
 .ins .ins { color: #080; font-weight:bold }
 .del .del { color: #800; font-weight:bold }
diff --git a/lib/coderay/styles/murphy.rb b/lib/coderay/styles/murphy.rb
index 211ab6b..033d949 100644
--- a/lib/coderay/styles/murphy.rb
+++ b/lib/coderay/styles/murphy.rb
@@ -1,6 +1,7 @@
 module CodeRay
 module Styles
 
+  # A alternative color theme.
   class Murphy < Style
 
     register_for :murphy
@@ -10,7 +11,7 @@ module Styles
     border_color = 'silver'
     normal_color = '#C0C0C0'
 
-    CSS_MAIN_STYLES = <<-MAIN
+    CSS_MAIN_STYLES = <<-MAIN  # :nodoc:
 .CodeRay {
   background-color: #{code_background};
   border: 1px solid #{border_color};
@@ -42,10 +43,8 @@ ol.CodeRay li { white-space: pre; }
     MAIN
 
     TOKEN_COLORS = <<-'TOKENS'
-.af { color:#00C; }
 .an { color:#007; }
 .av { color:#700; }
-.aw { color:#C00; }
 .bi { color:#509; font-weight:bold; }
 .c  { color:#555; background-color: black; }
 
@@ -77,7 +76,6 @@ ol.CodeRay li { white-space: pre; }
 .la { color:#970; font-weight:bold; }
 .lv { color:#963; }
 .oc { color:#40E; font-weight:bold; }
-.of { color:#000; font-weight:bold; }
 .op { }
 .pc { color:#08f; font-weight:bold; }
 .pd { color:#369; font-weight:bold; }
@@ -109,7 +107,6 @@ ol.CodeRay li { white-space: pre; }
 .sy .dl { color:#F84; }
 
 .ta { color:#070; }
-.tf { color:#070; font-weight:bold; }
 .ts { color:#D70; font-weight:bold; }
 .ty { color:#339; font-weight:bold; }
 .v  { color:#036; }
diff --git a/lib/coderay/token_classes.rb b/lib/coderay/token_kinds.rb
index ae35c0f..c8e370c 100755
--- a/lib/coderay/token_classes.rb
+++ b/lib/coderay/token_kinds.rb
@@ -1,14 +1,12 @@
 module CodeRay
   class Tokens
-    ClassOfKind = Hash.new do |h, k|
-      h[k] = k.to_s
+    AbbreviationForKind = Hash.new do |h, k|  # :nodoc:
+      raise 'Undefined Token kind: %p' % [k]  # :nodoc:
     end
-    ClassOfKind.update with = {
+    AbbreviationForKind.update with = {  # :nodoc:
       :annotation => 'at',
       :attribute_name => 'an',
-      :attribute_name_fat => 'af',
       :attribute_value => 'av',
-      :attribute_value_fat => 'aw',
       :bin => 'bi',
       :char => 'ch',
       :class => 'cl',
@@ -29,6 +27,7 @@ module CodeRay
       :error => 'er',
       :escape => 'e',
       :exception => 'ex',
+      :filename => 'filename',
       :float => 'fl',
       :function => 'fu',
       :global_variable => 'gv',
@@ -41,17 +40,16 @@ module CodeRay
       :instance_variable => 'iv',
       :integer => 'i',
       :interpreted => 'in',
-      :keyword => 'kw',
       :key => 'ke',
+      :keyword => 'kw',
       :label => 'la',
       :local_variable => 'lv',
       :modifier => 'mod',
       :oct => 'oc',
-      :operator_fat => 'of',
-      :pre_constant => 'pc',
-      :pre_type => 'pt',
       :predefined => 'pd',
       :preprocessor => 'pp',
+      :pre_constant => 'pc',
+      :pre_type => 'pt',
       :pseudo_class => 'ps',
       :regexp => 'rx',
       :reserved => 'r',
@@ -59,12 +57,10 @@ module CodeRay
       :string => 's',
       :symbol => 'sy',
       :tag => 'ta',
-      :tag_fat => 'tf',
       :tag_special => 'ts',
       :type => 'ty',
-      :variable => 'v',
       :value => 'vl',
-      :xml_text => 'xt',
+      :variable => 'v',
       
       :insert => 'ins',
       :delete => 'del',
@@ -77,10 +73,10 @@ module CodeRay
       :space => :NO_HIGHLIGHT,  # 'sp'
       :plain => :NO_HIGHLIGHT,
     }
-    ClassOfKind[:method] = ClassOfKind[:function]
-    ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
-    ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
-    ClassOfKind[:escape] = ClassOfKind[:delimiter]
-    #ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+    AbbreviationForKind[:method] = AbbreviationForKind[:function]
+    AbbreviationForKind[:open] = AbbreviationForKind[:close] = AbbreviationForKind[:delimiter]
+    AbbreviationForKind[:nesting_delimiter] = AbbreviationForKind[:delimiter]
+    AbbreviationForKind[:escape] = AbbreviationForKind[:delimiter]
+    #AbbreviationForKind.default = AbbreviationForKind[:error] or raise 'no class found for :error!'
   end
-end
\ No newline at end of file
+end