Encoder refactoring to fix LOC and Statistic encoders

5 files changed, 37 insertions, 39 deletions

diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index 3acb9c4..e495ca0 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -46,7 +46,7 @@ module CodeRay
       DEFAULT_OPTIONS = { }
       
       # The options you gave the Encoder at creating.
-      attr_accessor :options
+      attr_accessor :options, :scanner
       
       # Creates a new Encoder.
       # +options+ is saved and used for all encode operations, as long
@@ -66,6 +66,7 @@ module CodeRay
       # Encode a Tokens object.
       def encode_tokens tokens, options = {}
         options = @options.merge options
+        @scanner = tokens.scanner if tokens.respond_to? :scanner
         setup options
         compile tokens, options
         finish options
@@ -74,10 +75,9 @@ module CodeRay
       # Encode the given +code+ using the Scanner for +lang+.
       def encode code, lang, options = {}
         options = @options.merge options
+        @scanner = Scanners[lang].new code, CodeRay.get_scanner_options(options).update(:tokens => self)
         setup options
-        scanner_options = CodeRay.get_scanner_options options
-        scanner_options[:tokens] = self
-        CodeRay.scan code, lang, scanner_options
+        @scanner.tokenize
         finish options
       end
       
diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb
index 4215d23..c49cade 100644
--- a/lib/coderay/encoders/lines_of_code.rb
+++ b/lib/coderay/encoders/lines_of_code.rb
@@ -14,7 +14,7 @@ module Encoders
   # 
   # A Scanner class should define the token kinds that are not code in the
   # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
-  class LinesOfCode < Encoder
+  class LinesOfCode < TokenKindFilter
     
     register_for :lines_of_code
     
@@ -23,22 +23,19 @@ module Encoders
   protected
     
     def setup options
-      @out = 0
-    end
-    
-    def compile tokens, options
-      if scanner = tokens.scanner
+      if scanner
         kinds_not_loc = scanner.class::KINDS_NOT_LOC
       else
         warn "Tokens have no associated scanner, counting all nonempty lines." if $VERBOSE
         kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
       end
-      code = tokens.token_kind_filter :exclude => kinds_not_loc
-      @out = code.to_s.scan(NON_EMPTY_LINE).size
+      
+      options[:exclude] = kinds_not_loc
+      super options
     end
     
     def finish options
-      @out
+      @out.to_s.scan(NON_EMPTY_LINE).size
     end
     
   end
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index c6ffd7a..4a6a845 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -1,29 +1,29 @@
 module CodeRay
 module Encoders
-
+  
   # Makes a statistic for the given tokens.
   # 
   # Alias: +stats+
   class Statistic < Encoder
-
+    
     register_for :stats, :statistic
-
+    
     attr_reader :type_stats, :real_token_count  # :nodoc:
-
+    
     TypeStats = Struct.new :count, :size  # :nodoc:
-
+    
   protected
-
+    
     def setup options
       @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
       @real_token_count = 0
     end
-
+    
     def generate tokens, options
       @tokens = tokens
       super
     end
-  
+    
     STATS = <<-STATS  # :nodoc:
 
 Code Statistics
@@ -36,12 +36,12 @@ Token Types (%d):
   type                     count     ratio    size (average)
 -------------------------------------------------------------
 %s
-      STATS
-# space                    12007   33.81 %     1.7
+    STATS
+    
     TOKEN_TYPES_ROW = <<-TKR  # :nodoc:
   %-20s  %8d  %6.2f %%   %5.1f
-      TKR
-
+    TKR
+    
     def finish options
       all = @type_stats['TOTAL']
       all_count, all_size = all.count, all.size
@@ -57,7 +57,7 @@ Token Types (%d):
         types_stats
       ]
     end
-
+    
   public
     
     def text_token text, kind
@@ -67,30 +67,31 @@ Token Types (%d):
       @type_stats['TOTAL'].size += text.size
       @type_stats['TOTAL'].count += 1
     end
-
+    
     # TODO Hierarchy handling
     def begin_group kind
-      block_token 'begin_group'
+      block_token ':begin_group', kind
     end
-
+    
     def end_group kind
-      block_token 'end_group'
+      block_token ':end_group', kind
     end
-
+    
     def begin_line kind
-      block_token 'begin_line'
+      block_token ':begin_line', kind
     end
-
+    
     def end_line kind
-      block_token 'end_line'
+      block_token ':end_line', kind
     end
     
-    def block_token action
+    def block_token action, kind
       @type_stats['TOTAL'].count += 1
       @type_stats[action].count += 1
+      @type_stats[kind].count += 1
     end
-
+    
   end
-
+  
 end
 end
diff --git a/lib/coderay/scanners/css.rb b/lib/coderay/scanners/css.rb
index ee76234..6413f8f 100644
--- a/lib/coderay/scanners/css.rb
+++ b/lib/coderay/scanners/css.rb
@@ -9,7 +9,7 @@ module Scanners
       :comment,
       :class, :pseudo_class, :type,
       :constant, :directive,
-      :key, :value, :operator, :color, :float,
+      :key, :value, :operator, :color, :float, :string,
       :error, :important,
     ]  # :nodoc:
     
diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 659ed39..a2e8553 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -13,7 +13,7 @@ module Scanners
     KINDS_NOT_LOC = [
       :comment, :doctype, :preprocessor,
       :tag, :attribute_name, :operator,
-      :attribute_value, :delimiter, :content,
+      :attribute_value, :string,
       :plain, :entity, :error,
     ]  # :nodoc: