Fixed example tests.

* test:clean task also deletes .expected.html files
* LinesOfCode encoder can deal with tokens that have no scanner.
  Tests were added for this.
* JSON encoder load rubygems if necessary.
* NEW :loc as an alias for :lines_of_code
* NEW Scanner methods marshal_dump, marshal_load
  FIXED Tokens dumping (failed while trying to dump @scanner)

4 files changed, 77 insertions, 5 deletions

diff --git a/lib/coderay/encoders/_map.rb b/lib/coderay/encoders/_map.rb
index 8e9732b..a66fcb8 100644
--- a/lib/coderay/encoders/_map.rb
+++ b/lib/coderay/encoders/_map.rb
@@ -1,8 +1,10 @@
 module CodeRay
 module Encoders
 
-  map :stats => :statistic,
+  map \
+    :loc => :lines_of_code,
     :plain => :text,
+    :stats => :statistic,
     :tex => :latex
 
 end
diff --git a/lib/coderay/encoders/json.rb b/lib/coderay/encoders/json.rb
index 072f334..7aa077c 100644
--- a/lib/coderay/encoders/json.rb
+++ b/lib/coderay/encoders/json.rb
@@ -10,7 +10,12 @@ module Encoders
     
   protected
     def setup options
-      require 'json'
+      begin
+        require 'json'
+      rescue LoadError
+        require 'rubygems'
+        require 'json'
+      end
       @out = []
     end
     
diff --git a/lib/coderay/encoders/lines_of_code.rb b/lib/coderay/encoders/lines_of_code.rb
index 32c0192..b24145e 100644
--- a/lib/coderay/encoders/lines_of_code.rb
+++ b/lib/coderay/encoders/lines_of_code.rb
@@ -1,8 +1,11 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
   # Counts the LoC (Lines of Code). Returns an Integer >= 0.
   # 
+  # Alias: :loc
+  # 
   # Everything that is not comment, markup, doctype/shebang, or an empty line,
   # is considered to be code.
   # 
@@ -11,7 +14,7 @@ module Encoders
   # * in a Java class without comments, LoC is the number of non-empty lines
   # 
   # A Scanner class should define the token kinds that are not code in the
-  # KINDS_NOT_LOC constant.
+  # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
   class LinesOfCode < Encoder
     
     register_for :lines_of_code
@@ -19,7 +22,12 @@ module Encoders
     NON_EMPTY_LINE = /^\s*\S.*$/
     
     def compile tokens, options
-      kinds_not_loc = tokens.scanner.class::KINDS_NOT_LOC
+      if scanner = tokens.scanner
+        kinds_not_loc = scanner.class::KINDS_NOT_LOC
+      else
+        warn ArgumentError, 'Tokens have no scanner.' if $DEBUG
+        kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
+      end
       code = tokens.token_class_filter :exclude => kinds_not_loc
       @loc = code.text.scan(NON_EMPTY_LINE).size
     end
@@ -32,3 +40,51 @@ module Encoders
   
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class LinesOfCodeTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :loc
+    end
+    assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+    assert_nothing_raised do
+      filter = CodeRay.encoder :lines_of_code
+    end
+    assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+  end
+  
+  def test_lines_of_code
+    tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+    assert_equal 1, tokens.lines_of_code
+    assert_equal 1, tokens.loc
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    tokens << ["Hello\n", :world]
+    tokens << ["Hello\n", :space]
+    tokens << ["Hello\n", :comment]
+    assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+    assert_equal 2, tokens.lines_of_code
+    assert_equal 2, tokens.loc
+  end
+  
+end
\ No newline at end of file
diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 229ba19..8233b2c 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -114,9 +114,10 @@ module CodeRay
       #
       # Else, a Tokens object is used.
       def initialize code='', options = {}, &block
-        @options = self.class::DEFAULT_OPTIONS.merge options
         raise "I am only the basic Scanner class. I can't scan "\
           "anything. :( Use my subclasses." if self.class == Scanner
+        
+        @options = self.class::DEFAULT_OPTIONS.merge options
 
         super Scanner.normify(code)
 
@@ -199,6 +200,14 @@ module CodeRay
         end
         pos - (string.rindex(?\n, pos) || 0)
       end
+      
+      def marshal_dump
+        @options
+      end
+      
+      def marshal_load options
+        @options = options
+      end
 
     protected