Fixed example tests.

* test:clean task also deletes .expected.html files
* LinesOfCode encoder can deal with tokens that have no scanner.
  Tests were added for this.
* JSON encoder load rubygems if necessary.
* NEW :loc as an alias for :lines_of_code
* NEW Scanner methods marshal_dump, marshal_load
  FIXED Tokens dumping (failed while trying to dump @scanner)

13 files changed, 63 insertions, 69 deletions

diff --git a/sample/css.expected b/sample/css.expected
index 7c02f53..09709ff 100644
--- a/sample/css.expected
+++ b/sample/css.expected
@@ -57,7 +57,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -82,8 +82,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/div.expected b/sample/div.expected
index 9ad26cb..f28ede3 100644
--- a/sample/div.expected
+++ b/sample/div.expected
@@ -2,14 +2,14 @@
   <div class="code"><pre><span style="color:#080;font-weight:bold">for</span> a <span style="color:#080;font-weight:bold">in</span> <span style="color:#00D;font-weight:bold">0</span>..<span style="color:#00D;font-weight:bold">255</span>
         a = a.chr
         <span style="color:#080;font-weight:bold">begin</span>
-                x = eval(<span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="">?</span><span style="color:#b0b">\\</span><span style="background:#eee;color:black"><span style="font-weight:bold;color:#777">#{</span>a<span style="font-weight:bold;color:#777">}</span></span><span style="color:#710">&quot;</span></span>)
+                x = eval(<span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="">?</span><span style="color:#b0b">\\</span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>a<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>)
                 <span style="color:#080;font-weight:bold">if</span> x == a[<span style="color:#00D;font-weight:bold">0</span>]
                         <span style="color:#080;font-weight:bold">next</span>
                 <span style="color:#080;font-weight:bold">else</span>
-                        print <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="background:#eee;color:black"><span style="font-weight:bold;color:#777">#{</span>a<span style="font-weight:bold;color:#777">}</span></span><span style="">: </span><span style="background:#eee;color:black"><span style="font-weight:bold;color:#777">#{</span>x<span style="font-weight:bold;color:#777">}</span></span><span style="color:#710">&quot;</span></span>
+                        print <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>a<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="">: </span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>x<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>
                 <span style="color:#080;font-weight:bold">end</span>
         <span style="color:#080;font-weight:bold">rescue</span> <span style="color:#036;font-weight:bold">SyntaxError</span> =&gt; boom
-                print <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="background:#eee;color:black"><span style="font-weight:bold;color:#777">#{</span>a<span style="font-weight:bold;color:#777">}</span></span><span style="">: error</span><span style="color:#710">&quot;</span></span>
+                print <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>a<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style="">: error</span><span style="color:#710">&quot;</span></span>
         <span style="color:#080;font-weight:bold">end</span>
         puts
 <span style="color:#080;font-weight:bold">end</span>
diff --git a/sample/dump.expected b/sample/dump.expected
index 2c24962..a451686 100644
--- a/sample/dump.expected
+++ b/sample/dump.expected
@@ -1,5 +1,5 @@
 YAML: 2358 bytes
-Dump: 1058 bytes
+Dump: 1109 bytes
 undumped:
 <div class="CodeRay">
   <div class="code"><pre>require <span class="s"><span class="dl">'</span><span class="k">coderay</span><span class="dl">'</span></span>
diff --git a/sample/encoder.expected b/sample/encoder.expected
index c221d25..438032a 100644
--- a/sample/encoder.expected
+++ b/sample/encoder.expected
@@ -19,15 +19,7 @@ Token Types (4):
 
 
 Original text:
-ident	puts
-space	 
-integer	17
-space	 
-operator	+
-space	 
-integer	4
-space	\
-
+[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}]
 
 YAML:
 --- 
@@ -51,8 +43,8 @@ YAML:
   - :space
 
 Dump:
-"x\332\355\314;\n\302@\024\205aP\311c\320\316\005\004[+A\020\356\224\331\201\330\245\n\346\"A\230\0312c\341\356M\"\242k\220\277\272\217\303wVE-\333\332wzn\237\"\027\177W\027\233E\265l\362]\031\036)\212\351;ui<\263JL\f\355U\307=?\234d\335\273\2447\035\346\310\346\323\330\313\306\a\035\332\344\177\277G[L\303\314\327\\j\263o<V\275\363O\207-\261X,\026\213\305b\261X,\026\213\305b\261X,\026\213\375{\373\002\212L\274o"
-compressed: 150 byte < 1200 byte
+"x\332\355\330\273\n\302@\020\005PQIL4\235\245E\260\265\022\004a\266\021\002B\332\250U\2525\031$\210\233\260\273)\202?o\036\370\370\006\271\325\354\314\345\334\017\330\351,\216h\031\2259'\262!:\227wV&\035\207\223\324]{Um\r\371E\316\312\266\253\023\222o*\231q\373v\267{Z\024\312\362\215u\037\t\267\e\e\n\312\212\265\264\345\357u'f\335\360\373\255/\025\3167n\253\206\374\335!<XXXXXXXXXXXXXXXXXXXXX\330\277\267\016\005\a\223I\245X\307\027Z}\276\352O\303\315\020%\365\265y:47V\263|\034G/\301K\315)"
+compressed: 188 byte < 1200 byte
 
 Undump:
 
diff --git a/sample/encoder.rb b/sample/encoder.rb
index 267676b..22eaa22 100644
--- a/sample/encoder.rb
+++ b/sample/encoder.rb
@@ -14,7 +14,7 @@ puts stats
 

 # alternative 1

 tokens = CodeRay.scan SAMPLE, :ruby

-encoder = CodeRay.encoder(:tokens)

+encoder = CodeRay.encoder(:json)

 textual = encoder.encode_tokens tokens

 puts

 puts 'Original text:'

diff --git a/sample/highlight.expected b/sample/highlight.expected
index afcdd1d..6a9b278 100644
--- a/sample/highlight.expected
+++ b/sample/highlight.expected
@@ -49,9 +49,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -61,7 +63,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -69,10 +71,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -85,8 +88,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
@@ -159,9 +162,9 @@ ol.CodeRay li { white-space: pre }
 </tt>puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&lt;&lt;HTML</span></span><span style="background-color:#fff0f0;color:#D20"><span style=""><tt>
 </tt>&lt;html&gt;<tt>
 </tt>&lt;head&gt;<tt>
-</tt></span><span style="background:#eee;color:black"><span style="font-weight:bold;color:#777">#{</span>output.stylesheet <span style="color:#038;font-weight:bold">true</span><span style="font-weight:bold;color:#777">}</span></span><span style=""><tt>
+</tt></span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>output.stylesheet <span style="color:#038;font-weight:bold">true</span><span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style=""><tt>
 </tt>&lt;body&gt;<tt>
-</tt></span><span style="background:#eee;color:black"><span style="font-weight:bold;color:#777">#{</span>output<span style="font-weight:bold;color:#777">}</span></span><span style=""><tt>
+</tt></span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>output<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style=""><tt>
 </tt>&lt;/body&gt;<tt>
 </tt>&lt;/html&gt;</span><span style="color:#710"><tt>
 </tt>HTML</span></span><tt>
diff --git a/sample/html.expected b/sample/html.expected
index 25616dc..e98d589 100644
--- a/sample/html.expected
+++ b/sample/html.expected
@@ -50,9 +50,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -62,7 +64,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -70,10 +72,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -86,8 +89,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/html2.expected b/sample/html2.expected
index 829ffbf..c8ae56a 100644
--- a/sample/html2.expected
+++ b/sample/html2.expected
@@ -50,9 +50,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -62,7 +64,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -70,10 +72,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -86,8 +89,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/html_list.expected b/sample/html_list.expected
index 5351672..a4092c8 100644
--- a/sample/html_list.expected
+++ b/sample/html_list.expected
@@ -50,9 +50,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .ch .dl { color:#039 }
 
 .CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
 .CodeRay .co { color:#036; font-weight:bold }
 .CodeRay .cr { color:#0A0 }
 .CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
 .CodeRay .df { color:#099; font-weight:bold }
 .CodeRay .di { color:#088; font-weight:bold }
 .CodeRay .dl { color:black }
@@ -62,7 +64,7 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .e  { color:#666; font-weight:bold }
 .CodeRay .en { color:#800; font-weight:bold }
 .CodeRay .er { color:#F00; background-color:#FAA }
-.CodeRay .ex { color:#F00; font-weight:bold }
+.CodeRay .ex { color:#C00; font-weight:bold }
 .CodeRay .fl { color:#60E; font-weight:bold }
 .CodeRay .fu { color:#06B; font-weight:bold }
 .CodeRay .gv { color:#d70; font-weight:bold }
@@ -70,10 +72,11 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .i  { color:#00D; font-weight:bold }
 .CodeRay .ic { color:#B44; font-weight:bold }
 
-.CodeRay .il { background: #eee; color: black }
-.CodeRay .il .il { background: #ddd }
-.CodeRay .il .il .il { background: #ccc }
-.CodeRay .il .idl { font-weight: bold; color: #777 }
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .CodeRay .im { color:#f00; }
 .CodeRay .in { color:#B2B; font-weight:bold }
@@ -86,8 +89,8 @@ ol.CodeRay li { white-space: pre }
 .CodeRay .pc { color:#038; font-weight:bold }
 .CodeRay .pd { color:#369; font-weight:bold }
 .CodeRay .pp { color:#579; }
-.CodeRay .ps { color:#00C; font-weight: bold; }
-.CodeRay .pt { color:#349; font-weight:bold }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
 .CodeRay .r, .kw  { color:#080; font-weight:bold }
 
 .CodeRay .ke { color: #808; }
diff --git a/sample/more.expected b/sample/more.expected
index 61fabe6..196904d 100644
--- a/sample/more.expected
+++ b/sample/more.expected
@@ -1,2 +1,2 @@
-Input: 4983B, Output: 23316B
+Input: 4983B, Output: 23484B
 Take a look with your browser.
diff --git a/sample/suite.rb b/sample/suite.rb
index c586697..fa24114 100644
--- a/sample/suite.rb
+++ b/sample/suite.rb
@@ -28,16 +28,19 @@ class CodeRaySuite < TestCase
         name = File.basename(input, ".rb")
         output = name + '.expected'
         code = File.open(input, 'rb') { |f| break f.read }
-
+        
         result = `ruby -wI../lib #{input}`
-
+        
+        diff = output.sub '.expected', '.diff'
+        File.delete diff if File.exist? diff
+        computed = output.sub '.expected', '.actual'
         if File.exist? output
           expected = File.read output
           ok = expected == result
-          computed = output.sub('.expected', '.actual')
           unless ok
             File.open(computed, 'w') { |f| f.write result }
-            print `diff #{output} #{computed}` if $DEBUG
+            `diff #{output} #{computed} > #{diff}` if $DEBUG
+            puts "Test failed; output written to #{diff}."
           end
           assert(ok, "Output error: #{computed} != #{output}") unless $DEBUG
         else
diff --git a/sample/tokens.expected b/sample/tokens.expected
index bbc2b94..747904e 100644
--- a/sample/tokens.expected
+++ b/sample/tokens.expected
@@ -1,14 +1 @@
-ident	puts
-space	 
-integer	3
-space	 
-operator	+
-space	 
-integer	4
-operator	,
-space	 
-:open	string	
-delimiter	'
-content	3 + 4
-delimiter	'
-:close	string	
+[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]]
diff --git a/sample/tokens.rb b/sample/tokens.rb
index eb8d448..91b8abb 100644
--- a/sample/tokens.rb
+++ b/sample/tokens.rb
@@ -1,3 +1,3 @@
 require 'coderay'

 

-puts CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby).tokens

+p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby)