Merge branch 'master' into go-scanner

Conflicts:
	lib/coderay/helpers/file_type.rb

49 files changed, 3013 insertions, 13 deletions

diff --git a/test/executable/suite.rb b/test/executable/suite.rb
index d386f4b..ac0ff1d 100644
--- a/test/executable/suite.rb
+++ b/test/executable/suite.rb
@@ -14,13 +14,13 @@ class TestCodeRayExecutable < Test::Unit::TestCase
   
   ROOT_DIR = Pathname.new(File.dirname(__FILE__)) + '..' + '..'
   EXECUTABLE = ROOT_DIR + 'bin' + 'coderay'
-  RUBY_COMMAND = RUBY_VERSION < '2.0.0' ? 'ruby -w' : 'ruby'  # Ruby 2 currently throws warnings for bundler
+  RUBY_COMMAND = 'ruby'
   EXE_COMMAND =
     if RUBY_PLATFORM === 'java' && `ruby --ng -e '' 2> /dev/null` && $?.success?
       # use Nailgun
-      "#{RUBY_COMMAND}--ng -I%s %s"
+      "#{RUBY_COMMAND}--ng -w -I%s %s"
     else
-      "#{RUBY_COMMAND} -I%s %s"
+      "#{RUBY_COMMAND} -w -I%s %s"
     end % [ROOT_DIR + 'lib', EXECUTABLE]
   
   def coderay args, options = {}
diff --git a/test/functional/examples.rb b/test/functional/examples.rb
index 15f9ca3..985ef87 100755
--- a/test/functional/examples.rb
+++ b/test/functional/examples.rb
@@ -114,7 +114,7 @@ Token Types (7):
     
     # format the tokens
     term = terminal_encoder.encode_tokens(tokens)
-    assert_equal "\e[1;31mimport\e[0m \e[33mthis\e[0m;  \e[37m# The Zen of Python\e[0m", term
+    assert_equal "\e[32mimport\e[0m \e[31mthis\e[0m;  \e[1;30m# The Zen of Python\e[0m", term
     
     # re-using scanner and encoder
     ruby_highlighter = CodeRay::Duo[:ruby, :div]
diff --git a/test/samples/README b/test/samples/README
new file mode 100644
index 0000000..b0ab6e4
--- /dev/null
+++ b/test/samples/README
@@ -0,0 +1 @@
+These demos rely on Ruby 1.8.5, so the tests might fail on other versions.
\ No newline at end of file
diff --git a/test/samples/cache.actual b/test/samples/cache.actual
new file mode 100644
index 0000000..c131857
--- /dev/null
+++ b/test/samples/cache.actual
@@ -0,0 +1,2 @@
+test &lt;test&gt;
+test <span class="tag">&lt;test&gt;</span>
diff --git a/test/samples/cache.expected b/test/samples/cache.expected
new file mode 100644
index 0000000..c131857
--- /dev/null
+++ b/test/samples/cache.expected
@@ -0,0 +1,2 @@
+test &lt;test&gt;
+test <span class="tag">&lt;test&gt;</span>
diff --git a/test/samples/cache.rb b/test/samples/cache.rb
new file mode 100644
index 0000000..0c0b847
--- /dev/null
+++ b/test/samples/cache.rb
@@ -0,0 +1,12 @@
+require 'coderay'

+

+html_encoder = CodeRay.encoder :html

+

+scanner = Hash.new do |h, lang|

+	h[lang] = CodeRay.scanner lang

+end

+

+for lang in [:ruby, :html]

+	tokens = scanner[lang].tokenize 'test <test>'

+	puts html_encoder.encode_tokens(tokens)

+end

diff --git a/test/samples/count.expected b/test/samples/count.expected
new file mode 100644
index 0000000..7f493b6
--- /dev/null
+++ b/test/samples/count.expected
@@ -0,0 +1 @@
+2 out of 4 tokens have the kind :integer.
diff --git a/test/samples/count.rb b/test/samples/count.rb
new file mode 100644
index 0000000..bcb7c2d
--- /dev/null
+++ b/test/samples/count.rb
@@ -0,0 +1,10 @@
+require 'coderay'

+

+stats = CodeRay.encoder(:statistic)

+stats.encode("puts 17 + 4\n", :ruby)

+

+puts '%d out of %d tokens have the kind :integer.' % [

+	stats.type_stats[:integer].count,

+	stats.real_token_count

+]

+#-> 2 out of 4 tokens have the kind :integer.

diff --git a/test/samples/css.actual b/test/samples/css.actual
new file mode 100644
index 0000000..be73a7f
--- /dev/null
+++ b/test/samples/css.actual
@@ -0,0 +1,127 @@
+.CodeRay {
+  background-color: hsl(0,0%,95%);
+  border: 1px solid silver;
+  color: black;
+}
+.CodeRay pre {
+  margin: 0px;
+}
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line-numbers {
+  background-color: hsl(180,65%,90%);
+  color: gray;
+  text-align: right;
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+}
+.CodeRay .line-numbers a {
+  background-color: hsl(180,65%,90%) !important;
+  color: gray !important;
+  text-decoration: none !important;
+}
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
+.CodeRay .line-numbers a:target { color: blue !important; }
+.CodeRay .line-numbers .highlighted { color: red !important; }
+.CodeRay .line-numbers .highlighted a { color: red !important; }
+.CodeRay span.line-numbers { padding: 0px 4px; }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .code { width: 100%; }
+
+.CodeRay .debug { color: white !important; background: blue !important; }
+
+.CodeRay .annotation { color:#007 }
+.CodeRay .attribute-name { color:#b48 }
+.CodeRay .attribute-value { color:#700 }
+.CodeRay .binary { color:#549 }
+.CodeRay .binary .char { color:#325 }
+.CodeRay .binary .delimiter { color:#325 }
+.CodeRay .char { color:#D20 }
+.CodeRay .char .content { color:#D20 }
+.CodeRay .char .delimiter { color:#710 }
+.CodeRay .class { color:#B06; font-weight:bold }
+.CodeRay .class-variable { color:#369 }
+.CodeRay .color { color:#0A0 }
+.CodeRay .comment { color:#777 }
+.CodeRay .comment .char { color:#444 }
+.CodeRay .comment .delimiter { color:#444 }
+.CodeRay .constant { color:#036; font-weight:bold }
+.CodeRay .decorator { color:#B0B }
+.CodeRay .definition { color:#099; font-weight:bold }
+.CodeRay .delimiter { color:black }
+.CodeRay .directive { color:#088; font-weight:bold }
+.CodeRay .docstring { color:#D42; }
+.CodeRay .doctype { color:#34b }
+.CodeRay .done { text-decoration: line-through; color: gray }
+.CodeRay .entity { color:#800; font-weight:bold }
+.CodeRay .error { color:#F00; background-color:#FAA }
+.CodeRay .escape  { color:#666 }
+.CodeRay .exception { color:#C00; font-weight:bold }
+.CodeRay .float { color:#60E }
+.CodeRay .function { color:#06B; font-weight:bold }
+.CodeRay .function .delimiter { color:#024; font-weight:bold }
+.CodeRay .global-variable { color:#d70 }
+.CodeRay .hex { color:#02b }
+.CodeRay .id  { color:#33D; font-weight:bold }
+.CodeRay .include { color:#B44; font-weight:bold }
+.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black }
+.CodeRay .inline-delimiter { font-weight: bold; color: #666 }
+.CodeRay .instance-variable { color:#33B }
+.CodeRay .integer  { color:#00D }
+.CodeRay .imaginary { color:#f00 }
+.CodeRay .important { color:#D00 }
+.CodeRay .key { color: #606 }
+.CodeRay .key .char { color: #60f }
+.CodeRay .key .delimiter { color: #404 }
+.CodeRay .keyword { color:#080; font-weight:bold }
+.CodeRay .label { color:#970; font-weight:bold }
+.CodeRay .local-variable { color:#963 }
+.CodeRay .namespace { color:#707; font-weight:bold }
+.CodeRay .octal { color:#40E }
+.CodeRay .operator { }
+.CodeRay .predefined { color:#369; font-weight:bold }
+.CodeRay .predefined-constant { color:#069 }
+.CodeRay .predefined-type { color:#0a5; font-weight:bold }
+.CodeRay .preprocessor { color:#579 }
+.CodeRay .pseudo-class { color:#00C; font-weight:bold }
+.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); }
+.CodeRay .regexp .content { color:#808 }
+.CodeRay .regexp .delimiter { color:#404 }
+.CodeRay .regexp .modifier { color:#C2C }
+.CodeRay .reserved { color:#080; font-weight:bold }
+.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); }
+.CodeRay .shell .content { color:#2B2 }
+.CodeRay .shell .delimiter { color:#161 }
+.CodeRay .string { background-color:hsla(0,100%,50%,0.05); }
+.CodeRay .string .char { color: #b0b }
+.CodeRay .string .content { color: #D20 }
+.CodeRay .string .delimiter { color: #710 }
+.CodeRay .string .modifier { color: #E40 }
+.CodeRay .symbol { color:#A60 }
+.CodeRay .symbol .content { color:#A60 }
+.CodeRay .symbol .delimiter { color:#630 }
+.CodeRay .tag { color:#070 }
+.CodeRay .type { color:#339; font-weight:bold }
+.CodeRay .value { color: #088 }
+.CodeRay .variable { color:#037 }
+
+.CodeRay .insert { background: hsla(120,100%,50%,0.12) }
+.CodeRay .delete { background: hsla(0,100%,50%,0.12) }
+.CodeRay .change { color: #bbf; background: #007 }
+.CodeRay .head { color: #f8f; background: #505 }
+.CodeRay .head .filename { color: white; }
+
+.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; }
+.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; }
+
+.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold }
+.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold }
+.CodeRay .change .change { color: #88f }
+.CodeRay .head .head { color: #f4f }
diff --git a/test/samples/css.expected b/test/samples/css.expected
new file mode 100644
index 0000000..be73a7f
--- /dev/null
+++ b/test/samples/css.expected
@@ -0,0 +1,127 @@
+.CodeRay {
+  background-color: hsl(0,0%,95%);
+  border: 1px solid silver;
+  color: black;
+}
+.CodeRay pre {
+  margin: 0px;
+}
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line-numbers {
+  background-color: hsl(180,65%,90%);
+  color: gray;
+  text-align: right;
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+}
+.CodeRay .line-numbers a {
+  background-color: hsl(180,65%,90%) !important;
+  color: gray !important;
+  text-decoration: none !important;
+}
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
+.CodeRay .line-numbers a:target { color: blue !important; }
+.CodeRay .line-numbers .highlighted { color: red !important; }
+.CodeRay .line-numbers .highlighted a { color: red !important; }
+.CodeRay span.line-numbers { padding: 0px 4px; }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .code { width: 100%; }
+
+.CodeRay .debug { color: white !important; background: blue !important; }
+
+.CodeRay .annotation { color:#007 }
+.CodeRay .attribute-name { color:#b48 }
+.CodeRay .attribute-value { color:#700 }
+.CodeRay .binary { color:#549 }
+.CodeRay .binary .char { color:#325 }
+.CodeRay .binary .delimiter { color:#325 }
+.CodeRay .char { color:#D20 }
+.CodeRay .char .content { color:#D20 }
+.CodeRay .char .delimiter { color:#710 }
+.CodeRay .class { color:#B06; font-weight:bold }
+.CodeRay .class-variable { color:#369 }
+.CodeRay .color { color:#0A0 }
+.CodeRay .comment { color:#777 }
+.CodeRay .comment .char { color:#444 }
+.CodeRay .comment .delimiter { color:#444 }
+.CodeRay .constant { color:#036; font-weight:bold }
+.CodeRay .decorator { color:#B0B }
+.CodeRay .definition { color:#099; font-weight:bold }
+.CodeRay .delimiter { color:black }
+.CodeRay .directive { color:#088; font-weight:bold }
+.CodeRay .docstring { color:#D42; }
+.CodeRay .doctype { color:#34b }
+.CodeRay .done { text-decoration: line-through; color: gray }
+.CodeRay .entity { color:#800; font-weight:bold }
+.CodeRay .error { color:#F00; background-color:#FAA }
+.CodeRay .escape  { color:#666 }
+.CodeRay .exception { color:#C00; font-weight:bold }
+.CodeRay .float { color:#60E }
+.CodeRay .function { color:#06B; font-weight:bold }
+.CodeRay .function .delimiter { color:#024; font-weight:bold }
+.CodeRay .global-variable { color:#d70 }
+.CodeRay .hex { color:#02b }
+.CodeRay .id  { color:#33D; font-weight:bold }
+.CodeRay .include { color:#B44; font-weight:bold }
+.CodeRay .inline { background-color: hsla(0,0%,0%,0.07); color: black }
+.CodeRay .inline-delimiter { font-weight: bold; color: #666 }
+.CodeRay .instance-variable { color:#33B }
+.CodeRay .integer  { color:#00D }
+.CodeRay .imaginary { color:#f00 }
+.CodeRay .important { color:#D00 }
+.CodeRay .key { color: #606 }
+.CodeRay .key .char { color: #60f }
+.CodeRay .key .delimiter { color: #404 }
+.CodeRay .keyword { color:#080; font-weight:bold }
+.CodeRay .label { color:#970; font-weight:bold }
+.CodeRay .local-variable { color:#963 }
+.CodeRay .namespace { color:#707; font-weight:bold }
+.CodeRay .octal { color:#40E }
+.CodeRay .operator { }
+.CodeRay .predefined { color:#369; font-weight:bold }
+.CodeRay .predefined-constant { color:#069 }
+.CodeRay .predefined-type { color:#0a5; font-weight:bold }
+.CodeRay .preprocessor { color:#579 }
+.CodeRay .pseudo-class { color:#00C; font-weight:bold }
+.CodeRay .regexp { background-color:hsla(300,100%,50%,0.06); }
+.CodeRay .regexp .content { color:#808 }
+.CodeRay .regexp .delimiter { color:#404 }
+.CodeRay .regexp .modifier { color:#C2C }
+.CodeRay .reserved { color:#080; font-weight:bold }
+.CodeRay .shell { background-color:hsla(120,100%,50%,0.06); }
+.CodeRay .shell .content { color:#2B2 }
+.CodeRay .shell .delimiter { color:#161 }
+.CodeRay .string { background-color:hsla(0,100%,50%,0.05); }
+.CodeRay .string .char { color: #b0b }
+.CodeRay .string .content { color: #D20 }
+.CodeRay .string .delimiter { color: #710 }
+.CodeRay .string .modifier { color: #E40 }
+.CodeRay .symbol { color:#A60 }
+.CodeRay .symbol .content { color:#A60 }
+.CodeRay .symbol .delimiter { color:#630 }
+.CodeRay .tag { color:#070 }
+.CodeRay .type { color:#339; font-weight:bold }
+.CodeRay .value { color: #088 }
+.CodeRay .variable { color:#037 }
+
+.CodeRay .insert { background: hsla(120,100%,50%,0.12) }
+.CodeRay .delete { background: hsla(0,100%,50%,0.12) }
+.CodeRay .change { color: #bbf; background: #007 }
+.CodeRay .head { color: #f8f; background: #505 }
+.CodeRay .head .filename { color: white; }
+
+.CodeRay .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; }
+.CodeRay .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; }
+
+.CodeRay .insert .insert { color: #0c0; background:transparent; font-weight:bold }
+.CodeRay .delete .delete { color: #c00; background:transparent; font-weight:bold }
+.CodeRay .change .change { color: #88f }
+.CodeRay .head .head { color: #f4f }
diff --git a/test/samples/css.rb b/test/samples/css.rb
new file mode 100644
index 0000000..52e4bcc
--- /dev/null
+++ b/test/samples/css.rb
@@ -0,0 +1,4 @@
+require 'coderay'

+

+# print the default stylesheet for HTML codes

+puts CodeRay::Encoders[:html]::CSS.new.stylesheet

diff --git a/test/samples/div.actual b/test/samples/div.actual
new file mode 100644
index 0000000..d1e692a
--- /dev/null
+++ b/test/samples/div.actual
@@ -0,0 +1,17 @@
+<div class="CodeRay">
+  <div class="code"><pre><span style="color:#080;font-weight:bold">for</span> a <span style="color:#080;font-weight:bold">in</span> <span style="color:#00D">0</span>..<span style="color:#00D">255</span>
+        a = a.chr
+        <span style="color:#080;font-weight:bold">begin</span>
+                x = eval(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">?</span><span style="color:#b0b">\\</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>)
+                <span style="color:#080;font-weight:bold">if</span> x == a[<span style="color:#00D">0</span>]
+                        <span style="color:#080;font-weight:bold">next</span>
+                <span style="color:#080;font-weight:bold">else</span>
+                        print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: </span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>x<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>
+                <span style="color:#080;font-weight:bold">end</span>
+        <span style="color:#080;font-weight:bold">rescue</span> <span style="color:#036;font-weight:bold">SyntaxError</span> =&gt; boom
+                print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: error</span><span style="color:#710">&quot;</span></span>
+        <span style="color:#080;font-weight:bold">end</span>
+        puts
+<span style="color:#080;font-weight:bold">end</span>
+</pre></div>
+</div>
diff --git a/test/samples/div.expected b/test/samples/div.expected
new file mode 100644
index 0000000..d1e692a
--- /dev/null
+++ b/test/samples/div.expected
@@ -0,0 +1,17 @@
+<div class="CodeRay">
+  <div class="code"><pre><span style="color:#080;font-weight:bold">for</span> a <span style="color:#080;font-weight:bold">in</span> <span style="color:#00D">0</span>..<span style="color:#00D">255</span>
+        a = a.chr
+        <span style="color:#080;font-weight:bold">begin</span>
+                x = eval(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">?</span><span style="color:#b0b">\\</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>)
+                <span style="color:#080;font-weight:bold">if</span> x == a[<span style="color:#00D">0</span>]
+                        <span style="color:#080;font-weight:bold">next</span>
+                <span style="color:#080;font-weight:bold">else</span>
+                        print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: </span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>x<span style="font-weight:bold;color:#666">}</span></span><span style="color:#710">&quot;</span></span>
+                <span style="color:#080;font-weight:bold">end</span>
+        <span style="color:#080;font-weight:bold">rescue</span> <span style="color:#036;font-weight:bold">SyntaxError</span> =&gt; boom
+                print <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="background-color:hsla(0,0%,0%,0.07);color:black"><span style="font-weight:bold;color:#666">#{</span>a<span style="font-weight:bold;color:#666">}</span></span><span style="color:#D20">: error</span><span style="color:#710">&quot;</span></span>
+        <span style="color:#080;font-weight:bold">end</span>
+        puts
+<span style="color:#080;font-weight:bold">end</span>
+</pre></div>
+</div>
diff --git a/test/samples/div.rb b/test/samples/div.rb
new file mode 100644
index 0000000..27b6f32
--- /dev/null
+++ b/test/samples/div.rb
@@ -0,0 +1,19 @@
+require 'coderay'
+
+puts CodeRay.scan(DATA.read, :ruby).div
+
+__END__
+for a in 0..255
+	a = a.chr
+	begin
+		x = eval("?\\#{a}")
+		if x == a[0]
+			next
+		else
+			print "#{a}: #{x}"
+		end
+	rescue SyntaxError => boom
+		print "#{a}: error"
+	end
+	puts
+end
diff --git a/test/samples/encoder.actual b/test/samples/encoder.actual
new file mode 100644
index 0000000..8bd83a9
--- /dev/null
+++ b/test/samples/encoder.actual
@@ -0,0 +1,65 @@
+Encoders Demo: puts 17 + 4
+
+Statistic:
+
+Code Statistics
+
+Tokens                   8
+  Non-Whitespace         4
+Bytes Total             12
+
+Token Types (4):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                        8  100.00 %     1.5
+  space                        4   50.00 %     1.0
+  integer                      2   25.00 %     1.5
+  ident                        1   12.50 %     4.0
+  operator                     1   12.50 %     1.0
+
+
+Original text:
+[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}]
+
+YAML:
+--- 
+- - puts
+  - :ident
+- - " "
+  - :space
+- - "17"
+  - :integer
+- - " "
+  - :space
+- - +
+  - :operator
+- - " "
+  - :space
+- - "4"
+  - :integer
+- - |
+    
+
+  - :space
+
+Dump:
+"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257"
+compressed: 79 byte < 1200 byte
+
+Undump:
+
+Code Statistics
+
+Tokens                 800
+  Non-Whitespace       400
+Bytes Total           1200
+
+Token Types (4):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                      800  100.00 %     1.5
+  space                      400   50.00 %     1.0
+  integer                    200   25.00 %     1.5
+  ident                      100   12.50 %     4.0
+  operator                   100   12.50 %     1.0
+
diff --git a/test/samples/encoder.expected b/test/samples/encoder.expected
new file mode 100644
index 0000000..8bd83a9
--- /dev/null
+++ b/test/samples/encoder.expected
@@ -0,0 +1,65 @@
+Encoders Demo: puts 17 + 4
+
+Statistic:
+
+Code Statistics
+
+Tokens                   8
+  Non-Whitespace         4
+Bytes Total             12
+
+Token Types (4):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                        8  100.00 %     1.5
+  space                        4   50.00 %     1.0
+  integer                      2   25.00 %     1.5
+  ident                        1   12.50 %     4.0
+  operator                     1   12.50 %     1.0
+
+
+Original text:
+[{"type":"text","text":"puts","kind":"ident"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"17","kind":"integer"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"+","kind":"operator"},{"type":"text","text":" ","kind":"space"},{"type":"text","text":"4","kind":"integer"},{"type":"text","text":"\n","kind":"space"}]
+
+YAML:
+--- 
+- - puts
+  - :ident
+- - " "
+  - :space
+- - "17"
+  - :integer
+- - " "
+  - :space
+- - +
+  - :operator
+- - " "
+  - :space
+- - "4"
+  - :integer
+- - |
+    
+
+  - :space
+
+Dump:
+"x\234\355\3121\n\2000\f@\321\335StLp\022\204\236G0H\226\266\304\364\376\235\304K\374\365\361\374\266\2262f\276Z\274\245=\026rT-}X\\\331C\366\337O\335\234N\247\323\351t:\235N\247\323\351t:\235N\377\372\002\2613\031\257"
+compressed: 79 byte < 1200 byte
+
+Undump:
+
+Code Statistics
+
+Tokens                 800
+  Non-Whitespace       400
+Bytes Total           1200
+
+Token Types (4):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                      800  100.00 %     1.5
+  space                      400   50.00 %     1.0
+  integer                    200   25.00 %     1.5
+  ident                      100   12.50 %     4.0
+  operator                   100   12.50 %     1.0
+
diff --git a/test/samples/encoder.rb b/test/samples/encoder.rb
new file mode 100644
index 0000000..bfcfbfa
--- /dev/null
+++ b/test/samples/encoder.rb
@@ -0,0 +1,40 @@
+require 'coderay'

+

+SAMPLE = "puts 17 + 4\n"

+puts 'Encoders Demo: ' + SAMPLE

+scanner = CodeRay::Scanners[:ruby].new SAMPLE

+encoder = CodeRay::Encoders[:statistic].new

+

+tokens = scanner.tokenize

+stats = encoder.encode_tokens tokens

+

+puts

+puts 'Statistic:'

+puts stats

+

+# alternative 1

+tokens = CodeRay.scan SAMPLE, :ruby

+encoder = CodeRay.encoder(:json)

+textual = encoder.encode_tokens tokens

+puts

+puts 'Original text:'

+puts textual

+

+# alternative 2

+yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby

+puts

+puts 'YAML:'

+puts yaml

+

+# alternative 3

+require 'zlib'

+BIGSAMPLE = SAMPLE * 100

+dump = Zlib::Deflate.deflate(CodeRay.scan(BIGSAMPLE, :ruby).debug)

+puts

+puts 'Dump:'

+p dump

+puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size]

+

+puts

+puts 'Undump:'

+puts CodeRay.scan(Zlib::Inflate.inflate(dump), :debug).statistic

diff --git a/test/samples/global_vars.actual b/test/samples/global_vars.actual
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/samples/global_vars.actual
diff --git a/test/samples/global_vars.diff b/test/samples/global_vars.diff
new file mode 100644
index 0000000..2ed3382
--- /dev/null
+++ b/test/samples/global_vars.diff
@@ -0,0 +1,5 @@
+1,3d0
+< <--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL
+< ODER
+< TEXT = <--$BLA-->.TEST(...) 
+\ No newline at end of file
diff --git a/test/samples/global_vars.expected b/test/samples/global_vars.expected
new file mode 100644
index 0000000..0dc13c8
--- /dev/null
+++ b/test/samples/global_vars.expected
@@ -0,0 +1,3 @@
+<--$IE-->.TEXT_FIELD(:NAME, "PANFRAGE OHNE $GV UND MIT #{<--$GV-->}").SET ARTIKEL
+ODER
+TEXT = <--$BLA-->.TEST(...) 
\ No newline at end of file
diff --git a/test/samples/global_vars.rb b/test/samples/global_vars.rb
new file mode 100644
index 0000000..8066d67
--- /dev/null
+++ b/test/samples/global_vars.rb
@@ -0,0 +1,13 @@
+code = <<'CODE'

+$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel

+oder

+text = $bla.test(...) 

+CODE

+

+require 'coderay'

+

+tokens = CodeRay.scan code, :ruby

+tokens.each_text_token { |text, kind| text.upcase! }

+tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text }

+

+print tokens

diff --git a/test/samples/global_vars2.expected b/test/samples/global_vars2.expected
new file mode 100644
index 0000000..964cf50
--- /dev/null
+++ b/test/samples/global_vars2.expected
@@ -0,0 +1,10 @@
+<html>
+<head>
+<style>span.glob-var { color: green; font-weight: bold; }</style>
+</head>
+<body>
+<span class="glob-var">$ie</span>.text_field(:name, &quot;pAnfrage ohne $gV und mit #{<span class="glob-var">$gv</span>}&quot;).set artikel
+oder
+text = <span class="glob-var">$bla</span>.test(...) 
+</body>
+</html>
diff --git a/test/samples/global_vars2.rb b/test/samples/global_vars2.rb
new file mode 100644
index 0000000..7646890
--- /dev/null
+++ b/test/samples/global_vars2.rb
@@ -0,0 +1,28 @@
+require 'coderay'

+require 'erb'

+include ERB::Util

+

+code = <<'CODE'

+$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel

+oder

+text = $bla.test(...) 

+CODE

+puts <<HTML

+<html>

+<head>

+<style>span.glob-var { color: green; font-weight: bold; }</style>

+</head>

+<body>

+HTML

+

+CodeRay.scan_stream code, :ruby do |text, kind|

+	next if text.is_a? Symbol

+	text = h(text)

+	text = '<span class="glob-var">%s</span>' % text if kind == :global_variable

+	print text

+end

+

+puts <<HTML

+</body>

+</html>

+HTML

diff --git a/test/samples/highlight.expected b/test/samples/highlight.expected
new file mode 100644
index 0000000..6a9b278
--- /dev/null
+++ b/test/samples/highlight.expected
@@ -0,0 +1,175 @@
+<div class="CodeRay">
+  <div class="code"><pre>puts <span class="s"><span class="dl">&quot;</span><span class="k">Hello, World!</span><span class="dl">&quot;</span></span></pre></div>
+</div>
+<html>
+<head>
+<style type="text/css">
+.CodeRay {
+  background-color: #f8f8f8;
+  border: 1px solid silver;
+  font-family: 'Courier New', 'Terminal', monospace;
+  color: #000;
+}
+.CodeRay pre { margin: 0px }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
+table.CodeRay td { padding: 2px 4px; vertical-align: top }
+
+.CodeRay .line_numbers, .CodeRay .no {
+  background-color: #def;
+  color: gray;
+  text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .no { padding: 0px 4px }
+.CodeRay .code { width: 100% }
+
+ol.CodeRay { font-size: 10pt }
+ol.CodeRay li { white-space: pre }
+
+.CodeRay .code pre { overflow: auto }
+
+.CodeRay .debug { color:white ! important; background:blue ! important; }
+
+.CodeRay .af { color:#00C }
+.CodeRay .an { color:#007 }
+.CodeRay .at { color:#f08 }
+.CodeRay .av { color:#700 }
+.CodeRay .aw { color:#C00 }
+.CodeRay .bi { color:#509; font-weight:bold }
+.CodeRay .c  { color:#888; }
+
+.CodeRay .ch { color:#04D }
+.CodeRay .ch .k { color:#04D }
+.CodeRay .ch .dl { color:#039 }
+
+.CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
+.CodeRay .co { color:#036; font-weight:bold }
+.CodeRay .cr { color:#0A0 }
+.CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
+.CodeRay .df { color:#099; font-weight:bold }
+.CodeRay .di { color:#088; font-weight:bold }
+.CodeRay .dl { color:black }
+.CodeRay .do { color:#970 }
+.CodeRay .dt { color:#34b }
+.CodeRay .ds { color:#D42; font-weight:bold }
+.CodeRay .e  { color:#666; font-weight:bold }
+.CodeRay .en { color:#800; font-weight:bold }
+.CodeRay .er { color:#F00; background-color:#FAA }
+.CodeRay .ex { color:#C00; font-weight:bold }
+.CodeRay .fl { color:#60E; font-weight:bold }
+.CodeRay .fu { color:#06B; font-weight:bold }
+.CodeRay .gv { color:#d70; font-weight:bold }
+.CodeRay .hx { color:#058; font-weight:bold }
+.CodeRay .i  { color:#00D; font-weight:bold }
+.CodeRay .ic { color:#B44; font-weight:bold }
+
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
+
+.CodeRay .im { color:#f00; }
+.CodeRay .in { color:#B2B; font-weight:bold }
+.CodeRay .iv { color:#33B }
+.CodeRay .la { color:#970; font-weight:bold }
+.CodeRay .lv { color:#963 }
+.CodeRay .oc { color:#40E; font-weight:bold }
+.CodeRay .of { color:#000; font-weight:bold }
+.CodeRay .op { }
+.CodeRay .pc { color:#038; font-weight:bold }
+.CodeRay .pd { color:#369; font-weight:bold }
+.CodeRay .pp { color:#579; }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
+.CodeRay .r, .kw  { color:#080; font-weight:bold }
+
+.CodeRay .ke { color: #808; }
+.CodeRay .ke .dl { color: #606; }
+.CodeRay .ke .ch { color: #80f; }
+.CodeRay .vl { color: #088; }
+
+.CodeRay .rx { background-color:#fff0ff }
+.CodeRay .rx .k { color:#808 }
+.CodeRay .rx .dl { color:#404 }
+.CodeRay .rx .mod { color:#C2C }
+.CodeRay .rx .fu  { color:#404; font-weight: bold }
+
+.CodeRay .s { background-color:#fff0f0; color: #D20; }
+.CodeRay .s .s { background-color:#ffe0e0 }
+.CodeRay .s .s  .s { background-color:#ffd0d0 }
+.CodeRay .s .k { }
+.CodeRay .s .ch { color: #b0b; }
+.CodeRay .s .dl { color: #710; }
+
+.CodeRay .sh { background-color:#f0fff0; color:#2B2 }
+.CodeRay .sh .k { }
+.CodeRay .sh .dl { color:#161 }
+
+.CodeRay .sy { color:#A60 }
+.CodeRay .sy .k { color:#A60 }
+.CodeRay .sy .dl { color:#630 }
+
+.CodeRay .ta { color:#070 }
+.CodeRay .tf { color:#070; font-weight:bold }
+.CodeRay .ts { color:#D70; font-weight:bold }
+.CodeRay .ty { color:#339; font-weight:bold }
+.CodeRay .v  { color:#036 }
+.CodeRay .xt { color:#444 }
+
+.CodeRay .ins { background: #afa; }
+.CodeRay .del { background: #faa; }
+.CodeRay .chg { color: #aaf; background: #007; }
+.CodeRay .head { color: #f8f; background: #505 }
+
+.CodeRay .ins .ins { color: #080; font-weight:bold }
+.CodeRay .del .del { color: #800; font-weight:bold }
+.CodeRay .chg .chg { color: #66f; }
+.CodeRay .head .head { color: #f4f; }
+
+</style>
+
+<body>
+<table class="CodeRay"><tr>
+  <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre>1<tt>
+</tt>2<tt>
+</tt>3<tt>
+</tt>4<tt>
+</tt>5<tt>
+</tt>6<tt>
+</tt>7<tt>
+</tt>8<tt>
+</tt>9<tt>
+</tt><strong>10</strong><tt>
+</tt>11<tt>
+</tt>12<tt>
+</tt>13<tt>
+</tt>14<tt>
+</tt></pre></td>
+  <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }">require <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">coderay</span><span style="color:#710">'</span></span><tt>
+</tt><tt>
+</tt>puts <span style="color:#036;font-weight:bold">CodeRay</span>.highlight(<span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">puts &quot;Hello, World!&quot;</span><span style="color:#710">'</span></span>, <span style="color:#A60">:ruby</span>)<tt>
+</tt><tt>
+</tt>output = <span style="color:#036;font-weight:bold">CodeRay</span>.highlight_file(<span style="color:#d70;font-weight:bold">$0</span>, <span style="color:#A60">:line_numbers</span> =&gt; <span style="color:#A60">:table</span>)<tt>
+</tt>puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&lt;&lt;HTML</span></span><span style="background-color:#fff0f0;color:#D20"><span style=""><tt>
+</tt>&lt;html&gt;<tt>
+</tt>&lt;head&gt;<tt>
+</tt></span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>output.stylesheet <span style="color:#038;font-weight:bold">true</span><span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style=""><tt>
+</tt>&lt;body&gt;<tt>
+</tt></span><span style="background:#ddd;color:black"><span style="background:#ddd;font-weight:bold;color:#666">#{</span>output<span style="background:#ddd;font-weight:bold;color:#666">}</span></span><span style=""><tt>
+</tt>&lt;/body&gt;<tt>
+</tt>&lt;/html&gt;</span><span style="color:#710"><tt>
+</tt>HTML</span></span><tt>
+</tt></pre></td>
+</tr></table>
+
+</body>
+</html>
diff --git a/test/samples/highlight.rb b/test/samples/highlight.rb
new file mode 100644
index 0000000..846efa4
--- /dev/null
+++ b/test/samples/highlight.rb
@@ -0,0 +1,14 @@
+require 'coderay'

+

+puts CodeRay.highlight('puts "Hello, World!"', :ruby)

+

+output = CodeRay.highlight_file($0, :line_numbers => :table)

+puts <<HTML

+<html>

+<head>

+#{output.stylesheet true}

+<body>

+#{output}

+</body>

+</html>

+HTML

diff --git a/test/samples/html.expected b/test/samples/html.expected
new file mode 100644
index 0000000..e98d589
--- /dev/null
+++ b/test/samples/html.expected
@@ -0,0 +1,919 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+  <title>CodeRay HTML Encoder Example</title>
+  <style type="text/css">
+.CodeRay {
+  background-color: #f8f8f8;
+  border: 1px solid silver;
+  font-family: 'Courier New', 'Terminal', monospace;
+  color: #000;
+}
+.CodeRay pre { margin: 0px }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
+table.CodeRay td { padding: 2px 4px; vertical-align: top }
+
+.CodeRay .line_numbers, .CodeRay .no {
+  background-color: #def;
+  color: gray;
+  text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .no { padding: 0px 4px }
+.CodeRay .code { width: 100% }
+
+ol.CodeRay { font-size: 10pt }
+ol.CodeRay li { white-space: pre }
+
+.CodeRay .code pre { overflow: auto }
+
+.CodeRay .debug { color:white ! important; background:blue ! important; }
+
+.CodeRay .af { color:#00C }
+.CodeRay .an { color:#007 }
+.CodeRay .at { color:#f08 }
+.CodeRay .av { color:#700 }
+.CodeRay .aw { color:#C00 }
+.CodeRay .bi { color:#509; font-weight:bold }
+.CodeRay .c  { color:#888; }
+
+.CodeRay .ch { color:#04D }
+.CodeRay .ch .k { color:#04D }
+.CodeRay .ch .dl { color:#039 }
+
+.CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
+.CodeRay .co { color:#036; font-weight:bold }
+.CodeRay .cr { color:#0A0 }
+.CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
+.CodeRay .df { color:#099; font-weight:bold }
+.CodeRay .di { color:#088; font-weight:bold }
+.CodeRay .dl { color:black }
+.CodeRay .do { color:#970 }
+.CodeRay .dt { color:#34b }
+.CodeRay .ds { color:#D42; font-weight:bold }
+.CodeRay .e  { color:#666; font-weight:bold }
+.CodeRay .en { color:#800; font-weight:bold }
+.CodeRay .er { color:#F00; background-color:#FAA }
+.CodeRay .ex { color:#C00; font-weight:bold }
+.CodeRay .fl { color:#60E; font-weight:bold }
+.CodeRay .fu { color:#06B; font-weight:bold }
+.CodeRay .gv { color:#d70; font-weight:bold }
+.CodeRay .hx { color:#058; font-weight:bold }
+.CodeRay .i  { color:#00D; font-weight:bold }
+.CodeRay .ic { color:#B44; font-weight:bold }
+
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
+
+.CodeRay .im { color:#f00; }
+.CodeRay .in { color:#B2B; font-weight:bold }
+.CodeRay .iv { color:#33B }
+.CodeRay .la { color:#970; font-weight:bold }
+.CodeRay .lv { color:#963 }
+.CodeRay .oc { color:#40E; font-weight:bold }
+.CodeRay .of { color:#000; font-weight:bold }
+.CodeRay .op { }
+.CodeRay .pc { color:#038; font-weight:bold }
+.CodeRay .pd { color:#369; font-weight:bold }
+.CodeRay .pp { color:#579; }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
+.CodeRay .r, .kw  { color:#080; font-weight:bold }
+
+.CodeRay .ke { color: #808; }
+.CodeRay .ke .dl { color: #606; }
+.CodeRay .ke .ch { color: #80f; }
+.CodeRay .vl { color: #088; }
+
+.CodeRay .rx { background-color:#fff0ff }
+.CodeRay .rx .k { color:#808 }
+.CodeRay .rx .dl { color:#404 }
+.CodeRay .rx .mod { color:#C2C }
+.CodeRay .rx .fu  { color:#404; font-weight: bold }
+
+.CodeRay .s { background-color:#fff0f0; color: #D20; }
+.CodeRay .s .s { background-color:#ffe0e0 }
+.CodeRay .s .s  .s { background-color:#ffd0d0 }
+.CodeRay .s .k { }
+.CodeRay .s .ch { color: #b0b; }
+.CodeRay .s .dl { color: #710; }
+
+.CodeRay .sh { background-color:#f0fff0; color:#2B2 }
+.CodeRay .sh .k { }
+.CodeRay .sh .dl { color:#161 }
+
+.CodeRay .sy { color:#A60 }
+.CodeRay .sy .k { color:#A60 }
+.CodeRay .sy .dl { color:#630 }
+
+.CodeRay .ta { color:#070 }
+.CodeRay .tf { color:#070; font-weight:bold }
+.CodeRay .ts { color:#D70; font-weight:bold }
+.CodeRay .ty { color:#339; font-weight:bold }
+.CodeRay .v  { color:#036 }
+.CodeRay .xt { color:#444 }
+
+.CodeRay .ins { background: #afa; }
+.CodeRay .del { background: #faa; }
+.CodeRay .chg { color: #aaf; background: #007; }
+.CodeRay .head { color: #f8f; background: #505 }
+
+.CodeRay .ins .ins { color: #080; font-weight:bold }
+.CodeRay .del .del { color: #800; font-weight:bold }
+.CodeRay .chg .chg { color: #66f; }
+.CodeRay .head .head { color: #f4f; }
+
+  </style>
+</head>
+<body style="background-color: white;">
+
+<table class="CodeRay"><tr>
+  <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre>1<tt>
+</tt>2<tt>
+</tt>3<tt>
+</tt>4<tt>
+</tt>5<tt>
+</tt>6<tt>
+</tt>7<tt>
+</tt>8<tt>
+</tt>9<tt>
+</tt><strong>10</strong><tt>
+</tt>11<tt>
+</tt>12<tt>
+</tt>13<tt>
+</tt>14<tt>
+</tt>15<tt>
+</tt>16<tt>
+</tt>17<tt>
+</tt>18<tt>
+</tt>19<tt>
+</tt><strong>20</strong><tt>
+</tt>21<tt>
+</tt>22<tt>
+</tt>23<tt>
+</tt>24<tt>
+</tt>25<tt>
+</tt>26<tt>
+</tt>27<tt>
+</tt>28<tt>
+</tt>29<tt>
+</tt><strong>30</strong><tt>
+</tt>31<tt>
+</tt>32<tt>
+</tt>33<tt>
+</tt>34<tt>
+</tt>35<tt>
+</tt>36<tt>
+</tt>37<tt>
+</tt>38<tt>
+</tt>39<tt>
+</tt><strong>40</strong><tt>
+</tt>41<tt>
+</tt>42<tt>
+</tt>43<tt>
+</tt>44<tt>
+</tt>45<tt>
+</tt>46<tt>
+</tt>47<tt>
+</tt>48<tt>
+</tt>49<tt>
+</tt><strong>50</strong><tt>
+</tt>51<tt>
+</tt>52<tt>
+</tt>53<tt>
+</tt>54<tt>
+</tt>55<tt>
+</tt>56<tt>
+</tt>57<tt>
+</tt>58<tt>
+</tt>59<tt>
+</tt><strong>60</strong><tt>
+</tt>61<tt>
+</tt>62<tt>
+</tt>63<tt>
+</tt>64<tt>
+</tt>65<tt>
+</tt>66<tt>
+</tt>67<tt>
+</tt>68<tt>
+</tt>69<tt>
+</tt><strong>70</strong><tt>
+</tt>71<tt>
+</tt>72<tt>
+</tt>73<tt>
+</tt>74<tt>
+</tt>75<tt>
+</tt>76<tt>
+</tt>77<tt>
+</tt>78<tt>
+</tt>79<tt>
+</tt><strong>80</strong><tt>
+</tt>81<tt>
+</tt>82<tt>
+</tt>83<tt>
+</tt>84<tt>
+</tt>85<tt>
+</tt>86<tt>
+</tt>87<tt>
+</tt>88<tt>
+</tt>89<tt>
+</tt><strong>90</strong><tt>
+</tt>91<tt>
+</tt>92<tt>
+</tt>93<tt>
+</tt>94<tt>
+</tt>95<tt>
+</tt>96<tt>
+</tt>97<tt>
+</tt>98<tt>
+</tt>99<tt>
+</tt><strong>100</strong><tt>
+</tt>101<tt>
+</tt>102<tt>
+</tt>103<tt>
+</tt>104<tt>
+</tt>105<tt>
+</tt>106<tt>
+</tt>107<tt>
+</tt>108<tt>
+</tt>109<tt>
+</tt><strong>110</strong><tt>
+</tt>111<tt>
+</tt>112<tt>
+</tt>113<tt>
+</tt>114<tt>
+</tt>115<tt>
+</tt>116<tt>
+</tt>117<tt>
+</tt>118<tt>
+</tt>119<tt>
+</tt><strong>120</strong><tt>
+</tt>121<tt>
+</tt>122<tt>
+</tt>123<tt>
+</tt>124<tt>
+</tt>125<tt>
+</tt>126<tt>
+</tt>127<tt>
+</tt>128<tt>
+</tt>129<tt>
+</tt><strong>130</strong><tt>
+</tt>131<tt>
+</tt>132<tt>
+</tt>133<tt>
+</tt>134<tt>
+</tt>135<tt>
+</tt>136<tt>
+</tt>137<tt>
+</tt>138<tt>
+</tt>139<tt>
+</tt><strong>140</strong><tt>
+</tt>141<tt>
+</tt>142<tt>
+</tt>143<tt>
+</tt>144<tt>
+</tt>145<tt>
+</tt>146<tt>
+</tt>147<tt>
+</tt>148<tt>
+</tt>149<tt>
+</tt><strong>150</strong><tt>
+</tt>151<tt>
+</tt>152<tt>
+</tt>153<tt>
+</tt>154<tt>
+</tt>155<tt>
+</tt>156<tt>
+</tt>157<tt>
+</tt>158<tt>
+</tt>159<tt>
+</tt><strong>160</strong><tt>
+</tt>161<tt>
+</tt>162<tt>
+</tt>163<tt>
+</tt>164<tt>
+</tt>165<tt>
+</tt>166<tt>
+</tt>167<tt>
+</tt>168<tt>
+</tt>169<tt>
+</tt><strong>170</strong><tt>
+</tt>171<tt>
+</tt>172<tt>
+</tt>173<tt>
+</tt>174<tt>
+</tt>175<tt>
+</tt>176<tt>
+</tt>177<tt>
+</tt>178<tt>
+</tt>179<tt>
+</tt><strong>180</strong><tt>
+</tt>181<tt>
+</tt>182<tt>
+</tt>183<tt>
+</tt>184<tt>
+</tt>185<tt>
+</tt>186<tt>
+</tt>187<tt>
+</tt>188<tt>
+</tt>189<tt>
+</tt><strong>190</strong><tt>
+</tt>191<tt>
+</tt>192<tt>
+</tt>193<tt>
+</tt>194<tt>
+</tt>195<tt>
+</tt>196<tt>
+</tt>197<tt>
+</tt>198<tt>
+</tt>199<tt>
+</tt><strong>200</strong><tt>
+</tt>201<tt>
+</tt>202<tt>
+</tt>203<tt>
+</tt>204<tt>
+</tt>205<tt>
+</tt>206<tt>
+</tt>207<tt>
+</tt>208<tt>
+</tt>209<tt>
+</tt><strong>210</strong><tt>
+</tt>211<tt>
+</tt>212<tt>
+</tt>213<tt>
+</tt>214<tt>
+</tt>215<tt>
+</tt>216<tt>
+</tt>217<tt>
+</tt>218<tt>
+</tt>219<tt>
+</tt><strong>220</strong><tt>
+</tt>221<tt>
+</tt>222<tt>
+</tt>223<tt>
+</tt>224<tt>
+</tt>225<tt>
+</tt>226<tt>
+</tt>227<tt>
+</tt>228<tt>
+</tt>229<tt>
+</tt><strong>230</strong><tt>
+</tt>231<tt>
+</tt>232<tt>
+</tt>233<tt>
+</tt>234<tt>
+</tt>235<tt>
+</tt>236<tt>
+</tt>237<tt>
+</tt>238<tt>
+</tt>239<tt>
+</tt><strong>240</strong><tt>
+</tt>241<tt>
+</tt>242<tt>
+</tt>243<tt>
+</tt>244<tt>
+</tt>245<tt>
+</tt>246<tt>
+</tt>247<tt>
+</tt>248<tt>
+</tt>249<tt>
+</tt><strong>250</strong><tt>
+</tt>251<tt>
+</tt>252<tt>
+</tt>253<tt>
+</tt>254<tt>
+</tt>255<tt>
+</tt>256<tt>
+</tt>257<tt>
+</tt>258<tt>
+</tt>259<tt>
+</tt><strong>260</strong><tt>
+</tt>261<tt>
+</tt>262<tt>
+</tt>263<tt>
+</tt>264<tt>
+</tt>265<tt>
+</tt>266<tt>
+</tt>267<tt>
+</tt>268<tt>
+</tt>269<tt>
+</tt><strong>270</strong><tt>
+</tt>271<tt>
+</tt>272<tt>
+</tt>273<tt>
+</tt>274<tt>
+</tt>275<tt>
+</tt>276<tt>
+</tt>277<tt>
+</tt>278<tt>
+</tt>279<tt>
+</tt><strong>280</strong><tt>
+</tt>281<tt>
+</tt>282<tt>
+</tt>283<tt>
+</tt>284<tt>
+</tt>285<tt>
+</tt>286<tt>
+</tt>287<tt>
+</tt>288<tt>
+</tt>289<tt>
+</tt><strong>290</strong><tt>
+</tt>291<tt>
+</tt>292<tt>
+</tt>293<tt>
+</tt>294<tt>
+</tt>295<tt>
+</tt>296<tt>
+</tt>297<tt>
+</tt>298<tt>
+</tt>299<tt>
+</tt><strong>300</strong><tt>
+</tt>301<tt>
+</tt>302<tt>
+</tt>303<tt>
+</tt>304<tt>
+</tt>305<tt>
+</tt>306<tt>
+</tt>307<tt>
+</tt>308<tt>
+</tt>309<tt>
+</tt><strong>310</strong><tt>
+</tt>311<tt>
+</tt>312<tt>
+</tt>313<tt>
+</tt>314<tt>
+</tt>315<tt>
+</tt>316<tt>
+</tt>317<tt>
+</tt>318<tt>
+</tt>319<tt>
+</tt><strong>320</strong><tt>
+</tt>321<tt>
+</tt>322<tt>
+</tt>323<tt>
+</tt>324<tt>
+</tt>325<tt>
+</tt>326<tt>
+</tt>327<tt>
+</tt>328<tt>
+</tt>329<tt>
+</tt><strong>330</strong><tt>
+</tt>331<tt>
+</tt>332<tt>
+</tt>333<tt>
+</tt>334<tt>
+</tt>335<tt>
+</tt>336<tt>
+</tt>337<tt>
+</tt>338<tt>
+</tt>339<tt>
+</tt><strong>340</strong><tt>
+</tt>341<tt>
+</tt>342<tt>
+</tt>343<tt>
+</tt>344<tt>
+</tt>345<tt>
+</tt>346<tt>
+</tt>347<tt>
+</tt>348<tt>
+</tt>349<tt>
+</tt><strong>350</strong><tt>
+</tt>351<tt>
+</tt>352<tt>
+</tt>353<tt>
+</tt>354<tt>
+</tt>355<tt>
+</tt>356<tt>
+</tt>357<tt>
+</tt>358<tt>
+</tt>359<tt>
+</tt><strong>360</strong><tt>
+</tt>361<tt>
+</tt>362<tt>
+</tt>363<tt>
+</tt>364<tt>
+</tt>365<tt>
+</tt>366<tt>
+</tt>367<tt>
+</tt>368<tt>
+</tt>369<tt>
+</tt><strong>370</strong><tt>
+</tt>371<tt>
+</tt>372<tt>
+</tt>373<tt>
+</tt>374<tt>
+</tt>375<tt>
+</tt>376<tt>
+</tt>377<tt>
+</tt>378<tt>
+</tt>379<tt>
+</tt><strong>380</strong><tt>
+</tt>381<tt>
+</tt>382<tt>
+</tt>383<tt>
+</tt>384<tt>
+</tt>385<tt>
+</tt></pre></td>
+  <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }">require <span class="s"><span class="dl">'</span><span class="k">scanner</span><span class="dl">'</span></span><tt>
+</tt><tt>
+</tt><span class="r">module</span> <span class="cl">CodeRay</span><tt>
+</tt>  <tt>
+</tt>  <span class="r">class</span> <span class="cl">RubyScanner</span> &lt; <span class="co">Scanner</span><tt>
+</tt>    <tt>
+</tt>    <span class="co">RESERVED_WORDS</span> = [<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">and</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">def</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">end</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">in</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">or</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">unless</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">begin</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">defined?</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">ensure</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">module</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">redo</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">super</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">until</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">BEGIN</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">break</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">do</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">next</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">rescue</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">then</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">when</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">END</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">case</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">else</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">for</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">retry</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">while</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">alias</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">class</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">elsif</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">if</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">not</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">return</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">undef</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">yield</span><span class="dl">'</span></span>,<tt>
+</tt>    ]<tt>
+</tt><tt>
+</tt>    <span class="co">DEF_KEYWORDS</span> = [<span class="s"><span class="dl">'</span><span class="k">def</span><span class="dl">'</span></span>]<tt>
+</tt>    <span class="co">MODULE_KEYWORDS</span> = [<span class="s"><span class="dl">'</span><span class="k">class</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">module</span><span class="dl">'</span></span>]<tt>
+</tt>    <span class="co">DEF_NEW_STATE</span> = <span class="co">WordList</span>.new(<span class="sy">:initial</span>).<tt>
+</tt>      add(<span class="co">DEF_KEYWORDS</span>, <span class="sy">:def_expected</span>).<tt>
+</tt>      add(<span class="co">MODULE_KEYWORDS</span>, <span class="sy">:module_expected</span>)<tt>
+</tt><tt>
+</tt>    <span class="co">WORDS_ALLOWING_REGEXP</span> = [<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">and</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">or</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">not</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">while</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">until</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">unless</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">if</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">elsif</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">when</span><span class="dl">'</span></span><tt>
+</tt>    ]<tt>
+</tt>    <span class="co">REGEXP_ALLOWED</span> = <span class="co">WordList</span>.new(<span class="pc">false</span>).<tt>
+</tt>      add(<span class="co">WORDS_ALLOWING_REGEXP</span>, <span class="sy">:set</span>)<tt>
+</tt>    <tt>
+</tt>    <span class="co">PREDEFINED_CONSTANTS</span> = [<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">nil</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">true</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">false</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">self</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="s"><span class="dl">'</span><span class="k">DATA</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">ARGV</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">ARGF</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">__FILE__</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">__LINE__</span><span class="dl">'</span></span>,<tt>
+</tt>    ]<tt>
+</tt><tt>
+</tt>    <span class="co">IDENT_KIND</span> = <span class="co">WordList</span>.new(<span class="sy">:ident</span>).<tt>
+</tt>      add(<span class="co">RESERVED_WORDS</span>, <span class="sy">:reserved</span>).<tt>
+</tt>      add(<span class="co">PREDEFINED_CONSTANTS</span>, <span class="sy">:pre_constant</span>)<tt>
+</tt><tt>
+</tt>    <span class="co">METHOD_NAME</span> = <span class="rx"><span class="dl">/</span><span class="k"> </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k"> [?!]? </span><span class="dl">/</span><span class="mod">xo</span></span><tt>
+</tt>    <span class="co">METHOD_NAME_EX</span> = <span class="rx"><span class="dl">/</span><span class="k"><tt>
+</tt>     </span><span class="il"><span class="idl">#{</span><span class="co">METHOD_NAME</span><span class="idl">}</span></span><span class="k">  # common methods: split, foo=, empty?, gsub!<tt>
+</tt>     | </span><span class="ch">\*</span><span class="ch">\*</span><span class="k">?         # multiplication and power<tt>
+</tt>     | [-+~]@?       # plus, minus<tt>
+</tt>     | [</span><span class="ch">\/</span><span class="k">%&amp;|^`]     # division, modulo or format strings, &amp;and, |or, ^xor, `system`<tt>
+</tt>     | </span><span class="ch">\[</span><span class="ch">\]</span><span class="k">=?        # array getter and setter<tt>
+</tt>     | &lt;=?&gt;? | &gt;=?   # comparison, rocket operator<tt>
+</tt>     | &lt;&lt; | &gt;&gt;       # append or shift left, shift right<tt>
+</tt>     | ===?          # simple equality and case equality<tt>
+</tt>    </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <span class="co">GLOBAL_VARIABLE</span> = <span class="rx"><span class="dl">/</span><span class="k"> </span><span class="ch">\$</span><span class="k"> (?: </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k"> | </span><span class="ch">\d</span><span class="k">+ | [~&amp;+`'=</span><span class="ch">\/</span><span class="k">,;_.&lt;&gt;!@0$?*&quot;:F</span><span class="ch">\\</span><span class="k">] | -[a-zA-Z_0-9] ) </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt><tt>
+</tt>    <span class="co">DOUBLEQ</span> = <span class="rx"><span class="dl">/</span><span class="k"> &quot;  [^&quot;</span><span class="ch">\#</span><span class="ch">\\</span><span class="k">]*  (?: (?: </span><span class="ch">\#</span><span class="ch">\{</span><span class="k">.*?</span><span class="ch">\}</span><span class="k"> | </span><span class="ch">\#</span><span class="k">(?:$&quot;)?  | </span><span class="ch">\\</span><span class="k">. ) [^&quot;</span><span class="ch">\#</span><span class="ch">\\</span><span class="k">]*  )* &quot;?  </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <span class="co">SINGLEQ</span> = <span class="rx"><span class="dl">/</span><span class="k"> '  [^'</span><span class="ch">\\</span><span class="k">]*    (?:                              </span><span class="ch">\\</span><span class="k">.   [^'</span><span class="ch">\\</span><span class="k">]*    )* '?  </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <span class="co">STRING</span>  = <span class="rx"><span class="dl">/</span><span class="k"> </span><span class="il"><span class="idl">#{</span><span class="co">SINGLEQ</span><span class="idl">}</span></span><span class="k"> | </span><span class="il"><span class="idl">#{</span><span class="co">DOUBLEQ</span><span class="idl">}</span></span><span class="k"> </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <span class="co">SHELL</span>   = <span class="rx"><span class="dl">/</span><span class="k"> `  [^`</span><span class="ch">\#</span><span class="ch">\\</span><span class="k">]*  (?: (?: </span><span class="ch">\#</span><span class="ch">\{</span><span class="k">.*?</span><span class="ch">\}</span><span class="k"> | </span><span class="ch">\#</span><span class="k">(?:$`)?  | </span><span class="ch">\\</span><span class="k">. ) [^`</span><span class="ch">\#</span><span class="ch">\\</span><span class="k">]*  )* `?  </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <span class="co">REGEXP</span>  = <span class="rx"><span class="dl">/</span><span class="k"> </span><span class="ch">\/</span><span class="k"> [^</span><span class="ch">\/</span><span class="ch">\#</span><span class="ch">\\</span><span class="k">]* (?: (?: </span><span class="ch">\#</span><span class="ch">\{</span><span class="k">.*?</span><span class="ch">\}</span><span class="k"> | </span><span class="ch">\#</span><span class="k">(?:$</span><span class="ch">\/</span><span class="k">)? | </span><span class="ch">\\</span><span class="k">. ) [^</span><span class="ch">\/</span><span class="ch">\#</span><span class="ch">\\</span><span class="k">]* )* </span><span class="ch">\/</span><span class="k">? </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <tt>
+</tt>    <span class="co">DECIMAL</span> = <span class="rx"><span class="dl">/</span><span class="ch">\d</span><span class="k">+(?:_</span><span class="ch">\d</span><span class="k">+)*</span><span class="dl">/</span></span>  <span class="c"># doesn't recognize 09 as octal error</span><tt>
+</tt>    <span class="co">OCTAL</span> = <span class="rx"><span class="dl">/</span><span class="k">0_?[0-7]+(?:_[0-7]+)*</span><span class="dl">/</span></span><tt>
+</tt>    <span class="co">HEXADECIMAL</span> = <span class="rx"><span class="dl">/</span><span class="k">0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*</span><span class="dl">/</span></span><tt>
+</tt>    <span class="co">BINARY</span> = <span class="rx"><span class="dl">/</span><span class="k">0b[01]+(?:_[01]+)*</span><span class="dl">/</span></span><tt>
+</tt><tt>
+</tt>    <span class="co">EXPONENT</span> = <span class="rx"><span class="dl">/</span><span class="k"> [eE] [+-]? </span><span class="il"><span class="idl">#{</span><span class="co">DECIMAL</span><span class="idl">}</span></span><span class="k"> </span><span class="dl">/</span><span class="mod">ox</span></span><tt>
+</tt>    <span class="co">FLOAT</span> = <span class="rx"><span class="dl">/</span><span class="k"> </span><span class="il"><span class="idl">#{</span><span class="co">DECIMAL</span><span class="idl">}</span></span><span class="k"> (?: </span><span class="il"><span class="idl">#{</span><span class="co">EXPONENT</span><span class="idl">}</span></span><span class="k"> | </span><span class="ch">\.</span><span class="k"> </span><span class="il"><span class="idl">#{</span><span class="co">DECIMAL</span><span class="idl">}</span></span><span class="k"> </span><span class="il"><span class="idl">#{</span><span class="co">EXPONENT</span><span class="idl">}</span></span><span class="k">? ) </span><span class="dl">/</span></span><tt>
+</tt>    <span class="co">INTEGER</span> = <span class="rx"><span class="dl">/</span><span class="il"><span class="idl">#{</span><span class="co">OCTAL</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">HEXADECIMAL</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">BINARY</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">DECIMAL</span><span class="idl">}</span></span><span class="dl">/</span></span><tt>
+</tt>    <tt>
+</tt>    <span class="r">def</span> <span class="fu">reset</span><tt>
+</tt>      <span class="r">super</span><tt>
+</tt>      <span class="iv">@regexp_allowed</span> = <span class="pc">false</span><tt>
+</tt>    <span class="r">end</span><tt>
+</tt>    <tt>
+</tt>    <span class="r">def</span> <span class="fu">next_token</span><tt>
+</tt>      <span class="r">return</span> <span class="r">if</span> <span class="iv">@scanner</span>.eos?<tt>
+</tt><tt>
+</tt>      kind = <span class="sy">:error</span><tt>
+</tt>      <span class="r">if</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="ch">\s</span><span class="k">+</span><span class="dl">/</span></span>)  <span class="c"># in every state</span><tt>
+</tt>        kind = <span class="sy">:space</span><tt>
+</tt>        <span class="iv">@regexp_allowed</span> = <span class="sy">:set</span> <span class="r">if</span> <span class="iv">@regexp_allowed</span> <span class="r">or</span> <span class="iv">@scanner</span>.matched.index(<span class="i">?\n</span>)  <span class="c"># delayed flag setting</span><tt>
+</tt><tt>
+</tt>      <span class="r">elsif</span> <span class="iv">@state</span> == <span class="sy">:def_expected</span><tt>
+</tt>        <span class="r">if</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"> (?: (?:</span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k">(?:</span><span class="ch">\.</span><span class="k">|::))* | (?:@@?|$)? </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k">(?:</span><span class="ch">\.</span><span class="k">|::) ) </span><span class="il"><span class="idl">#{</span><span class="co">METHOD_NAME_EX</span><span class="idl">}</span></span><span class="k"> </span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>          kind = <span class="sy">:method</span><tt>
+</tt>          <span class="iv">@state</span> = <span class="sy">:initial</span><tt>
+</tt>        <span class="r">else</span><tt>
+</tt>          <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">.</span><span class="dl">/</span></span>)<tt>
+</tt>          kind = <span class="sy">:error</span><tt>
+</tt>        <span class="r">end</span><tt>
+</tt>        <span class="iv">@state</span> = <span class="sy">:initial</span><tt>
+</tt>        <tt>
+</tt>      <span class="r">elsif</span> <span class="iv">@state</span> == <span class="sy">:module_expected</span><tt>
+</tt>        <span class="r">if</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">&lt;&lt;</span><span class="dl">/</span></span>)<tt>
+</tt>          kind = <span class="sy">:operator</span><tt>
+</tt>        <span class="r">else</span><tt>
+</tt>          <span class="r">if</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"> (?: </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k"> (?:</span><span class="ch">\.</span><span class="k">|::))* </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k"> </span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>            kind = <span class="sy">:method</span><tt>
+</tt>          <span class="r">else</span><tt>
+</tt>            <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">.</span><span class="dl">/</span></span>)<tt>
+</tt>            kind = <span class="sy">:error</span><tt>
+</tt>          <span class="r">end</span><tt>
+</tt>          <span class="iv">@state</span> = <span class="sy">:initial</span><tt>
+</tt>        <span class="r">end</span><tt>
+</tt>        <tt>
+</tt>      <span class="r">elsif</span> <span class="c"># state == :initial</span><tt>
+</tt>        <span class="c"># IDENTIFIERS, KEYWORDS</span><tt>
+</tt>        <span class="r">if</span> <span class="iv">@scanner</span>.scan(<span class="co">GLOBAL_VARIABLE</span>)<tt>
+</tt>          kind = <span class="sy">:global_variable</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"> @@ </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k"> </span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>          kind = <span class="sy">:class_variable</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"> @ </span><span class="il"><span class="idl">#{</span><span class="co">IDENT</span><span class="idl">}</span></span><span class="k"> </span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>          kind = <span class="sy">:instance_variable</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"> __END__</span><span class="ch">\n</span><span class="k"> ( (?!</span><span class="ch">\#</span><span class="k">CODE</span><span class="ch">\#</span><span class="k">) .* )? | </span><span class="ch">\#</span><span class="k">[^</span><span class="ch">\n</span><span class="k">]* | =begin(?=</span><span class="ch">\s</span><span class="k">).*? </span><span class="ch">\n</span><span class="k">=end(?=</span><span class="ch">\s</span><span class="k">|</span><span class="ch">\z</span><span class="k">)(?:[^</span><span class="ch">\n</span><span class="k">]*)? </span><span class="dl">/</span><span class="mod">x</span></span>)<tt>
+</tt>          kind = <span class="sy">:comment</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="co">METHOD_NAME</span>)<tt>
+</tt>          <span class="r">if</span> <span class="iv">@last_token_dot</span><tt>
+</tt>            kind = <span class="sy">:ident</span><tt>
+</tt>          <span class="r">else</span><tt>
+</tt>            matched = <span class="iv">@scanner</span>.matched<tt>
+</tt>            kind = <span class="co">IDENT_KIND</span>[matched]<tt>
+</tt>            <span class="r">if</span> kind == <span class="sy">:ident</span> <span class="r">and</span> matched =~ <span class="rx"><span class="dl">/</span><span class="k">^[A-Z]</span><span class="dl">/</span></span><tt>
+</tt>              kind = <span class="sy">:constant</span><tt>
+</tt>            <span class="r">elsif</span> kind == <span class="sy">:reserved</span><tt>
+</tt>              <span class="iv">@state</span> = <span class="co">DEF_NEW_STATE</span>[matched]<tt>
+</tt>              <span class="iv">@regexp_allowed</span> = <span class="co">REGEXP_ALLOWED</span>[matched]<tt>
+</tt>            <span class="r">end</span><tt>
+</tt>          <span class="r">end</span><tt>
+</tt>          <tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="co">STRING</span>)<tt>
+</tt>          kind = <span class="sy">:string</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="co">SHELL</span>)<tt>
+</tt>          kind = <span class="sy">:shell</span><tt>
+</tt>        <span class="c">## HEREDOCS</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="ch">\/</span><span class="dl">/</span></span>) <span class="r">and</span> <span class="iv">@regexp_allowed</span><tt>
+</tt>           <span class="iv">@scanner</span>.unscan<tt>
+</tt>           <span class="iv">@scanner</span>.scan(<span class="co">REGEXP</span>)<tt>
+</tt>          kind = <span class="sy">:regexp</span><tt>
+</tt>        <span class="c">## %strings</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">:(?:</span><span class="il"><span class="idl">#{</span><span class="co">GLOBAL_VARIABLE</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">METHOD_NAME_EX</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">STRING</span><span class="idl">}</span></span><span class="k">)</span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>          kind = <span class="sy">:global_variable</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"><tt>
+</tt>          </span><span class="ch">\?</span><span class="k"> (?:<tt>
+</tt>            [^</span><span class="ch">\s</span><span class="ch">\\</span><span class="k">]<tt>
+</tt>          | <tt>
+</tt>            </span><span class="ch">\\</span><span class="k"> (?:M-</span><span class="ch">\\</span><span class="k">C-|C-</span><span class="ch">\\</span><span class="k">M-|M-</span><span class="ch">\\</span><span class="k">c|c</span><span class="ch">\\</span><span class="k">M-|c|C-|M-))? (?: </span><span class="ch">\\</span><span class="k"> (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )<tt>
+</tt>          )<tt>
+</tt>        </span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>          kind = <span class="sy">:integer</span><tt>
+</tt>          <tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k"> [-+*</span><span class="ch">\/</span><span class="k">%=&lt;&gt;;,|&amp;!()</span><span class="ch">\[</span><span class="ch">\]</span><span class="k">{}~?] | </span><span class="ch">\.</span><span class="ch">\.</span><span class="k">?</span><span class="ch">\.</span><span class="k">? | ::? </span><span class="dl">/</span><span class="mod">x</span></span>)<tt>
+</tt>          kind = <span class="sy">:operator</span><tt>
+</tt>          <span class="iv">@regexp_allowed</span> = <span class="sy">:set</span> <span class="r">if</span> <span class="iv">@scanner</span>.matched[<span class="i">-1</span>,<span class="i">1</span>] =~ <span class="rx"><span class="dl">/</span><span class="k">[~=!&lt;&gt;|&amp;^,</span><span class="ch">\(</span><span class="ch">\[</span><span class="k">+</span><span class="ch">\-</span><span class="ch">\/</span><span class="ch">\*</span><span class="k">%]</span><span class="ch">\z</span><span class="dl">/</span></span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="co">FLOAT</span>)<tt>
+</tt>          kind = <span class="sy">:float</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="co">INTEGER</span>)<tt>
+</tt>          kind = <span class="sy">:integer</span><tt>
+</tt>        <span class="r">elsif</span> <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">:(?:</span><span class="il"><span class="idl">#{</span><span class="co">GLOBAL_VARIABLE</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">METHOD_NAME_EX</span><span class="idl">}</span></span><span class="k">|</span><span class="il"><span class="idl">#{</span><span class="co">STRING</span><span class="idl">}</span></span><span class="k">)</span><span class="dl">/</span><span class="mod">ox</span></span>)<tt>
+</tt>          kind = <span class="sy">:global_variable</span><tt>
+</tt>        <span class="r">else</span><tt>
+</tt>          <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">.</span><span class="dl">/</span><span class="mod">m</span></span>)<tt>
+</tt>        <span class="r">end</span><tt>
+</tt>      <span class="r">end</span><tt>
+</tt>      <tt>
+</tt>      token = <span class="co">Token</span>.new <span class="iv">@scanner</span>.matched, kind<tt>
+</tt><tt>
+</tt>      <span class="r">if</span> kind == <span class="sy">:regexp</span><tt>
+</tt>        token.text &lt;&lt; <span class="iv">@scanner</span>.scan(<span class="rx"><span class="dl">/</span><span class="k">[eimnosux]*</span><span class="dl">/</span></span>)<tt>
+</tt>      <span class="r">end</span><tt>
+</tt>      <tt>
+</tt>      <span class="iv">@regexp_allowed</span> = (<span class="iv">@regexp_allowed</span> == <span class="sy">:set</span>)  <span class="c"># delayed flag setting</span><tt>
+</tt><tt>
+</tt>      token<tt>
+</tt>    <span class="r">end</span><tt>
+</tt>  <span class="r">end</span><tt>
+</tt>  <tt>
+</tt>  <span class="co">ScannerList</span>.register <span class="co">RubyScanner</span>, <span class="s"><span class="dl">'</span><span class="k">ruby</span><span class="dl">'</span></span><tt>
+</tt><tt>
+</tt><span class="r">end</span><tt>
+</tt><tt>
+</tt><span class="r">module</span> <span class="cl">CodeRay</span><tt>
+</tt>  require <span class="s"><span class="dl">'</span><span class="k">scanner</span><span class="dl">'</span></span><tt>
+</tt><tt>
+</tt>  <span class="r">class</span> <span class="cl">Highlighter</span><tt>
+</tt><tt>
+</tt>    <span class="r">def</span> <span class="fu">initialize</span> lang<tt>
+</tt>      <span class="iv">@scanner</span> = <span class="co">Scanner</span>[lang].new<tt>
+</tt>    <span class="r">end</span><tt>
+</tt><tt>
+</tt>    <span class="r">def</span> <span class="fu">highlight</span> code<tt>
+</tt>      <span class="iv">@scanner</span>.feed code<tt>
+</tt>      <span class="iv">@scanner</span>.all_tokens.map { |t| t.inspect }.join <span class="s"><span class="dl">&quot;</span><span class="ch">\n</span><span class="dl">&quot;</span></span><tt>
+</tt>    <span class="r">end</span><tt>
+</tt><tt>
+</tt>  <span class="r">end</span><tt>
+</tt><tt>
+</tt>  <span class="r">class</span> <span class="cl">HTMLHighlighter</span> &lt; <span class="co">Highlighter</span><tt>
+</tt>    <tt>
+</tt>    <span class="co">ClassOfKind</span> = {<tt>
+</tt>      <span class="sy">:attribute_name</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">an</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:attribute_name_fat</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">af</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:attribute_value</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">av</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:attribute_value_fat</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">aw</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:bin</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">bi</span><span class="dl">'</span></span>,<tt>
+</tt>       <span class="sy">:char</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ch</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:class</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">cl</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:class_variable</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">cv</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:color</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">cr</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:comment</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">c</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:constant</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">co</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:definition</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">df</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:directive</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">di</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:doc</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">do</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:doc_string</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ds</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:exception</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ex</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:error</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">er</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:float</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">fl</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:function</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">fu</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:global_variable</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">gv</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:hex</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">hx</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:include</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ic</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:instance_variable</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">iv</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:integer</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">i</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:interpreted</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">in</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:label</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">la</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:local_variable</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">lv</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:oct</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">oc</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:operator_name</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">on</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:pre_constant</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">pc</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:pre_type</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">pt</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:predefined</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">pd</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:preprocessor</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">pp</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:regexp</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">rx</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:reserved</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">r</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:shell</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">sh</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:string</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">s</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:symbol</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">sy</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:tag</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ta</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:tag_fat</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">tf</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:tag_special</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ts</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:type</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">ty</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:variable</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">v</span><span class="dl">'</span></span>,<tt>
+</tt>      <span class="sy">:xml_text</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">xt</span><span class="dl">'</span></span>,<tt>
+</tt><tt>
+</tt>      <span class="sy">:ident</span> =&gt; <span class="sy">:NO_HIGHLIGHT</span>,<tt>
+</tt>      <span class="sy">:operator</span> =&gt; <span class="sy">:NO_HIGHLIGHT</span>,<tt>
+</tt>      <span class="sy">:space</span> =&gt; <span class="sy">:NO_HIGHLIGHT</span>,<tt>
+</tt>    }<tt>
+</tt>    <span class="co">ClassOfKind</span>[<span class="sy">:procedure</span>] = <span class="co">ClassOfKind</span>[<span class="sy">:method</span>] = <span class="co">ClassOfKind</span>[<span class="sy">:function</span>]<tt>
+</tt>    <span class="co">ClassOfKind</span>.default = <span class="co">ClassOfKind</span>[<span class="sy">:error</span>] <span class="r">or</span> raise <span class="s"><span class="dl">'</span><span class="k">no class found for :error!</span><span class="dl">'</span></span><tt>
+</tt>    <tt>
+</tt>    <span class="r">def</span> <span class="fu">initialize</span> lang, options = {}<tt>
+</tt>      <span class="r">super</span> lang<tt>
+</tt>      <tt>
+</tt>      <span class="iv">@HTML_TAB</span> = <span class="s"><span class="dl">'</span><span class="k"> </span><span class="dl">'</span></span> * options.fetch(<span class="sy">:tabs2space</span>, <span class="i">8</span>)<tt>
+</tt>      <span class="r">case</span> level = options.fetch(<span class="sy">:level</span>, <span class="s"><span class="dl">'</span><span class="k">xhtml</span><span class="dl">'</span></span>)<tt>
+</tt>        <span class="r">when</span> <span class="s"><span class="dl">'</span><span class="k">html</span><span class="dl">'</span></span><tt>
+</tt>          <span class="iv">@HTML_BR</span> = <span class="s"><span class="dl">&quot;</span><span class="k">&lt;BR&gt;</span><span class="ch">\n</span><span class="dl">&quot;</span></span><tt>
+</tt>        <span class="r">when</span> <span class="s"><span class="dl">'</span><span class="k">xhtml</span><span class="dl">'</span></span><tt>
+</tt>          <span class="iv">@HTML_BR</span> = <span class="s"><span class="dl">&quot;</span><span class="k">&lt;br /&gt;</span><span class="ch">\n</span><span class="dl">&quot;</span></span><tt>
+</tt>      <span class="r">else</span><tt>
+</tt>        raise <span class="s"><span class="dl">&quot;</span><span class="k">Unknown HTML level: </span><span class="il"><span class="idl">#{</span>level<span class="idl">}</span></span><span class="dl">&quot;</span></span><tt>
+</tt>      <span class="r">end</span><tt>
+</tt>    <span class="r">end</span><tt>
+</tt><tt>
+</tt>    <span class="r">def</span> <span class="fu">highlight</span> code<tt>
+</tt>      <span class="iv">@scanner</span>.feed code<tt>
+</tt>      <tt>
+</tt>      out = <span class="s"><span class="dl">'</span><span class="dl">'</span></span><tt>
+</tt>      <span class="r">while</span> t = <span class="iv">@scanner</span>.next_token<tt>
+</tt>        warn t.inspect <span class="r">if</span> t.text.nil?<tt>
+</tt>        out &lt;&lt; to_html(t)<tt>
+</tt>      <span class="r">end</span><tt>
+</tt>      <span class="co">TEMPLATE</span> =~ <span class="rx"><span class="dl">/</span><span class="k">&lt;%CONTENT%&gt;</span><span class="dl">/</span></span><tt>
+</tt>      <span class="gv">$`</span> + out + <span class="gv">$'</span><tt>
+</tt>    <span class="r">end</span><tt>
+</tt>    <tt>
+</tt>  private<tt>
+</tt>    <span class="r">def</span> <span class="fu">to_html</span> token<tt>
+</tt>      css_class = <span class="co">ClassOfKind</span>[token.kind]<tt>
+</tt>      <span class="r">if</span> <span class="r">defined?</span> ::<span class="co">DEBUG</span> <span class="r">and</span> <span class="r">not</span> <span class="co">ClassOfKind</span>.has_key? token.kind<tt>
+</tt>        warn <span class="s"><span class="dl">&quot;</span><span class="k">no token class found for :</span><span class="il"><span class="idl">#{</span>token.kind<span class="idl">}</span></span><span class="dl">&quot;</span></span><tt>
+</tt>      <span class="r">end</span><tt>
+</tt>        <tt>
+</tt>      text = text_to_html token.text<tt>
+</tt>      <span class="r">if</span> css_class == <span class="sy">:NO_HIGHLIGHT</span><tt>
+</tt>        text<tt>
+</tt>      <span class="r">else</span><tt>
+</tt>        <span class="s"><span class="dl">&quot;</span><span class="k">&lt;span class=</span><span class="ch">\&quot;</span><span class="il"><span class="idl">#{</span>css_class<span class="idl">}</span></span><span class="ch">\&quot;</span><span class="k">&gt;</span><span class="il"><span class="idl">#{</span>text<span class="idl">}</span></span><span class="k">&lt;/span&gt;</span><span class="dl">&quot;</span></span><tt>
+</tt>      <span class="r">end</span><tt>
+</tt>    <span class="r">end</span><tt>
+</tt>    <tt>
+</tt>    <span class="r">def</span> <span class="fu">text_to_html</span> text<tt>
+</tt>      <span class="r">return</span> <span class="s"><span class="dl">'</span><span class="dl">'</span></span> <span class="r">if</span> text.empty?<tt>
+</tt>      text = text.dup  <span class="c"># important</span><tt>
+</tt>      <span class="r">if</span> text.index(<span class="rx"><span class="dl">/</span><span class="k">[&quot;&gt;&lt;&amp;]</span><span class="dl">/</span></span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">'</span><span class="k">&amp;</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">&amp;amp;</span><span class="dl">'</span></span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">'</span><span class="k">&quot;</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">&amp;quot;</span><span class="dl">'</span></span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">'</span><span class="k">&gt;</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">&amp;gt;</span><span class="dl">'</span></span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">'</span><span class="k">&lt;</span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k">&amp;lt;</span><span class="dl">'</span></span>)<tt>
+</tt>      <span class="r">end</span><tt>
+</tt>      <span class="r">if</span> text.index(<span class="rx"><span class="dl">/</span><span class="ch">\s</span><span class="dl">/</span></span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">&quot;</span><span class="ch">\n</span><span class="dl">&quot;</span></span>, <span class="iv">@HTML_BR</span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">&quot;</span><span class="ch">\t</span><span class="dl">&quot;</span></span>, <span class="iv">@HTML_TAB</span>)<tt>
+</tt>        text.gsub!(<span class="rx"><span class="dl">/</span><span class="k">^ </span><span class="dl">/</span></span>, <span class="s"><span class="dl">'</span><span class="k">&amp;nbsp;</span><span class="dl">'</span></span>)<tt>
+</tt>        text.gsub!(<span class="s"><span class="dl">'</span><span class="k">  </span><span class="dl">'</span></span>, <span class="s"><span class="dl">'</span><span class="k"> &amp;nbsp;</span><span class="dl">'</span></span>)<tt>
+</tt>      <span class="r">end</span><tt>
+</tt>      text<tt>
+</tt>    <span class="r">end</span><tt>
+</tt>    <tt>
+</tt>    <span class="co">TEMPLATE</span> = <span class="s"><span class="dl">&lt;&lt;-'TEMPLATE'</span></span><span class="s"><span class="k"><tt>
+</tt>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.01//EN&quot; &quot;http://www.w3.org/TR/html4/strict.dtd&quot;&gt;<tt>
+</tt>&lt;html dir=&quot;ltr&quot;&gt;<tt>
+</tt>&lt;head&gt;<tt>
+</tt>&lt;meta http-equiv=&quot;Content-Type&quot; content=&quot;text/html; charset=iso-8859-1&quot;&gt;<tt>
+</tt>&lt;meta http-equiv=&quot;Content-Style-Type&quot; content=&quot;text/css&quot;&gt;<tt>
+</tt><tt>
+</tt>&lt;title&gt;RubyBB BBCode&lt;/title&gt;<tt>
+</tt>&lt;style type=&quot;text/css&quot;&gt;<tt>
+</tt>.code {<tt>
+</tt>  width: 100%;<tt>
+</tt>  background-color: #FAFAFA;<tt>
+</tt>  border: 1px solid #D1D7DC;<tt>
+</tt>  font-family: 'Courier New', 'Terminal', monospace;<tt>
+</tt>  font-size: 10pt;<tt>
+</tt>  color: black;<tt>
+</tt>  vertical-align: top;<tt>
+</tt>  text-align: left;<tt>
+</tt>}<tt>
+</tt>.code .af { color:#00C; }<tt>
+</tt>.code .an { color:#007; }<tt>
+</tt>.code .av { color:#700; }<tt>
+</tt>.code .aw { color:#C00; }<tt>
+</tt>.code .bi { color:#509; font-weight:bold; }<tt>
+</tt>.code .c  { color:#888; }<tt>
+</tt>.code .ch { color:#C28; font-weight:bold; }<tt>
+</tt>.code .cl { color:#B06; font-weight:bold; }<tt>
+</tt>.code .co { color:#036; font-weight:bold; }<tt>
+</tt>.code .cr { color:#0A0; }<tt>
+</tt>.code .cv { color:#369; }<tt>
+</tt>.code .df { color:#099; font-weight:bold; }<tt>
+</tt>.code .di { color:#088; font-weight:bold; }<tt>
+</tt>.code .do { color:#970; }<tt>
+</tt>.code .ds { color:#D42; font-weight:bold; }<tt>
+</tt>.code .er { color:#F00; background-color:#FAA; }<tt>
+</tt>.code .ex { color:#F00; font-weight:bold; }<tt>
+</tt>.code .fl { color:#60E; font-weight:bold; }<tt>
+</tt>.code .fu { color:#06B; font-weight:bold; }<tt>
+</tt>.code .gv { color:#800; font-weight:bold; }<tt>
+</tt>.code .hx { color:#058; font-weight:bold; }<tt>
+</tt>.code .i  { color:#00D; font-weight:bold; }<tt>
+</tt>.code .ic { color:#B44; font-weight:bold; }<tt>
+</tt>.code .in { color:#B2B; font-weight:bold; }<tt>
+</tt>.code .iv { color:#33B; }<tt>
+</tt>.code .la { color:#970; font-weight:bold; }<tt>
+</tt>.code .lv { color:#963; }<tt>
+</tt>.code .oc { color:#40E; font-weight:bold; }<tt>
+</tt>.code .on { color:#000; font-weight:bold; }<tt>
+</tt>.code .pc { color:#038; font-weight:bold; }<tt>
+</tt>.code .pd { color:#369; font-weight:bold; }<tt>
+</tt>.code .pp { color:#579; }<tt>
+</tt>.code .pt { color:#339; font-weight:bold; }<tt>
+</tt>.code .r  { color:#080; font-weight:bold; }<tt>
+</tt>.code .rx { color:#927; font-weight:bold; }<tt>
+</tt>.code .s  { color:#D42; font-weight:bold; }<tt>
+</tt>.code .sh { color:#B2B; font-weight:bold; }<tt>
+</tt>.code .sy { color:#A60; }<tt>
+</tt>.code .ta { color:#070; }<tt>
+</tt>.code .tf { color:#070; font-weight:bold; }<tt>
+</tt>.code .ts { color:#D70; font-weight:bold; }<tt>
+</tt>.code .ty { color:#339; font-weight:bold; }<tt>
+</tt>.code .v  { color:#036; }<tt>
+</tt>.code .xt { color:#444; }<tt>
+</tt>&lt;/style&gt;<tt>
+</tt>&lt;/head&gt;<tt>
+</tt>&lt;body&gt;<tt>
+</tt>&lt;div class=&quot;code&quot;&gt;<tt>
+</tt>&lt;%CONTENT%&gt;<tt>
+</tt>&lt;/div&gt;<tt>
+</tt>&lt;div class=&quot;validators&quot;&gt;<tt>
+</tt>&lt;a href=&quot;http://validator.w3.org/check?uri=referer&quot;&gt;&lt;img src=&quot;http://www.w3.org/Icons/valid-html401&quot; alt=&quot;Valid HTML 4.01!&quot; height=&quot;31&quot; width=&quot;88&quot; style=&quot;border:none;&quot;&gt;&lt;/a&gt;<tt>
+</tt>&lt;img style=&quot;border:0&quot; src=&quot;http://jigsaw.w3.org/css-validator/images/vcss&quot; alt=&quot;Valid CSS!&quot; &gt;<tt>
+</tt>&lt;/div&gt;    <tt>
+</tt>&lt;/body&gt;<tt>
+</tt>&lt;/html&gt;</span><span class="dl"><tt>
+</tt>    TEMPLATE</span></span><tt>
+</tt><tt>
+</tt>  <span class="r">end</span><tt>
+</tt><tt>
+</tt><span class="r">end</span><tt>
+</tt></pre></td>
+</tr></table>
+
+</body>
+</html>
diff --git a/test/samples/html.rb b/test/samples/html.rb
new file mode 100644
index 0000000..c18284a
--- /dev/null
+++ b/test/samples/html.rb
@@ -0,0 +1,394 @@
+$: << '..'
+require 'coderay'
+
+tokens = CodeRay.scan DATA.read, :ruby
+html = tokens.page(:tab_width => 2, :line_numbers => :table, :title => 'CodeRay HTML Encoder Example')
+
+puts html
+
+__END__
+require 'scanner'
+
+module CodeRay
+	
+	class RubyScanner < Scanner
+		
+		RESERVED_WORDS = [
+			'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
+			'defined?', 'ensure', 'module', 'redo', 'super', 'until',
+			'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
+			'when', 'END', 'case', 'else', 'for', 'retry',
+			'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
+			'undef', 'yield',
+		]
+
+		DEF_KEYWORDS = ['def']
+		MODULE_KEYWORDS = ['class', 'module']
+		DEF_NEW_STATE = WordList.new(:initial).
+			add(DEF_KEYWORDS, :def_expected).
+			add(MODULE_KEYWORDS, :module_expected)
+
+		WORDS_ALLOWING_REGEXP = [
+			'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
+		]
+		REGEXP_ALLOWED = WordList.new(false).
+			add(WORDS_ALLOWING_REGEXP, :set)
+		
+		PREDEFINED_CONSTANTS = [
+			'nil', 'true', 'false', 'self',
+			'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
+		]
+
+		IDENT_KIND = WordList.new(:ident).
+			add(RESERVED_WORDS, :reserved).
+			add(PREDEFINED_CONSTANTS, :pre_constant)
+
+		METHOD_NAME = / #{IDENT} [?!]? /xo
+		METHOD_NAME_EX = /
+		 #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
+		 | \*\*?         # multiplication and power
+		 | [-+~]@?       # plus, minus
+		 | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
+		 | \[\]=?        # array getter and setter
+		 | <=?>? | >=?   # comparison, rocket operator
+		 | << | >>       # append or shift left, shift right
+		 | ===?          # simple equality and case equality
+		/ox
+		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
+
+		DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
+		SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
+		STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
+		SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
+		REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
+		
+		DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
+		OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+		HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+		BINARY = /0b[01]+(?:_[01]+)*/
+
+		EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+		FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
+		INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
+		
+		def reset
+			super
+			@regexp_allowed = false
+		end
+		
+		def next_token
+			return if @scanner.eos?
+
+			kind = :error
+			if @scanner.scan(/\s+/)  # in every state
+				kind = :space
+				@regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
+
+			elsif @state == :def_expected
+				if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
+					kind = :method
+					@state = :initial
+				else
+					@scanner.scan(/./)
+					kind = :error
+				end
+				@state = :initial
+				
+			elsif @state == :module_expected
+				if @scanner.scan(/<</)
+					kind = :operator
+				else
+					if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
+						kind = :method
+					else
+						@scanner.scan(/./)
+						kind = :error
+					end
+					@state = :initial
+				end
+				
+			elsif # state == :initial
+				# IDENTIFIERS, KEYWORDS
+				if @scanner.scan(GLOBAL_VARIABLE)
+					kind = :global_variable
+				elsif @scanner.scan(/ @@ #{IDENT} /ox)
+					kind = :class_variable
+				elsif @scanner.scan(/ @ #{IDENT} /ox)
+					kind = :instance_variable
+				elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
+					kind = :comment
+				elsif @scanner.scan(METHOD_NAME)
+					if @last_token_dot
+						kind = :ident
+					else
+						matched = @scanner.matched
+						kind = IDENT_KIND[matched]
+						if kind == :ident and matched =~ /^[A-Z]/
+							kind = :constant
+						elsif kind == :reserved
+							@state = DEF_NEW_STATE[matched]
+							@regexp_allowed = REGEXP_ALLOWED[matched]
+						end
+					end
+					
+				elsif @scanner.scan(STRING)
+					kind = :string
+				elsif @scanner.scan(SHELL)
+					kind = :shell
+				## HEREDOCS
+				elsif @scanner.scan(/\//) and @regexp_allowed
+				 	@scanner.unscan
+				 	@scanner.scan(REGEXP)
+					kind = :regexp
+				## %strings
+				elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
+					kind = :global_variable
+				elsif @scanner.scan(/
+				  \? (?:
+				    [^\s\\]
+				  | 
+				    \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
+				  )
+				/ox)
+					kind = :integer
+					
+				elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
+					kind = :operator
+					@regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
+				elsif @scanner.scan(FLOAT)
+					kind = :float
+				elsif @scanner.scan(INTEGER)
+					kind = :integer
+				elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
+					kind = :global_variable
+				else
+					@scanner.scan(/./m)
+				end
+			end
+			
+			token = Token.new @scanner.matched, kind
+
+			if kind == :regexp
+				token.text << @scanner.scan(/[eimnosux]*/)
+			end
+			
+			@regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
+
+			token
+		end
+	end
+	
+	ScannerList.register RubyScanner, 'ruby'
+
+end
+
+module CodeRay
+	require 'scanner'
+
+	class Highlighter
+
+		def initialize lang
+			@scanner = Scanner[lang].new
+		end
+
+		def highlight code
+			@scanner.feed code
+			@scanner.all_tokens.map { |t| t.inspect }.join "\n"
+		end
+
+	end
+
+	class HTMLHighlighter < Highlighter
+		
+		ClassOfKind = {
+			:attribute_name => 'an',
+			:attribute_name_fat => 'af',
+			:attribute_value => 'av',
+			:attribute_value_fat => 'aw',
+			:bin => 'bi',
+	 		:char => 'ch',
+			:class => 'cl',
+			:class_variable => 'cv',
+			:color => 'cr',
+			:comment => 'c',
+			:constant => 'co',
+			:definition => 'df',
+			:directive => 'di',
+			:doc => 'do',
+			:doc_string => 'ds',
+			:exception => 'ex',
+			:error => 'er',
+			:float => 'fl',
+			:function => 'fu',
+			:global_variable => 'gv',
+			:hex => 'hx',
+			:include => 'ic',
+			:instance_variable => 'iv',
+			:integer => 'i',
+			:interpreted => 'in',
+			:label => 'la',
+			:local_variable => 'lv',
+			:oct => 'oc',
+			:operator_name => 'on',
+			:pre_constant => 'pc',
+			:pre_type => 'pt',
+			:predefined => 'pd',
+			:preprocessor => 'pp',
+			:regexp => 'rx',
+			:reserved => 'r',
+			:shell => 'sh',
+			:string => 's',
+			:symbol => 'sy',
+			:tag => 'ta',
+			:tag_fat => 'tf',
+			:tag_special => 'ts',
+			:type => 'ty',
+			:variable => 'v',
+			:xml_text => 'xt',
+
+			:ident => :NO_HIGHLIGHT,
+			:operator => :NO_HIGHLIGHT,
+			:space => :NO_HIGHLIGHT,
+		}
+		ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+		ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+		
+		def initialize lang, options = {}
+			super lang
+			
+			@HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
+			case level = options.fetch(:level, 'xhtml')
+				when 'html'
+					@HTML_BR = "<BR>\n"
+				when 'xhtml'
+					@HTML_BR = "<br />\n"
+			else
+				raise "Unknown HTML level: #{level}"
+			end
+		end
+
+		def highlight code
+			@scanner.feed code
+			
+			out = ''
+			while t = @scanner.next_token
+				warn t.inspect if t.text.nil?
+				out << to_html(t)
+			end
+			TEMPLATE =~ /<%CONTENT%>/
+			$` + out + $'
+		end
+		
+	private
+		def to_html token
+			css_class = ClassOfKind[token.kind]
+			if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
+				warn "no token class found for :#{token.kind}"
+			end
+				
+			text = text_to_html token.text
+			if css_class == :NO_HIGHLIGHT
+				text
+			else
+				"<span class=\"#{css_class}\">#{text}</span>"
+			end
+		end
+		
+		def text_to_html text
+			return '' if text.empty?
+			text = text.dup  # important
+			if text.index(/["><&]/)
+				text.gsub!('&', '&amp;')
+				text.gsub!('"', '&quot;')
+				text.gsub!('>', '&gt;')
+				text.gsub!('<', '&lt;')
+			end
+			if text.index(/\s/)
+				text.gsub!("\n", @HTML_BR)
+				text.gsub!("\t", @HTML_TAB)
+				text.gsub!(/^ /, '&nbsp;')
+				text.gsub!('  ', ' &nbsp;')
+			end
+			text
+		end
+		
+		TEMPLATE = <<-'TEMPLATE'
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html dir="ltr">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta http-equiv="Content-Style-Type" content="text/css">
+
+<title>RubyBB BBCode</title>
+<style type="text/css">
+.code {
+	width: 100%;
+	background-color: #FAFAFA;
+	border: 1px solid #D1D7DC;
+	font-family: 'Courier New', 'Terminal', monospace;
+	font-size: 10pt;
+	color: black;
+	vertical-align: top;
+	text-align: left;
+}
+.code .af { color:#00C; }
+.code .an { color:#007; }
+.code .av { color:#700; }
+.code .aw { color:#C00; }
+.code .bi { color:#509; font-weight:bold; }
+.code .c  { color:#888; }
+.code .ch { color:#C28; font-weight:bold; }
+.code .cl { color:#B06; font-weight:bold; }
+.code .co { color:#036; font-weight:bold; }
+.code .cr { color:#0A0; }
+.code .cv { color:#369; }
+.code .df { color:#099; font-weight:bold; }
+.code .di { color:#088; font-weight:bold; }
+.code .do { color:#970; }
+.code .ds { color:#D42; font-weight:bold; }
+.code .er { color:#F00; background-color:#FAA; }
+.code .ex { color:#F00; font-weight:bold; }
+.code .fl { color:#60E; font-weight:bold; }
+.code .fu { color:#06B; font-weight:bold; }
+.code .gv { color:#800; font-weight:bold; }
+.code .hx { color:#058; font-weight:bold; }
+.code .i  { color:#00D; font-weight:bold; }
+.code .ic { color:#B44; font-weight:bold; }
+.code .in { color:#B2B; font-weight:bold; }
+.code .iv { color:#33B; }
+.code .la { color:#970; font-weight:bold; }
+.code .lv { color:#963; }
+.code .oc { color:#40E; font-weight:bold; }
+.code .on { color:#000; font-weight:bold; }
+.code .pc { color:#038; font-weight:bold; }
+.code .pd { color:#369; font-weight:bold; }
+.code .pp { color:#579; }
+.code .pt { color:#339; font-weight:bold; }
+.code .r  { color:#080; font-weight:bold; }
+.code .rx { color:#927; font-weight:bold; }
+.code .s  { color:#D42; font-weight:bold; }
+.code .sh { color:#B2B; font-weight:bold; }
+.code .sy { color:#A60; }
+.code .ta { color:#070; }
+.code .tf { color:#070; font-weight:bold; }
+.code .ts { color:#D70; font-weight:bold; }
+.code .ty { color:#339; font-weight:bold; }
+.code .v  { color:#036; }
+.code .xt { color:#444; }
+</style>
+</head>
+<body>
+<div class="code">
+<%CONTENT%>
+</div>
+<div class="validators">
+<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
+<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
+</div>    
+</body>
+</html>
+		TEMPLATE
+
+	end
+
+end
diff --git a/test/samples/html2.expected b/test/samples/html2.expected
new file mode 100644
index 0000000..c8ae56a
--- /dev/null
+++ b/test/samples/html2.expected
@@ -0,0 +1,185 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+  <title>CodeRay HTML Encoder Example</title>
+  <style type="text/css">
+.CodeRay {
+  background-color: #f8f8f8;
+  border: 1px solid silver;
+  font-family: 'Courier New', 'Terminal', monospace;
+  color: #000;
+}
+.CodeRay pre { margin: 0px }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
+table.CodeRay td { padding: 2px 4px; vertical-align: top }
+
+.CodeRay .line_numbers, .CodeRay .no {
+  background-color: #def;
+  color: gray;
+  text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .no { padding: 0px 4px }
+.CodeRay .code { width: 100% }
+
+ol.CodeRay { font-size: 10pt }
+ol.CodeRay li { white-space: pre }
+
+.CodeRay .code pre { overflow: auto }
+
+.CodeRay .debug { color:white ! important; background:blue ! important; }
+
+.CodeRay .af { color:#00C }
+.CodeRay .an { color:#007 }
+.CodeRay .at { color:#f08 }
+.CodeRay .av { color:#700 }
+.CodeRay .aw { color:#C00 }
+.CodeRay .bi { color:#509; font-weight:bold }
+.CodeRay .c  { color:#888; }
+
+.CodeRay .ch { color:#04D }
+.CodeRay .ch .k { color:#04D }
+.CodeRay .ch .dl { color:#039 }
+
+.CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
+.CodeRay .co { color:#036; font-weight:bold }
+.CodeRay .cr { color:#0A0 }
+.CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
+.CodeRay .df { color:#099; font-weight:bold }
+.CodeRay .di { color:#088; font-weight:bold }
+.CodeRay .dl { color:black }
+.CodeRay .do { color:#970 }
+.CodeRay .dt { color:#34b }
+.CodeRay .ds { color:#D42; font-weight:bold }
+.CodeRay .e  { color:#666; font-weight:bold }
+.CodeRay .en { color:#800; font-weight:bold }
+.CodeRay .er { color:#F00; background-color:#FAA }
+.CodeRay .ex { color:#C00; font-weight:bold }
+.CodeRay .fl { color:#60E; font-weight:bold }
+.CodeRay .fu { color:#06B; font-weight:bold }
+.CodeRay .gv { color:#d70; font-weight:bold }
+.CodeRay .hx { color:#058; font-weight:bold }
+.CodeRay .i  { color:#00D; font-weight:bold }
+.CodeRay .ic { color:#B44; font-weight:bold }
+
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
+
+.CodeRay .im { color:#f00; }
+.CodeRay .in { color:#B2B; font-weight:bold }
+.CodeRay .iv { color:#33B }
+.CodeRay .la { color:#970; font-weight:bold }
+.CodeRay .lv { color:#963 }
+.CodeRay .oc { color:#40E; font-weight:bold }
+.CodeRay .of { color:#000; font-weight:bold }
+.CodeRay .op { }
+.CodeRay .pc { color:#038; font-weight:bold }
+.CodeRay .pd { color:#369; font-weight:bold }
+.CodeRay .pp { color:#579; }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
+.CodeRay .r, .kw  { color:#080; font-weight:bold }
+
+.CodeRay .ke { color: #808; }
+.CodeRay .ke .dl { color: #606; }
+.CodeRay .ke .ch { color: #80f; }
+.CodeRay .vl { color: #088; }
+
+.CodeRay .rx { background-color:#fff0ff }
+.CodeRay .rx .k { color:#808 }
+.CodeRay .rx .dl { color:#404 }
+.CodeRay .rx .mod { color:#C2C }
+.CodeRay .rx .fu  { color:#404; font-weight: bold }
+
+.CodeRay .s { background-color:#fff0f0; color: #D20; }
+.CodeRay .s .s { background-color:#ffe0e0 }
+.CodeRay .s .s  .s { background-color:#ffd0d0 }
+.CodeRay .s .k { }
+.CodeRay .s .ch { color: #b0b; }
+.CodeRay .s .dl { color: #710; }
+
+.CodeRay .sh { background-color:#f0fff0; color:#2B2 }
+.CodeRay .sh .k { }
+.CodeRay .sh .dl { color:#161 }
+
+.CodeRay .sy { color:#A60 }
+.CodeRay .sy .k { color:#A60 }
+.CodeRay .sy .dl { color:#630 }
+
+.CodeRay .ta { color:#070 }
+.CodeRay .tf { color:#070; font-weight:bold }
+.CodeRay .ts { color:#D70; font-weight:bold }
+.CodeRay .ty { color:#339; font-weight:bold }
+.CodeRay .v  { color:#036 }
+.CodeRay .xt { color:#444 }
+
+.CodeRay .ins { background: #afa; }
+.CodeRay .del { background: #faa; }
+.CodeRay .chg { color: #aaf; background: #007; }
+.CodeRay .head { color: #f8f; background: #505 }
+
+.CodeRay .ins .ins { color: #080; font-weight:bold }
+.CodeRay .del .del { color: #800; font-weight:bold }
+.CodeRay .chg .chg { color: #66f; }
+.CodeRay .head .head { color: #f4f; }
+
+  </style>
+</head>
+<body style="background-color: white;">
+
+<table class="CodeRay"><tr>
+  <td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre>1<tt>
+</tt>2<tt>
+</tt>3<tt>
+</tt>4<tt>
+</tt>5<tt>
+</tt>6<tt>
+</tt>7<tt>
+</tt>8<tt>
+</tt>9<tt>
+</tt><strong>10</strong><tt>
+</tt>11<tt>
+</tt></pre></td>
+  <td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }">require <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">coderay</span><span style="color:#710">'</span></span><tt>
+</tt><tt>
+</tt><span style="color:#888"># scan this file</span><tt>
+</tt>tokens = <span style="color:#036;font-weight:bold">CodeRay</span>.scan(<span style="color:#036;font-weight:bold">File</span>.read(<span style="color:#d70;font-weight:bold">$0</span>) * <span style="color:#00D;font-weight:bold">1</span>, <span style="color:#A60">:ruby</span>)<tt>
+</tt><tt>
+</tt><span style="color:#888"># output it with two styles of line numbers</span><tt>
+</tt>out = tokens.div(<span style="color:#A60">:line_numbers</span> =&gt; <span style="color:#A60">:table</span>)<tt>
+</tt>out &lt;&lt; <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">&lt;hr /&gt;</span><span style="color:#710">'</span></span><tt>
+</tt>out &lt;&lt; tokens.div(<span style="color:#A60">:line_numbers</span> =&gt; <span style="color:#A60">:inline</span>, <span style="color:#A60">:line_number_start</span> =&gt; <span style="color:#00D;font-weight:bold">8</span>)<tt>
+</tt><tt>
+</tt>puts out.page(<span style="color:#A60">:title</span> =&gt; <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">CodeRay HTML Encoder Example</span><span style="color:#710">'</span></span>)<tt>
+</tt></pre></td>
+</tr></table>
+<hr /><div class="CodeRay">
+  <div class="code"><pre><span class="no"> 8</span> require <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">coderay</span><span style="color:#710">'</span></span>
+<span class="no"> 9</span> 
+<span class="no"><strong>10</strong></span> <span style="color:#888"># scan this file</span>
+<span class="no">11</span> tokens = <span style="color:#036;font-weight:bold">CodeRay</span>.scan(<span style="color:#036;font-weight:bold">File</span>.read(<span style="color:#d70;font-weight:bold">$0</span>) * <span style="color:#00D;font-weight:bold">1</span>, <span style="color:#A60">:ruby</span>)
+<span class="no">12</span> 
+<span class="no">13</span> <span style="color:#888"># output it with two styles of line numbers</span>
+<span class="no">14</span> out = tokens.div(<span style="color:#A60">:line_numbers</span> =&gt; <span style="color:#A60">:table</span>)
+<span class="no">15</span> out &lt;&lt; <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">&lt;hr /&gt;</span><span style="color:#710">'</span></span>
+<span class="no">16</span> out &lt;&lt; tokens.div(<span style="color:#A60">:line_numbers</span> =&gt; <span style="color:#A60">:inline</span>, <span style="color:#A60">:line_number_start</span> =&gt; <span style="color:#00D;font-weight:bold">8</span>)
+<span class="no">17</span> 
+<span class="no">18</span> puts out.page(<span style="color:#A60">:title</span> =&gt; <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">CodeRay HTML Encoder Example</span><span style="color:#710">'</span></span>)
+</pre></div>
+</div>
+
+</body>
+</html>
diff --git a/test/samples/html2.rb b/test/samples/html2.rb
new file mode 100644
index 0000000..618d168
--- /dev/null
+++ b/test/samples/html2.rb
@@ -0,0 +1,11 @@
+require 'coderay'
+
+# scan this file
+tokens = CodeRay.scan(File.read($0) * 1, :ruby)
+
+# output it with two styles of line numbers
+out = tokens.div(:line_numbers => :table)
+out << '<hr />'
+out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
+
+puts out.page(:title => 'CodeRay HTML Encoder Example')
diff --git a/test/samples/html_list.expected b/test/samples/html_list.expected
new file mode 100644
index 0000000..a4092c8
--- /dev/null
+++ b/test/samples/html_list.expected
@@ -0,0 +1,160 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+  <title>CodeRay HTML Encoder Example</title>
+  <style type="text/css">
+.CodeRay {
+  background-color: #f8f8f8;
+  border: 1px solid silver;
+  font-family: 'Courier New', 'Terminal', monospace;
+  color: #000;
+}
+.CodeRay pre { margin: 0px }
+
+div.CodeRay { }
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px }
+table.CodeRay td { padding: 2px 4px; vertical-align: top }
+
+.CodeRay .line_numbers, .CodeRay .no {
+  background-color: #def;
+  color: gray;
+  text-align: right;
+}
+.CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .no { padding: 0px 4px }
+.CodeRay .code { width: 100% }
+
+ol.CodeRay { font-size: 10pt }
+ol.CodeRay li { white-space: pre }
+
+.CodeRay .code pre { overflow: auto }
+
+.CodeRay .debug { color:white ! important; background:blue ! important; }
+
+.CodeRay .af { color:#00C }
+.CodeRay .an { color:#007 }
+.CodeRay .at { color:#f08 }
+.CodeRay .av { color:#700 }
+.CodeRay .aw { color:#C00 }
+.CodeRay .bi { color:#509; font-weight:bold }
+.CodeRay .c  { color:#888; }
+
+.CodeRay .ch { color:#04D }
+.CodeRay .ch .k { color:#04D }
+.CodeRay .ch .dl { color:#039 }
+
+.CodeRay .cl { color:#B06; font-weight:bold }
+.CodeRay .cm { color:#A08; font-weight:bold }
+.CodeRay .co { color:#036; font-weight:bold }
+.CodeRay .cr { color:#0A0 }
+.CodeRay .cv { color:#369 }
+.CodeRay .de { color:#B0B; }
+.CodeRay .df { color:#099; font-weight:bold }
+.CodeRay .di { color:#088; font-weight:bold }
+.CodeRay .dl { color:black }
+.CodeRay .do { color:#970 }
+.CodeRay .dt { color:#34b }
+.CodeRay .ds { color:#D42; font-weight:bold }
+.CodeRay .e  { color:#666; font-weight:bold }
+.CodeRay .en { color:#800; font-weight:bold }
+.CodeRay .er { color:#F00; background-color:#FAA }
+.CodeRay .ex { color:#C00; font-weight:bold }
+.CodeRay .fl { color:#60E; font-weight:bold }
+.CodeRay .fu { color:#06B; font-weight:bold }
+.CodeRay .gv { color:#d70; font-weight:bold }
+.CodeRay .hx { color:#058; font-weight:bold }
+.CodeRay .i  { color:#00D; font-weight:bold }
+.CodeRay .ic { color:#B44; font-weight:bold }
+
+.CodeRay .il { background: #ddd; color: black }
+.CodeRay .il .il { background: #ccc }
+.CodeRay .il .il .il { background: #bbb }
+.CodeRay .il .idl { background: #ddd; font-weight: bold; color: #666 }
+.CodeRay .idl { background-color: #bbb; font-weight: bold; color: #666; }
+
+.CodeRay .im { color:#f00; }
+.CodeRay .in { color:#B2B; font-weight:bold }
+.CodeRay .iv { color:#33B }
+.CodeRay .la { color:#970; font-weight:bold }
+.CodeRay .lv { color:#963 }
+.CodeRay .oc { color:#40E; font-weight:bold }
+.CodeRay .of { color:#000; font-weight:bold }
+.CodeRay .op { }
+.CodeRay .pc { color:#038; font-weight:bold }
+.CodeRay .pd { color:#369; font-weight:bold }
+.CodeRay .pp { color:#579; }
+.CodeRay .ps { color:#00C; font-weight:bold }
+.CodeRay .pt { color:#074; font-weight:bold }
+.CodeRay .r, .kw  { color:#080; font-weight:bold }
+
+.CodeRay .ke { color: #808; }
+.CodeRay .ke .dl { color: #606; }
+.CodeRay .ke .ch { color: #80f; }
+.CodeRay .vl { color: #088; }
+
+.CodeRay .rx { background-color:#fff0ff }
+.CodeRay .rx .k { color:#808 }
+.CodeRay .rx .dl { color:#404 }
+.CodeRay .rx .mod { color:#C2C }
+.CodeRay .rx .fu  { color:#404; font-weight: bold }
+
+.CodeRay .s { background-color:#fff0f0; color: #D20; }
+.CodeRay .s .s { background-color:#ffe0e0 }
+.CodeRay .s .s  .s { background-color:#ffd0d0 }
+.CodeRay .s .k { }
+.CodeRay .s .ch { color: #b0b; }
+.CodeRay .s .dl { color: #710; }
+
+.CodeRay .sh { background-color:#f0fff0; color:#2B2 }
+.CodeRay .sh .k { }
+.CodeRay .sh .dl { color:#161 }
+
+.CodeRay .sy { color:#A60 }
+.CodeRay .sy .k { color:#A60 }
+.CodeRay .sy .dl { color:#630 }
+
+.CodeRay .ta { color:#070 }
+.CodeRay .tf { color:#070; font-weight:bold }
+.CodeRay .ts { color:#D70; font-weight:bold }
+.CodeRay .ty { color:#339; font-weight:bold }
+.CodeRay .v  { color:#036 }
+.CodeRay .xt { color:#444 }
+
+.CodeRay .ins { background: #afa; }
+.CodeRay .del { background: #faa; }
+.CodeRay .chg { color: #aaf; background: #007; }
+.CodeRay .head { color: #f8f; background: #505 }
+
+.CodeRay .ins .ins { color: #080; font-weight:bold }
+.CodeRay .del .del { color: #800; font-weight:bold }
+.CodeRay .chg .chg { color: #66f; }
+.CodeRay .head .head { color: #f4f; }
+
+  </style>
+</head>
+<body style="background-color: white;">
+
+<div class="CodeRay">
+  <div class="code"><pre><span class="no">-1</span> <span class="gv">$:</span> &lt;&lt; <span class="s"><span class="dl">'</span><span class="k">..</span><span class="dl">'</span></span>
+<span class="no"> <strong>0</strong></span> require <span class="s"><span class="dl">'</span><span class="k">coderay</span><span class="dl">'</span></span>
+<span class="no"> 1</span> 
+<span class="no"> 2</span> tokens = <span class="co">CodeRay</span>.scan <span class="co">File</span>.read(<span class="pc">__FILE__</span>), <span class="sy">:ruby</span>
+<span class="no"> 3</span> html = tokens.html(<span class="sy">:tab_width</span> =&gt; <span class="i">2</span>, <span class="sy">:line_numbers</span> =&gt; <span class="sy">:inline</span>, <span class="sy">:line_number_start</span> =&gt; <span class="i">-1</span>)
+<span class="no"> 4</span> 
+<span class="no"> 5</span> puts html.page(<span class="sy">:title</span> =&gt; <span class="s"><span class="dl">'</span><span class="k">CodeRay HTML Encoder Example</span><span class="dl">'</span></span>)
+<span class="no"> 6</span> 
+<span class="no"> 7</span> commment = <span class="s"><span class="dl">&lt;&lt;_</span></span><span class="s"><span class="k">
+<span class="no"> 8</span> This code must be &gt; 10 lines
+<span class="no"> 9</span> because I want to test the correct adjustment of the line numbers.</span><span class="dl">
+<span class="no"><strong>10</strong></span> _</span></span>
+</pre></div>
+</div>
+
+</body>
+</html>
diff --git a/test/samples/html_list.rb b/test/samples/html_list.rb
new file mode 100644
index 0000000..fdfa512
--- /dev/null
+++ b/test/samples/html_list.rb
@@ -0,0 +1,12 @@
+$: << '..'

+require 'coderay'

+

+tokens = CodeRay.scan File.read(__FILE__), :ruby

+html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)

+

+puts html.page(:title => 'CodeRay HTML Encoder Example')

+

+commment = <<_

+This code must be > 10 lines

+because I want to test the correct adjustment of the line numbers.

+_

diff --git a/test/samples/load_encoder.expected b/test/samples/load_encoder.expected
new file mode 100644
index 0000000..1cff356
--- /dev/null
+++ b/test/samples/load_encoder.expected
@@ -0,0 +1,8 @@
+CodeRay::Encoders::YAML is not defined; you must load it first.
+Now it is loaded: CodeRay::Encoders::YAML
+See?
+Require is also possible: CodeRay::Encoders::Tokens
+See?
+Now load some mapped encoders: stats and plain.
+Require all Encoders:
+[[:count, CodeRay::Encoders::Count], [:debug, CodeRay::Encoders::Debug], [:div, CodeRay::Encoders::Div], [:html, CodeRay::Encoders::HTML], [:null, CodeRay::Encoders::Null], [:page, CodeRay::Encoders::Page], [:plain, :text], [:span, CodeRay::Encoders::Span], [:statistic, CodeRay::Encoders::Statistic], [:stats, CodeRay::Encoders::Statistic], [:text, CodeRay::Encoders::Text], [:tokens, CodeRay::Encoders::Tokens], [:xml, CodeRay::Encoders::XML], [:yaml, CodeRay::Encoders::YAML]]
diff --git a/test/samples/load_encoder.rb b/test/samples/load_encoder.rb
new file mode 100644
index 0000000..9594bfa
--- /dev/null
+++ b/test/samples/load_encoder.rb
@@ -0,0 +1,25 @@
+require 'coderay'

+

+begin

+	CodeRay::Encoders::YAML

+rescue

+	puts 'CodeRay::Encoders::YAML is not defined; you must load it first.'

+end

+

+yaml_encoder = CodeRay::Encoders[:yaml] 

+print 'Now it is loaded: '

+p yaml_encoder

+puts 'See?'

+

+tokens_encoder = CodeRay.require_plugin 'CodeRay::Encoders/tokens'

+print 'Require is also possible: '

+p tokens_encoder

+puts 'See?'

+

+puts 'Now load some mapped encoders: stats and plain.'

+CodeRay.require_plugin 'CodeRay::Encoders/stats'

+CodeRay.require_plugin 'CodeRay::Encoders/plain'

+

+puts 'Require all Encoders:'

+CodeRay::Encoders.load_all

+p CodeRay::Encoders.plugin_hash.sort_by { |k,v| k.to_s }

diff --git a/test/samples/load_scanner.expected b/test/samples/load_scanner.expected
new file mode 100644
index 0000000..a2d200d
--- /dev/null
+++ b/test/samples/load_scanner.expected
@@ -0,0 +1,8 @@
+CodeRay::Encoders::Ruby is not defined; you must load it first.
+Now it is loaded: CodeRay::Scanners::Ruby
+See?
+Require is also possible: CodeRay::Scanners::C
+See?
+Now load some mapped scanners: cpp and plain.
+Require all Scanners:
+[[nil, :plain], [:c, CodeRay::Scanners::C], [:cpp, :c], [:delphi, CodeRay::Scanners::Delphi], [:html, CodeRay::Scanners::HTML], [:irb, :ruby], [:nitro, :nitro_xhtml], [:nitro_xhtml, CodeRay::Scanners::NitroXHTML], [:pascal, :delphi], [:plain, CodeRay::Scanners::Plaintext], [:plaintext, CodeRay::Scanners::Plaintext], [:rhtml, CodeRay::Scanners::RHTML], [:ruby, CodeRay::Scanners::Ruby], [:xhtml, :nitro_xhtml], [:xml, :html]]
diff --git a/test/samples/load_scanner.rb b/test/samples/load_scanner.rb
new file mode 100644
index 0000000..23be8a2
--- /dev/null
+++ b/test/samples/load_scanner.rb
@@ -0,0 +1,25 @@
+require 'coderay'

+

+begin

+	CodeRay::Scanners::Ruby

+rescue

+	puts 'CodeRay::Encoders::Ruby is not defined; you must load it first.'

+end

+

+ruby_scanner = CodeRay::Scanners[:ruby]

+print 'Now it is loaded: '

+p ruby_scanner

+puts 'See?'

+

+c_scanner = CodeRay.require_plugin 'CodeRay::Scanners/c'

+print 'Require is also possible: '

+p c_scanner

+puts 'See?'

+

+puts 'Now load some mapped scanners: cpp and plain.'

+CodeRay.require_plugin 'CodeRay::Scanners/cpp'

+CodeRay.require_plugin 'CodeRay::Scanners/plain'

+

+puts 'Require all Scanners:'

+CodeRay::Scanners.load_all

+p CodeRay::Scanners.plugin_hash.sort_by { |k,v| k.to_s }

diff --git a/test/samples/more.expected b/test/samples/more.expected
new file mode 100644
index 0000000..196904d
--- /dev/null
+++ b/test/samples/more.expected
@@ -0,0 +1,2 @@
+Input: 4983B, Output: 23484B
+Take a look with your browser.
diff --git a/test/samples/more.rb b/test/samples/more.rb
new file mode 100644
index 0000000..0db7ba4
--- /dev/null
+++ b/test/samples/more.rb
@@ -0,0 +1,205 @@
+require 'coderay'
+
+c, ruby = DATA.read.split(/^---$/)
+DATA.rewind
+me = DATA.read[/.*^__END__$/m]
+$input = c + ruby + me
+
+require 'benchmark'
+time = Benchmark.realtime do
+
+	# here CodeRay comes to play
+	hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div)
+	c = hl.highlight c, :c
+	ruby = hl.highlight ruby, :ruby
+	me = hl.highlight me, :ruby
+
+	body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code|
+		"<h1>#{title}</h1>\n#{code}"
+	end.join
+	body = hl.class::Output.new(body, hl.css, :div).page!
+
+	# CodeRay also provides a simple page generator
+	$output = body #hl.class.wrap_in_page body
+end
+
+File.open('test.html', 'w') do |f|
+	f.write $output
+end
+puts 'Input: %dB, Output: %dB' % [$input.size, $output.size]
+#puts 'Created "test.html" in %0.3f seconds (%d KB/s).' % [time, $input.size / 1024.0 / time]
+puts 'Take a look with your browser.'
+
+__END__
+/**********************************************************************
+
+  version.c -
+
+  $Author: nobu $
+  $Date: 2004/03/25 12:01:40 $
+  created at: Thu Sep 30 20:08:01 JST 1993
+
+  Copyright (C) 1993-2003 Yukihiro Matsumoto
+
+**********************************************************************/
+
+#include "ruby.h"
+#include "version.h"
+#include <stdio.h>
+
+const char ruby_version[] = RUBY_VERSION;
+const char ruby_release_date[] = RUBY_RELEASE_DATE;
+const char ruby_platform[] = RUBY_PLATFORM;
+
+void
+Init_version()
+{
+    VALUE v = rb_obj_freeze(rb_str_new2(ruby_version));
+    VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date));
+    VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform));
+
+    rb_define_global_const("RUBY_VERSION", v);
+    rb_define_global_const("RUBY_RELEASE_DATE", d);
+    rb_define_global_const("RUBY_PLATFORM", p);
+}
+
+void
+ruby_show_version()
+{
+    printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM);
+}
+
+void
+ruby_show_copyright()
+{
+    printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR);
+    exit(0);
+}
+---
+#
+# = ostruct.rb: OpenStruct implementation
+#
+# Author:: Yukihiro Matsumoto
+# Documentation:: Gavin Sinclair
+#
+# OpenStruct allows the creation of data objects with arbitrary attributes.
+# See OpenStruct for an example.
+#
+
+#
+# OpenStruct allows you to create data objects and set arbitrary attributes.
+# For example:
+#
+#   require 'ostruct' 
+#
+#   record = OpenStruct.new
+#   record.name    = "John Smith"
+#   record.age     = 70
+#   record.pension = 300
+#   
+#   puts record.name     # -> "John Smith"
+#   puts record.address  # -> nil
+#
+# It is like a hash with a different way to access the data.  In fact, it is
+# implemented with a hash, and you can initialize it with one.
+#
+#   hash = { "country" => "Australia", :population => 20_000_000 }
+#   data = OpenStruct.new(hash)
+#
+#   p data        # -> <OpenStruct country="Australia" population=20000000>
+#
+class OpenStruct
+  #
+  # Create a new OpenStruct object.  The optional +hash+, if given, will
+  # generate attributes and values.  For example.
+  #
+  #   require 'ostruct'
+  #   hash = { "country" => "Australia", :population => 20_000_000 }
+  #   data = OpenStruct.new(hash)
+  #
+  #   p data        # -> <OpenStruct country="Australia" population=20000000>
+  #
+  # By default, the resulting OpenStruct object will have no attributes. 
+  #
+  def initialize(hash=nil)
+    @table = {}
+    if hash
+      for k,v in hash
+	@table[k.to_sym] = v
+        new_ostruct_member(k)
+      end
+    end
+  end
+
+  # Duplicate an OpenStruct object members. 
+  def initialize_copy(orig)
+    super
+    @table = @table.dup
+  end
+
+  def marshal_dump
+    @table
+  end
+  def marshal_load(x)
+    @table = x
+    @table.each_key{|key| new_ostruct_member(key)}
+  end
+
+  def new_ostruct_member(name)
+    unless self.respond_to?(name)
+      self.instance_eval %{
+        def #{name}; @table[:#{name}]; end
+        def #{name}=(x); @table[:#{name}] = x; end
+      }
+    end
+  end
+
+  def method_missing(mid, *args) # :nodoc:
+    mname = mid.id2name
+    len = args.length
+    if mname =~ /=$/
+      if len != 1
+	raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1)
+      end
+      if self.frozen?
+	raise TypeError, "can't modify frozen #{self.class}", caller(1)
+      end
+      mname.chop!
+      @table[mname.intern] = args[0]
+      self.new_ostruct_member(mname)
+    elsif len == 0
+      @table[mid]
+    else
+      raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1)
+    end
+  end
+
+  #
+  # Remove the named field from the object.
+  #
+  def delete_field(name)
+    @table.delete name.to_sym
+  end
+
+  #
+  # Returns a string containing a detailed summary of the keys and values.
+  #
+  def inspect
+    str = "<#{self.class}"
+    for k,v in @table
+      str << " #{k}=#{v.inspect}"
+    end
+    str << ">"
+  end
+
+  attr_reader :table # :nodoc:
+  protected :table
+
+  #
+  # Compare this object and +other+ for equality.
+  #
+  def ==(other)
+    return false unless(other.kind_of?(OpenStruct))
+    return @table == other.table
+  end
+end
diff --git a/test/samples/scanner.expected b/test/samples/scanner.expected
new file mode 100644
index 0000000..5015168
--- /dev/null
+++ b/test/samples/scanner.expected
@@ -0,0 +1,16 @@
+C Code: if (*p == '{') nest++;
+
+> print only operators:
+(*==)++;
+------------------------------
+
+Ruby Code: ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)
+
+> has a string?
+false
+
+> number of regexps?
+2
+
+> has a string?
+"ruby_code" (ident), "(" (operator), ":can" (symbol), "," (operator), " " (space), "BE" (constant), "," (operator), " " (space), "%r[" (delimiter), "q" (content), "[" (nesting_delimiter), "ui" (content), "]" (nesting_delimiter), "te " (content), "#{" (inline_delimiter), " " (space), "/" (delimiter), "comple" (content), "/" (delimiter), "x" (modifier), " " (space), "}" (inline_delimiter), "," (content), "]" (delimiter), " " (space), "=" (operator), ">" (operator), " " (space), "$-s" (global_variable), "," (operator), " " (space), "&" (operator), "?\xee" (integer), ")" (operator)
diff --git a/test/samples/scanner.rb b/test/samples/scanner.rb
new file mode 100644
index 0000000..6a0245e
--- /dev/null
+++ b/test/samples/scanner.rb
@@ -0,0 +1,36 @@
+require 'coderay'

+

+c_code = "if (*p == '{') nest++;"

+puts 'C Code: ' + c_code

+puts

+

+c_scanner = CodeRay::Scanners[:c].new c_code

+

+puts '> print only operators:'

+for text, kind in c_scanner

+	print text if kind == :operator

+end

+puts

+puts '-' * 30

+puts

+

+ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)!

+puts 'Ruby Code: ' + ruby_code

+puts

+

+ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code

+

+puts '> has a string?'

+puts ruby_scanner.

+	any? { |text, kind| kind == :string }

+puts

+

+puts '> number of regexps?'

+puts ruby_scanner.

+	select { |token| token == [:open, :regexp] }.size

+puts

+

+puts '> has a string?'

+puts ruby_scanner.

+	reject { |text, kind| not text.is_a? String }.

+	map { |text, kind| %("#{text}" (#{kind})) }.join(', ')

diff --git a/test/samples/server.rb b/test/samples/server.rb
new file mode 100644
index 0000000..ccdff32
--- /dev/null
+++ b/test/samples/server.rb
@@ -0,0 +1,110 @@
+# CodeRay dynamic highlighter

+

+unless ARGV.grep(/-[hv]|--(help|version)/).empty?

+	puts <<-USAGE

+CodeRay Server 0.5

+$Id: demo_server.rb 113 2006-03-15 23:24:37Z murphy $

+

+Usage:

+	1) Start this and your browser.

+	2) Go to http://localhost:2468/?<path to the file>

+	   and you should get the highlighted version.

+

+Parameters:

+	-d     Debug mode; reload CodeRay engine for every file.

+	       (prepare for MANY "already initialized" and "method redefined"

+	       messages - ingore it.)

+	

+	...    More to come.

+	USAGE

+	exit

+end

+

+require 'webrick'

+require 'pathname'

+

+class << File

+	alias dir? directory?

+end

+

+require 'erb'

+include ERB::Util

+def url_decode s

+	s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' }

+end

+

+class String

+	def to_link name = File.basename(self)

+		"<a href=\"?path=#{url_encode self}\">#{name}</a>"

+	end

+end

+

+require 'coderay'

+class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet

+

+	STYLE = 'style="font-family: sans-serif; color: navy;"'

+	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="Highlighted by CodeRay"/></p>'

+

+	def do_GET req, res

+		q = req.query_string || ''

+		args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) }

+		path = args.fetch 'path', '.'

+		

+		backlinks = '<p>current path: %s<br />' % html_escape(path) +

+			(Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' +

+			'.'.to_link('current') + '</p>'

+		

+		res.body = 

+			if File.dir? path

+				path = Pathname.new(path).cleanpath.to_s

+				dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p }

+

+				page = "<html><head></head><body #{STYLE}>"

+				page << backlinks

+				

+				page << '<dl>'

+				page << "<dt>Directories</dt>\n" + dirs.map do |p|

+					"<dd>#{p.to_link}</dd>\n"

+				end.join << "\n"

+				page << "<dt>Files</dt>\n" + files.map do |p|

+					"<dd>#{p.to_link}</dd>\n"

+				end.join << "\n"

+				page << "</dl>\n"

+				page << "#{BANNER}</body></html>"

+			

+			elsif File.exist? path

+				if $DEBUG

+					$".delete_if { |f| f =~ /coderay/ }

+					require 'coderay'

+				end

+				div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div, :hint => :info

+				div.replace <<-DIV

+	<div #{STYLE}>

+		#{backlinks}

+#{div}

+	</div>

+	#{BANNER}

+				DIV

+				div.page

+			end

+

+		res['Content-Type'] = 'text/html'

+	end

+end

+

+# This port is taken by "qip_msgd" - I don't know that. Do you?

+module CodeRay

+	PORT = 0xC0DE / 20

+end

+

+server = WEBrick::HTTPServer.new :Port => CodeRay::PORT

+

+server.mount '/', CodeRayServlet

+

+server.mount_proc '/version' do |req, res|

+	res.body = 'CodeRay::Version = ' + CodeRay::Version

+	res['Content-Type'] = "text/plain"

+end

+

+trap("INT") { server.shutdown }

+server.start

diff --git a/test/samples/simple.expected b/test/samples/simple.expected
new file mode 100644
index 0000000..b3d7875
--- /dev/null
+++ b/test/samples/simple.expected
@@ -0,0 +1 @@
+<span class="CodeRay">puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">'</span><span style="">Hello, world!</span><span style="color:#710">'</span></span></span>
diff --git a/test/samples/simple.rb b/test/samples/simple.rb
new file mode 100644
index 0000000..a3129b0
--- /dev/null
+++ b/test/samples/simple.rb
@@ -0,0 +1,10 @@
+

+# Load CodeRay

+# If this doesn't work, try ruby -rubygems.

+require 'coderay'

+

+# Generate HTML page for Ruby code.

+page = CodeRay.scan("puts 'Hello, world!'", :ruby).span

+

+# Print it

+puts page

diff --git a/test/samples/stream.rb b/test/samples/stream.rb
new file mode 100644
index 0000000..7ed8a22
--- /dev/null
+++ b/test/samples/stream.rb
@@ -0,0 +1,25 @@
+require 'coderay'

+

+code = File.read($0) * 500

+puts "Size of code: %d KB" % [code.size / 1024]

+

+puts "Use your system's memory tracker to see how much RAM this takes."

+print 'Press some key to continue...'; gets

+

+require 'benchmark'

+e = CodeRay.encoder(:div)

+for do_stream in [true, false]

+	puts "Scanning and encoding in %s mode, please wait..." %

+		[do_stream ? 'streaming' : 'normal']

+	output = ''

+	time = Benchmark.realtime do

+		if do_stream

+			output = e.encode_stream(code, :ruby)

+		else

+			output = e.encode_tokens(t = CodeRay.scan(code, :ruby))

+		end

+	end

+	puts 'Finished after %4.2f seconds.' % time

+	puts "Size of output: %d KB" % [output.size / 1024]

+	print 'Press some key to continue...'; gets

+end

diff --git a/test/samples/stream2.expected b/test/samples/stream2.expected
new file mode 100644
index 0000000..83aee98
--- /dev/null
+++ b/test/samples/stream2.expected
@@ -0,0 +1,2 @@
+kind: regexp, text size: 5.
+kind: space, text size: 1.
diff --git a/test/samples/stream2.rb b/test/samples/stream2.rb
new file mode 100644
index 0000000..d43cc9a
--- /dev/null
+++ b/test/samples/stream2.rb
@@ -0,0 +1,8 @@
+require 'coderay'

+

+token_stream = CodeRay::TokenStream.new do |kind, text|

+  puts 'kind: %s, text size: %d.' % [kind, text.size]

+end

+

+token_stream << [:regexp, '/\d+/'] << [:space, "\n"]

+#-> kind: rexpexp, text size: 5.

diff --git a/test/samples/suite.rb b/test/samples/suite.rb
new file mode 100644
index 0000000..cfe53c0
--- /dev/null
+++ b/test/samples/suite.rb
@@ -0,0 +1,86 @@
+mydir = File.dirname(__FILE__)
+$:.unshift mydir + '/../../lib/'
+
+$VERBOSE = true
+
+require 'test/unit'
+include Test::Unit
+
+class CodeRaySuite < TestCase
+
+  def self.dir &block
+    @dir ||= File.dirname(__FILE__)
+    if block
+      Dir.chdir @dir, &block
+    end
+    @dir
+  end
+
+  def dir &block
+    self.class.dir(&block)
+  end
+
+  def test_ALL
+    dir do
+      for input in Dir["*.rb"] - %w(server.rb stream.rb suite.rb)
+        next if input[/^load_/]
+        puts "[ testing #{input}... ]"
+        name = File.basename(input, ".rb")
+        output = name + '.expected'
+        code = File.open(input, 'rb') { |f| break f.read }
+        
+        result = `ruby -wI../../lib #{input}`
+        
+        diff = output.sub '.expected', '.diff'
+        File.delete diff if File.exist? diff
+        computed = output.sub '.expected', '.actual'
+        if File.exist? output
+          expected = File.read output
+          ok = expected == result
+          unless ok
+            File.open(computed, 'w') { |f| f.write result }
+            `diff #{output} #{computed} > #{diff}`
+            puts "Test failed; output written to #{diff}."
+          end
+          assert(ok, "Output error: #{computed} != #{output}")
+        else
+          File.open(output, 'w') do |f| f.write result end
+          puts "New test: #{output}"
+        end
+
+      end
+    end
+  end
+
+end
+
+require 'test/unit/testsuite'
+$suite = TestSuite.new 'CodeRay Demos Test'
+$suite << CodeRaySuite.suite
+
+def load_suite name
+  begin
+    require name + '/suite.rb'
+  rescue LoadError
+    $stderr.puts <<-ERR
+
+!! Folder #{File.split(__FILE__).first + '/' + name} not found
+
+    ERR
+    false
+  end
+end
+
+if subsuite = ARGV.find { |a| break $1 if a[/^([^-].*)/] }
+  load_suite(subsuite) or exit
+else
+  Dir[mydir + '/*/'].each { |suite| load_suite suite }
+end
+
+if ARGV.include? '-f'
+  require 'test/unit/ui/fox/testrunner'
+  UI::Fox::TestRunner.run $suite
+else
+  require 'test/unit/ui/console/testrunner'
+  UI::Console::TestRunner.run $suite
+end
diff --git a/test/samples/tokens.expected b/test/samples/tokens.expected
new file mode 100644
index 0000000..747904e
--- /dev/null
+++ b/test/samples/tokens.expected
@@ -0,0 +1 @@
+[["puts", :ident], [" ", :space], ["3", :integer], [" ", :space], ["+", :operator], [" ", :space], ["4", :integer], [",", :operator], [" ", :space], [:open, :string], ["'", :delimiter], ["3 + 4", :content], ["'", :delimiter], [:close, :string]]
diff --git a/test/samples/tokens.rb b/test/samples/tokens.rb
new file mode 100644
index 0000000..91b8abb
--- /dev/null
+++ b/test/samples/tokens.rb
@@ -0,0 +1,3 @@
+require 'coderay'

+

+p CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby)

diff --git a/test/unit/tokens.rb b/test/unit/tokens.rb
index 86dc632..73b0fd5 100644
--- a/test/unit/tokens.rb
+++ b/test/unit/tokens.rb
@@ -18,15 +18,6 @@ class TokensTest < Test::Unit::TestCase
     assert_equal tokens.count, 4
   end
   
-  def test_dump_undump
-    tokens = make_tokens
-    tokens2 = nil
-    assert_nothing_raised do
-      tokens2 = tokens.dump.undump
-    end
-    assert_equal tokens, tokens2
-  end
-  
   def test_to_s
     assert_equal 'string()', make_tokens.to_s
   end