Demos updated, rewritten, enhanced, tested.

Some code cleanups.

Bugs fixed, scanner and encoder improved:
count.rb: marked Streamable
html_css.rb: style for inline numbers
html.rb: changed options; :line_numbers_offset is now :line_number_start
html_output.rb: offset for inline numbers fixed
html.rb: token text no longer changed by gsub! while highlighting (this is even faster!)
text.rb, plugin.rb: reindented
ruby.rb: eleminated multiple assignments for speed

tokens.rb: reindented, Tokens#to_s added, #<< returns self

Plugin system: bugs fixed, error messages improved.

17 files changed, 264 insertions, 190 deletions

diff --git a/demo/demo_global_vars.rb b/demo/demo_global_vars.rb
index 2bacfe5..8066d67 100644
--- a/demo/demo_global_vars.rb
+++ b/demo/demo_global_vars.rb
@@ -5,11 +5,9 @@ text = $bla.test(...)
 CODE

 

 require 'coderay'

-require 'erb'

-include ERB::Util

 

 tokens = CodeRay.scan code, :ruby

-tokens.each_text_token { |text, kind| text.replace h(text) }

-tokens.each(:global_variable) { |text, kind| text.replace '<span class="glob-var">%s</span>' % text }

+tokens.each_text_token { |text, kind| text.upcase! }

+tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text }

 

-puts tokens.text

+print tokens

diff --git a/demo/demo_html2.rb b/demo/demo_html2.rb
index 0982ad8..7ce9d60 100644
--- a/demo/demo_html2.rb
+++ b/demo/demo_html2.rb
@@ -1,4 +1,11 @@
 require 'coderay'
-require 'coderay/encoders/html'
 
-puts CodeRay["puts CodeRay['...', :ruby]", :ruby].div
+# scan this file
+tokens = CodeRay.scan(File.read($0) * 1, :ruby)
+
+# output it with two styles of line numbers
+out = tokens.div(:line_numbers => :table)
+out << '<hr />'
+out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
+
+puts out.page
diff --git a/demo/demo_load_encoder.rb b/demo/demo_load_encoder.rb
index 3c85463..4861ee7 100644
--- a/demo/demo_load_encoder.rb
+++ b/demo/demo_load_encoder.rb
@@ -7,7 +7,11 @@ rescue
 end

 

 yaml_encoder = CodeRay::Encoders[:yaml] 

-puts 'Now it is loaded.'

+print 'Now it is loaded: '

+p yaml_encoder

+puts 'See?'

 

-p yaml_encoder == CodeRay::Encoders::YAML  #-> true

+tokens_encoder = require_plugin 'CodeRay::Encoders/tokens'

+print 'Require is also possible: '

+p tokens_encoder

 puts 'See?'

diff --git a/demo/demo_more.rb b/demo/demo_more.rb
index 7ebf5c3..5056842 100644
--- a/demo/demo_more.rb
+++ b/demo/demo_more.rb
@@ -1,13 +1,11 @@
-require 'rubygems'
-$: << '..'
 require 'coderay'
-require 'benchmark'
 
 c, ruby = DATA.read.split(/^---$/)
 DATA.rewind
 me = DATA.read[/.*^__END__$/m]
 $input = c + ruby + me
 
+require 'benchmark'
 time = Benchmark.realtime do
 
 	# here CodeRay comes to play
diff --git a/demo/demo_scanner.rb b/demo/demo_scanner.rb
index a250f91..6a0245e 100644
--- a/demo/demo_scanner.rb
+++ b/demo/demo_scanner.rb
@@ -1,12 +1,36 @@
 require 'coderay'

-c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"

+

+c_code = "if (*p == '{') nest++;"

+puts 'C Code: ' + c_code

+puts

+

+c_scanner = CodeRay::Scanners[:c].new c_code

+

+puts '> print only operators:'

 for text, kind in c_scanner

 	print text if kind == :operator

 end

 puts

+puts '-' * 30

+puts

 

-ruby_scanner = CodeRay::Scanners[:ruby].new %q<c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;">

+ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)!

+puts 'Ruby Code: ' + ruby_code

+puts

+

+ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code

+

+puts '> has a string?'

+puts ruby_scanner.

+	any? { |text, kind| kind == :string }

+puts

+

+puts '> number of regexps?'

+puts ruby_scanner.

+	select { |token| token == [:open, :regexp] }.size

+puts

 

-puts ruby_scanner.any? { |text, kind| kind == :string and text == :open}

-puts ruby_scanner.find { |text, kind| kind == :regexp }

-puts ruby_scanner.map { |text, kind| text if kind != :space }.compact.join(' ')

+puts '> has a string?'

+puts ruby_scanner.

+	reject { |text, kind| not text.is_a? String }.

+	map { |text, kind| %("#{text}" (#{kind})) }.join(', ')

diff --git a/demo/demo_server.rb b/demo/demo_server.rb
index 44485f0..cc3db2b 100644
--- a/demo/demo_server.rb
+++ b/demo/demo_server.rb
@@ -74,9 +74,9 @@ class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 	end

 end

 

-# 0xCODE = 49374 

+# this is taken by "qip_msgd" - I don't know that.

 module CodeRay

-	PORT = 0xC0DE

+	PORT = 0xC0DE / 20

 end

 

 server = WEBrick::HTTPServer.new :Port => CodeRay::PORT

diff --git a/demo/demo_stream.rb b/demo/demo_stream.rb
index b1d8560..7ed8a22 100644
--- a/demo/demo_stream.rb
+++ b/demo/demo_stream.rb
@@ -1,8 +1,25 @@
-$: << '..'

 require 'coderay'

 

-e = CodeRay.encoder(:html)

-t = e.encode_stream('a LOT of :code', :ruby)

+code = File.read($0) * 500

+puts "Size of code: %d KB" % [code.size / 1024]

 

-puts t

-p t.class

+puts "Use your system's memory tracker to see how much RAM this takes."

+print 'Press some key to continue...'; gets

+

+require 'benchmark'

+e = CodeRay.encoder(:div)

+for do_stream in [true, false]

+	puts "Scanning and encoding in %s mode, please wait..." %

+		[do_stream ? 'streaming' : 'normal']

+	output = ''

+	time = Benchmark.realtime do

+		if do_stream

+			output = e.encode_stream(code, :ruby)

+		else

+			output = e.encode_tokens(t = CodeRay.scan(code, :ruby))

+		end

+	end

+	puts 'Finished after %4.2f seconds.' % time

+	puts "Size of output: %d KB" % [output.size / 1024]

+	print 'Press some key to continue...'; gets

+end

diff --git a/demo/demo_stream2.rb b/demo/demo_stream2.rb
index 8a6bec7..d43cc9a 100644
--- a/demo/demo_stream2.rb
+++ b/demo/demo_stream2.rb
@@ -4,5 +4,5 @@ token_stream = CodeRay::TokenStream.new do |kind, text|
   puts 'kind: %s, text size: %d.' % [kind, text.size]

 end

 

-token_stream << [:regexp, '/\d+/']

+token_stream << [:regexp, '/\d+/'] << [:space, "\n"]

 #-> kind: rexpexp, text size: 5.

diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb
index 80aec57..d49e0bc 100644
--- a/lib/coderay/encoders/count.rb
+++ b/lib/coderay/encoders/count.rb
@@ -3,6 +3,7 @@ module Encoders
 

 	class Count < Encoder

 

+		include Streamable

 		register_for :count

 

 	protected		

diff --git a/lib/coderay/encoders/helpers/html_css.rb b/lib/coderay/encoders/helpers/html_css.rb
index 5c39527..a3c50eb 100644
--- a/lib/coderay/encoders/helpers/html_css.rb
+++ b/lib/coderay/encoders/helpers/html_css.rb
@@ -46,6 +46,8 @@ module CodeRay module Encoders
 	border: 1px solid silver;

 	font-family: 'Courier New', 'Terminal', monospace;

 	color: black;

+	width: 100%;

+	padding: 2px;

 }

 .CodeRay pre { margin: 0px; }

 

@@ -56,12 +58,14 @@ span.CodeRay { white-space: pre; border: 0; }
 table.CodeRay { border-collapse: collapse; }

 table.CodeRay td { padding: 2px 4px; vertical-align: top; }

 

-.CodeRay .line_numbers {

+.CodeRay .line_numbers, .CodeRay .no {

 	background-color: #def;

 	color: gray;

 	text-align: right;

 }

 .CodeRay .line_numbers tt { font-weight: bold; }

+.CodeRay .no { padding: 0px 4px; }

+.CodeRay .code { width: 100%; }

 

 .CodeRay .code {

 }

diff --git a/lib/coderay/encoders/helpers/html_output.rb b/lib/coderay/encoders/helpers/html_output.rb
index c873896..f6a2b11 100644
--- a/lib/coderay/encoders/helpers/html_output.rb
+++ b/lib/coderay/encoders/helpers/html_output.rb
@@ -100,9 +100,9 @@ module CodeRay
 			def numerize! mode = :table, options = {}

 				return self unless mode

 

-				offset = options.fetch :line_numbers_offset, DEFAULT_OPTIONS[:line_numbers_offset]

-				unless offset.is_a? Integer

-					raise ArgumentError, "Invalid value %p for :offset; Integer expected." % offset

+				start = options.fetch :line_number_start, DEFAULT_OPTIONS[:line_number_start]

+				unless start.is_a? Integer

+					raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start

 				end

 				

 				unless NUMERIZABLE_WRAPPINGS.include? options[:wrap]

@@ -130,11 +130,11 @@ module CodeRay
 

 				case mode				

 				when :inline

-					max_width = line_count.to_s.size

-					line = offset - 1

+					max_width = (start + line_count).to_s.size

+					line = start

 					gsub!(/^/) do

-						line += 1

 						line_number = bolding.call line

+						line += 1

 						"<span class=\"no\">#{ line_number.rjust(max_width) }</span>  "

 					end

 					wrap! :div

@@ -144,7 +144,7 @@ module CodeRay
 					# Because even monospace fonts seem to have different heights when bold, 

 					# I make the newline bold, both in the code and the line numbers.

 					# FIXME Still not working perfect for Mr. Internet Exploder

-					line_numbers = (offset ... offset + line_count).to_a.map(&bolding).join("\n")

+					line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")

 					line_numbers << "\n"  # also for Mr. MS Internet Exploder :-/

 					line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }

 					

@@ -196,19 +196,22 @@ module CodeRay
 

 #-- don't include the templates in docu

 			

-			SPAN = `<span class="code"><%CONTENT%></span>`

+			SPAN = `<span class="CodeRay"><%CONTENT%></span>`

 

-			DIV, DIV_TABLE, PAGE =

-				<<-`DIV`, <<-`DIV_TABLE`, <<-`PAGE`

+			DIV = <<-`DIV`

 <div class="CodeRay">

 	<div class="code"><pre><%CONTENT%></pre></div>	

 </div>

 			DIV

+

+			DIV_TABLE = <<-`DIV_TABLE`

 <table class="CodeRay"> <tr>

 	<td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>

 	<td class="code"><pre title="double click to expand" ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>

 </tr> </table>

 			DIV_TABLE

+

+			PAGE = <<-`PAGE`

 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"

 	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 58fdd86..15120a2 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -17,7 +17,7 @@ module Encoders
 			:wrap => :page,

 

 			:line_numbers => nil,

-			:line_numbers_offset => 1,

+			:line_number_start => 1,

 			:bold_every => 10,

 		}

 		NUMERIZABLE_WRAPPINGS = [:div, :page]

@@ -133,8 +133,9 @@ module Encoders
 

 		def token text, type

 			if text.is_a? String

-				# be careful when streaming: text is changed!

-				text.gsub!(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }

+				if text =~ /#{HTML_ESCAPE_PATTERN}/o

+					text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }

+				end

 				@opened[0] = type

 				style = @css_style[@opened]

 				if style

diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index 4f0a754..02f76cb 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -1,33 +1,33 @@
 module CodeRay

-	module Encoders

+module Encoders

 

-		class Text < Encoder

+	class Text < Encoder

 

-			include Streamable

-			register_for :text

+		include Streamable

+		register_for :text

 

-			FILE_EXTENSION = 'txt'

+		FILE_EXTENSION = 'txt'

 

-			DEFAULT_OPTIONS = {

-				:separator => ''

-			}

+		DEFAULT_OPTIONS = {

+			:separator => ''

+		}

 

-			protected

-			def setup options

-				super

-				@sep = options[:separator]

-			end

-

-			def token text, kind

-				return unless text.respond_to :to_str

-				@out << text + @sep

-			end

+	protected

+		def setup options

+			super

+			@sep = options[:separator]

+		end

 

-			def finish options

-				@out.chomp @sep

-			end

+		def token text, kind

+			return unless text.respond_to? :to_str

+			@out << text + @sep

+		end

 

+		def finish options

+			@out.chomp @sep

 		end

 

 	end

+

+end

 end

diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index b0bb49e..aacde99 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -1,4 +1,3 @@
-

 # = PluginHost

 #

 # $Id$

@@ -28,6 +27,7 @@ module PluginHost
 	# * a file could not be found

 	# * the requested Encoder is not registered

 	PluginNotFound = Class.new Exception

+	HostNotFound = Class.new Exception

 

 	PLUGIN_HOSTS = []

 	PLUGIN_HOSTS_BY_ID = {}  # dummy hash

@@ -45,11 +45,11 @@ module PluginHost
 		# Find the PluginHost for host_id.

 		def host_by_id host_id

 			unless PLUGIN_HOSTS_BY_ID.default_proc

-				ph = Hash.new do |h, _host_id|

+				ph = Hash.new do |h, a_host_id|

 					for host in PLUGIN_HOSTS

 						h[host.host_id] = host

 					end

-					h.fetch _host_id, nil

+					h.fetch a_host_id, nil

 				end				

 				PLUGIN_HOSTS_BY_ID.replace ph

 			end

@@ -58,138 +58,134 @@ module PluginHost
 

 	end

 	

-		def plugin_host_id host_id

-			if host_id.is_a? String

-				raise ArgumentError,

-					"String or Symbol expected, but #{lang.class} given."

-			end

-			

+	def plugin_host_id host_id

+		if host_id.is_a? String

+			raise ArgumentError,

+				"String or Symbol expected, but #{lang.class} given."

 		end

+	end

 

-		# The path where the plugins can be found.

-		def plugin_path *args

-			unless args.empty?

-				@plugin_path = File.join(*args)

-			end

-			@plugin_path

+	# The path where the plugins can be found.

+	def plugin_path *args

+		unless args.empty?

+			@plugin_path = File.join(*args)

 		end

+		@plugin_path

+	end

 

-		# The host's ID.

-		#

-		# If PLUGIN_HOST_ID is not set, it is simply the class name.

-		def host_id

-			if self.const_defined? :PLUGIN_HOST_ID

-				self::PLUGIN_HOST_ID

-			else

-				name

-			end

+	# The host's ID.

+	#

+	# If PLUGIN_HOST_ID is not set, it is simply the class name.

+	def host_id

+		if self.const_defined? :PLUGIN_HOST_ID

+			self::PLUGIN_HOST_ID

+		else

+			name

 		end

+	end

 

-		def create_plugin_hash

-			@plugin_hash =

-				Hash.new do |h, plugin_id|

-					id = validate_id(plugin_id)

-					path = path_to id

-					begin

-						puts 'Loading plugin: ' + path if $DEBUG

-						require path

-					rescue LoadError

-						raise PluginNotFound, "#{path} not found."

-					else

-						# Plugin should have registered by now

-						unless h.has_key? id

-							raise PluginNotFound,

-								"No #{self.name} plugin for #{id} found in #{path}."

-						end

+	def create_plugin_hash

+		@plugin_hash =

+			Hash.new do |h, plugin_id|

+				id = validate_id(plugin_id)

+				path = path_to id

+				begin

+					$stderr.puts 'Loading plugin: ' + path if $DEBUG

+					require path

+				rescue LoadError

+					raise PluginNotFound, "Plugin #{id.inspect} not found."

+				else

+					# Plugin should have registered by now

+					unless h.has_key? id

+						raise PluginNotFound,

+							"No #{self.name} plugin for #{id.inspect} found in #{path}."

 					end

-					h[id]

 				end

-		end

+				h[id]

+			end

+	end

 

-		def plugin_hash

-			@plugin_hash ||= create_plugin_hash

-		end

+	def plugin_hash

+		@plugin_hash ||= create_plugin_hash

+	end

 

 

-		# Every plugin must register itself for one or more

-		# +ids+ by calling register_for, which calls this method.

-		#

-		# See Plugin#register_for.

-		def register plugin, *ids

-			for id in ids

-				unless id.is_a? Symbol

-					raise ArgumentError,

-						"id must be a Symbol, but it was a #{id.class}" 

-				end

-				plugin_hash[validate_id(id)] = plugin

+	# Every plugin must register itself for one or more

+	# +ids+ by calling register_for, which calls this method.

+	#

+	# See Plugin#register_for.

+	def register plugin, *ids

+		for id in ids

+			unless id.is_a? Symbol

+				raise ArgumentError,

+					"id must be a Symbol, but it was a #{id.class}" 

 			end

+			plugin_hash[validate_id(id)] = plugin

 		end

+	end

 

 

-		# Returns an array of all .rb files in the plugin path.

-		# 

-		# The extension .rb is not included.

-		def all_plugin_names

-			Dir[path_to('*')].map do |file|

-				File.basename file, '.rb'

-			end

+	# Returns an array of all .rb files in the plugin path.

+	# 

+	# The extension .rb is not included.

+	def all_plugin_names

+		Dir[path_to('*')].map do |file|

+			File.basename file, '.rb'

 		end

+	end

 

-		# Loads all plugins using all_plugin_names and load.

-		def load_all

-			for plugin in all_plugin_names

-				load_plugin plugin

-			end

+	# Loads all plugins using all_plugin_names and load.

+	def load_all

+		for plugin in all_plugin_names

+			load_plugin plugin

 		end

+	end

 

 

-		# Returns the Plugin for +id+.

-		# 

-		# Example:

-		#  yaml_plugin = MyPluginHost[:yaml]

-		def [] id, *args, &blk

-			plugin_hash.[] validate_id(id), *args, &blk

-		end

+	# Returns the Plugin for +id+.

+	# 

+	# Example:

+	#  yaml_plugin = MyPluginHost[:yaml]

+	def [] id, *args, &blk

+		plugin_hash.[] validate_id(id), *args, &blk

+	end

 

-		# Alias for +[]+.

-		alias load_plugin []

+	# Alias for +[]+.

+	alias load_plugin []

 

-		# Returns the Plugin for +id+.

-		# Use it like Hash#fetch.

-		# 

-		# Example:

-		#  yaml_plugin = MyPluginHost[:yaml, :default]

-		def fetch id, *args, &blk

-			plugin_hash.fetch validate_id(id), *args, &blk

-		end

+	# Returns the Plugin for +id+.

+	# Use it like Hash#fetch.

+	# 

+	# Example:

+	#  yaml_plugin = MyPluginHost[:yaml, :default]

+	def fetch id, *args, &blk

+		plugin_hash.fetch validate_id(id), *args, &blk

+	end

 

-		# Returns the path to the encoder for format.

-		def path_to plugin_id

-			File.join plugin_path, "#{plugin_id}.rb"

-		end

+	# Returns the path to the encoder for format.

+	def path_to plugin_id

+		File.join plugin_path, "#{plugin_id}.rb"

+	end

 

-		# Converts +id+ to a downcase Symbol if it is a String,

-		# or returns +id+ if it already is a Symbol.

-		#

-		# Raises +ArgumentError+ for all other objects, or if the

-		# given String includes non-alphanumeric characters (\W).

-		def validate_id id

-			if id.is_a? Symbol

-				id

-			elsif id.is_a? String

-				if id[/\w+/] == id

-					id.downcase.to_sym

-				else

-					raise ArgumentError, "Invalid id: '#{id}' given."

-				end

+	# Converts +id+ to a downcase Symbol if it is a String,

+	# or returns +id+ if it already is a Symbol.

+	#

+	# Raises +ArgumentError+ for all other objects, or if the

+	# given String includes non-alphanumeric characters (\W).

+	def validate_id id

+		if id.is_a? Symbol

+			id

+		elsif id.is_a? String

+			if id[/\w+/] == id

+				id.downcase.to_sym

 			else

-				raise ArgumentError,

-					"String or Symbol expected, but #{id.class} given."

+				raise ArgumentError, "Invalid id: '#{id}' given."

 			end

+		else

+			raise ArgumentError,

+				"String or Symbol expected, but #{id.class} given."

 		end

-

-	#end

-

+	end

 

 end

 

@@ -239,8 +235,11 @@ end
 # 

 # Returns the loaded plugin.

 def require_plugin path

-	host, plugin_id = path.split '/', 2

-	PluginHost.host_by_id(host).load_plugin plugin_id

+	host_id, plugin_id = path.split '/', 2

+	host = PluginHost.host_by_id(host_id)

+	raise PluginHost::HostNotFound,

+		"No host for #{host_id.inspect} found." unless host

+	host.load_plugin plugin_id

 end

 

 

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index 433726b..72e59bd 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -72,7 +72,8 @@ module CodeRay module Scanners
 							tokens << [modifiers, :modifier] unless modifiers.empty?

 							if parse_regexp

 								extended = modifiers.index ?x

-								tokens, regexp = saved_tokens, tokens

+								tokens = saved_tokens

+								regexp = tokens

 								for text, type in regexp

 									if text.is_a? String

 										case type

@@ -125,7 +126,8 @@ module CodeRay module Scanners
 						when ?{

 							states.push [state, depth, heredocs]

 							fancy_allowed = regexp_allowed = true

-							state, depth = :initial, 1

+							state = :initial

+							depth = 1

 							tokens << [match + getch, :escape]

 						when ?$, ?@

 							tokens << [match, :escape]

@@ -192,7 +194,7 @@ module CodeRay module Scanners
 								when '}'

 									depth -= 1

 									if depth == 0

-										state, depth, heredocs = *states.pop

+										state, depth, heredocs = states.pop

 										type = :escape

 									end

 								end

@@ -214,7 +216,7 @@ module CodeRay module Scanners
 						elsif match = scan(/ ['"] /mx)

 							tokens << [:open, :string]

 							type = :delimiter

-							state = StringState.new :string, match != '\'', match.dup  # important for streaming

+							state = StringState.new :string, match != '\'', match  # important for streaming

 							

 						elsif match = scan(/#{INSTANCE_VARIABLE}/o)

 							type = :instance_variable

@@ -223,13 +225,14 @@ module CodeRay module Scanners
 							tokens << [:open, :regexp]

 							type = :delimiter

 							interpreted = true

-							state = StringState.new :regexp, interpreted, match.dup

+							state = StringState.new :regexp, interpreted, match

 							if parse_regexp

-								tokens, saved_tokens = [], tokens

+								tokens = []

+								saved_tokens = tokens

 							end

 							

 						elsif match = scan(/#{NUMERIC}/o)

-							type = if match[/#{FLOAT}/o] then :float else :integer end							

+							type = if match[/#{FLOAT}/o] then :float else :integer end

 

 						elsif fancy_allowed and match = scan(/#{SYMBOL}/o)

 							case match[1]

@@ -240,7 +243,8 @@ module CodeRay module Scanners
 							type = :symbol

 							

 						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)

-							indented, quote = self[1] == '-', self[3]

+							indented = self[1] == '-'

+							quote = self[3]

 							delim = self[quote ? 4 : 2]

 							type = QUOTE_TO_TYPE[quote]

 							tokens << [:open, type]

@@ -271,7 +275,7 @@ module CodeRay module Scanners
 							else

 								tokens << [:open, :shell]

 								type = :delimiter

-								state = StringState.new :shell, true, '`'

+								state = StringState.new :shell, true, match

 							end

 							

 						elsif match = scan(/#{GLOBAL_VARIABLE}/o)

diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 988008e..9318844 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -1,5 +1,7 @@
 module CodeRay

 

+	# = Tokens

+	#

 	# The Tokens class represents a list of tokens returnd from

 	# a Scanner.

 	#

@@ -137,6 +139,15 @@ module CodeRay
 			encoder.encode_tokens self, options

 		end

 

+

+		# Turn into a string using Encoders::Text.

+		#

+		# +options+ are passed to the encoder if given.

+		def to_s options = {}

+			encode :text, options

+		end

+

+

 		# Redirects unknown methods to encoder calls.

 		#

 		# For example, if you call +tokens.html+, the HTML encoder

@@ -152,13 +163,13 @@ module CodeRay
 		# in most Encoders.  It basically makes the output smaller.

 		#

 		# Combined with dump, it saves space for the cost

-    # calculating time.

-    #

-    # If the scanner is written carefully, this is not required - 

-    # for example, consecutive //-comment lines can already be 

-    # joined in one token by the Scanner.

+		# calculating time.

+		#

+		# If the scanner is written carefully, this is not required - 

+		# for example, consecutive //-comment lines can already be 

+		# joined in one token by the Scanner.

 		def optimize

-      print ' Tokens#optimize: before: %d - ' % size if $DEBUG

+			print ' Tokens#optimize: before: %d - ' % size if $DEBUG

 			last_kind = last_text = nil

 			new = self.class.new

 			each do |text, kind|

@@ -177,8 +188,8 @@ module CodeRay
 				end

 			end

 			new << [last_text, last_kind] if last_kind

-      print 'after: %d (%d saved = %2.0f%%)' % 

-        [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG

+			print 'after: %d (%d saved = %2.0f%%)' % 

+				[new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG

 			new

 		end

 

@@ -241,6 +252,8 @@ module CodeRay
 	end

 

 

+	# = TokenStream

+	#

 	# The TokenStream class is a fake Array without elements.

 	# 

 	# It redirects the method << to a block given at creation.

@@ -283,9 +296,12 @@ module CodeRay
 		end

 

 		# Calls +block+ with +token+ and increments size.

+		#

+		# Returns self.

 		def << token

 			@callback.call token

 			@size += 1

+			self

 		end

 

 		# This method is not implemented due to speed reasons. Use Tokens.

@@ -306,5 +322,3 @@ module CodeRay
 	end

 

 end

-

-# vim:sw=2:ts=2:et:tw=78

diff --git a/rake_helpers/html_coderay_generator.rb b/rake_helpers/html_coderay_generator.rb
index d31419a..b460974 100644
--- a/rake_helpers/html_coderay_generator.rb
+++ b/rake_helpers/html_coderay_generator.rb
@@ -1086,7 +1086,7 @@ module Generators
       code = tokens.map { |t| t.text }.join
       options = {
       	:css => :class,
-      	:line_numbers_offset => code[/\A.*?, line (\d+)/,1].to_i - 1,
+      	:line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1,
       	:bold_every => :no_bolding,
       }
       options[:line_numbers] = nil unless Options.instance.include_line_numbers