Demos updated, rewritten, enhanced, tested.

Some code cleanups.

Bugs fixed, scanner and encoder improved:
count.rb: marked Streamable
html_css.rb: style for inline numbers
html.rb: changed options; :line_numbers_offset is now :line_number_start
html_output.rb: offset for inline numbers fixed
html.rb: token text no longer changed by gsub! while highlighting (this is even faster!)
text.rb, plugin.rb: reindented
ruby.rb: eleminated multiple assignments for speed

tokens.rb: reindented, Tokens#to_s added, #<< returns self

Plugin system: bugs fixed, error messages improved.

8 files changed, 190 insertions, 164 deletions

diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb
index 80aec57..d49e0bc 100644
--- a/lib/coderay/encoders/count.rb
+++ b/lib/coderay/encoders/count.rb
@@ -3,6 +3,7 @@ module Encoders
 

 	class Count < Encoder

 

+		include Streamable

 		register_for :count

 

 	protected		

diff --git a/lib/coderay/encoders/helpers/html_css.rb b/lib/coderay/encoders/helpers/html_css.rb
index 5c39527..a3c50eb 100644
--- a/lib/coderay/encoders/helpers/html_css.rb
+++ b/lib/coderay/encoders/helpers/html_css.rb
@@ -46,6 +46,8 @@ module CodeRay module Encoders
 	border: 1px solid silver;

 	font-family: 'Courier New', 'Terminal', monospace;

 	color: black;

+	width: 100%;

+	padding: 2px;

 }

 .CodeRay pre { margin: 0px; }

 

@@ -56,12 +58,14 @@ span.CodeRay { white-space: pre; border: 0; }
 table.CodeRay { border-collapse: collapse; }

 table.CodeRay td { padding: 2px 4px; vertical-align: top; }

 

-.CodeRay .line_numbers {

+.CodeRay .line_numbers, .CodeRay .no {

 	background-color: #def;

 	color: gray;

 	text-align: right;

 }

 .CodeRay .line_numbers tt { font-weight: bold; }

+.CodeRay .no { padding: 0px 4px; }

+.CodeRay .code { width: 100%; }

 

 .CodeRay .code {

 }

diff --git a/lib/coderay/encoders/helpers/html_output.rb b/lib/coderay/encoders/helpers/html_output.rb
index c873896..f6a2b11 100644
--- a/lib/coderay/encoders/helpers/html_output.rb
+++ b/lib/coderay/encoders/helpers/html_output.rb
@@ -100,9 +100,9 @@ module CodeRay
 			def numerize! mode = :table, options = {}

 				return self unless mode

 

-				offset = options.fetch :line_numbers_offset, DEFAULT_OPTIONS[:line_numbers_offset]

-				unless offset.is_a? Integer

-					raise ArgumentError, "Invalid value %p for :offset; Integer expected." % offset

+				start = options.fetch :line_number_start, DEFAULT_OPTIONS[:line_number_start]

+				unless start.is_a? Integer

+					raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start

 				end

 				

 				unless NUMERIZABLE_WRAPPINGS.include? options[:wrap]

@@ -130,11 +130,11 @@ module CodeRay
 

 				case mode				

 				when :inline

-					max_width = line_count.to_s.size

-					line = offset - 1

+					max_width = (start + line_count).to_s.size

+					line = start

 					gsub!(/^/) do

-						line += 1

 						line_number = bolding.call line

+						line += 1

 						"<span class=\"no\">#{ line_number.rjust(max_width) }</span>  "

 					end

 					wrap! :div

@@ -144,7 +144,7 @@ module CodeRay
 					# Because even monospace fonts seem to have different heights when bold, 

 					# I make the newline bold, both in the code and the line numbers.

 					# FIXME Still not working perfect for Mr. Internet Exploder

-					line_numbers = (offset ... offset + line_count).to_a.map(&bolding).join("\n")

+					line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")

 					line_numbers << "\n"  # also for Mr. MS Internet Exploder :-/

 					line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }

 					

@@ -196,19 +196,22 @@ module CodeRay
 

 #-- don't include the templates in docu

 			

-			SPAN = `<span class="code"><%CONTENT%></span>`

+			SPAN = `<span class="CodeRay"><%CONTENT%></span>`

 

-			DIV, DIV_TABLE, PAGE =

-				<<-`DIV`, <<-`DIV_TABLE`, <<-`PAGE`

+			DIV = <<-`DIV`

 <div class="CodeRay">

 	<div class="code"><pre><%CONTENT%></pre></div>	

 </div>

 			DIV

+

+			DIV_TABLE = <<-`DIV_TABLE`

 <table class="CodeRay"> <tr>

 	<td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>

 	<td class="code"><pre title="double click to expand" ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>

 </tr> </table>

 			DIV_TABLE

+

+			PAGE = <<-`PAGE`

 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"

 	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 58fdd86..15120a2 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -17,7 +17,7 @@ module Encoders
 			:wrap => :page,

 

 			:line_numbers => nil,

-			:line_numbers_offset => 1,

+			:line_number_start => 1,

 			:bold_every => 10,

 		}

 		NUMERIZABLE_WRAPPINGS = [:div, :page]

@@ -133,8 +133,9 @@ module Encoders
 

 		def token text, type

 			if text.is_a? String

-				# be careful when streaming: text is changed!

-				text.gsub!(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }

+				if text =~ /#{HTML_ESCAPE_PATTERN}/o

+					text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }

+				end

 				@opened[0] = type

 				style = @css_style[@opened]

 				if style

diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index 4f0a754..02f76cb 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -1,33 +1,33 @@
 module CodeRay

-	module Encoders

+module Encoders

 

-		class Text < Encoder

+	class Text < Encoder

 

-			include Streamable

-			register_for :text

+		include Streamable

+		register_for :text

 

-			FILE_EXTENSION = 'txt'

+		FILE_EXTENSION = 'txt'

 

-			DEFAULT_OPTIONS = {

-				:separator => ''

-			}

+		DEFAULT_OPTIONS = {

+			:separator => ''

+		}

 

-			protected

-			def setup options

-				super

-				@sep = options[:separator]

-			end

-

-			def token text, kind

-				return unless text.respond_to :to_str

-				@out << text + @sep

-			end

+	protected

+		def setup options

+			super

+			@sep = options[:separator]

+		end

 

-			def finish options

-				@out.chomp @sep

-			end

+		def token text, kind

+			return unless text.respond_to? :to_str

+			@out << text + @sep

+		end

 

+		def finish options

+			@out.chomp @sep

 		end

 

 	end

+

+end

 end

diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
index b0bb49e..aacde99 100644
--- a/lib/coderay/helpers/plugin.rb
+++ b/lib/coderay/helpers/plugin.rb
@@ -1,4 +1,3 @@
-

 # = PluginHost

 #

 # $Id$

@@ -28,6 +27,7 @@ module PluginHost
 	# * a file could not be found

 	# * the requested Encoder is not registered

 	PluginNotFound = Class.new Exception

+	HostNotFound = Class.new Exception

 

 	PLUGIN_HOSTS = []

 	PLUGIN_HOSTS_BY_ID = {}  # dummy hash

@@ -45,11 +45,11 @@ module PluginHost
 		# Find the PluginHost for host_id.

 		def host_by_id host_id

 			unless PLUGIN_HOSTS_BY_ID.default_proc

-				ph = Hash.new do |h, _host_id|

+				ph = Hash.new do |h, a_host_id|

 					for host in PLUGIN_HOSTS

 						h[host.host_id] = host

 					end

-					h.fetch _host_id, nil

+					h.fetch a_host_id, nil

 				end				

 				PLUGIN_HOSTS_BY_ID.replace ph

 			end

@@ -58,138 +58,134 @@ module PluginHost
 

 	end

 	

-		def plugin_host_id host_id

-			if host_id.is_a? String

-				raise ArgumentError,

-					"String or Symbol expected, but #{lang.class} given."

-			end

-			

+	def plugin_host_id host_id

+		if host_id.is_a? String

+			raise ArgumentError,

+				"String or Symbol expected, but #{lang.class} given."

 		end

+	end

 

-		# The path where the plugins can be found.

-		def plugin_path *args

-			unless args.empty?

-				@plugin_path = File.join(*args)

-			end

-			@plugin_path

+	# The path where the plugins can be found.

+	def plugin_path *args

+		unless args.empty?

+			@plugin_path = File.join(*args)

 		end

+		@plugin_path

+	end

 

-		# The host's ID.

-		#

-		# If PLUGIN_HOST_ID is not set, it is simply the class name.

-		def host_id

-			if self.const_defined? :PLUGIN_HOST_ID

-				self::PLUGIN_HOST_ID

-			else

-				name

-			end

+	# The host's ID.

+	#

+	# If PLUGIN_HOST_ID is not set, it is simply the class name.

+	def host_id

+		if self.const_defined? :PLUGIN_HOST_ID

+			self::PLUGIN_HOST_ID

+		else

+			name

 		end

+	end

 

-		def create_plugin_hash

-			@plugin_hash =

-				Hash.new do |h, plugin_id|

-					id = validate_id(plugin_id)

-					path = path_to id

-					begin

-						puts 'Loading plugin: ' + path if $DEBUG

-						require path

-					rescue LoadError

-						raise PluginNotFound, "#{path} not found."

-					else

-						# Plugin should have registered by now

-						unless h.has_key? id

-							raise PluginNotFound,

-								"No #{self.name} plugin for #{id} found in #{path}."

-						end

+	def create_plugin_hash

+		@plugin_hash =

+			Hash.new do |h, plugin_id|

+				id = validate_id(plugin_id)

+				path = path_to id

+				begin

+					$stderr.puts 'Loading plugin: ' + path if $DEBUG

+					require path

+				rescue LoadError

+					raise PluginNotFound, "Plugin #{id.inspect} not found."

+				else

+					# Plugin should have registered by now

+					unless h.has_key? id

+						raise PluginNotFound,

+							"No #{self.name} plugin for #{id.inspect} found in #{path}."

 					end

-					h[id]

 				end

-		end

+				h[id]

+			end

+	end

 

-		def plugin_hash

-			@plugin_hash ||= create_plugin_hash

-		end

+	def plugin_hash

+		@plugin_hash ||= create_plugin_hash

+	end

 

 

-		# Every plugin must register itself for one or more

-		# +ids+ by calling register_for, which calls this method.

-		#

-		# See Plugin#register_for.

-		def register plugin, *ids

-			for id in ids

-				unless id.is_a? Symbol

-					raise ArgumentError,

-						"id must be a Symbol, but it was a #{id.class}" 

-				end

-				plugin_hash[validate_id(id)] = plugin

+	# Every plugin must register itself for one or more

+	# +ids+ by calling register_for, which calls this method.

+	#

+	# See Plugin#register_for.

+	def register plugin, *ids

+		for id in ids

+			unless id.is_a? Symbol

+				raise ArgumentError,

+					"id must be a Symbol, but it was a #{id.class}" 

 			end

+			plugin_hash[validate_id(id)] = plugin

 		end

+	end

 

 

-		# Returns an array of all .rb files in the plugin path.

-		# 

-		# The extension .rb is not included.

-		def all_plugin_names

-			Dir[path_to('*')].map do |file|

-				File.basename file, '.rb'

-			end

+	# Returns an array of all .rb files in the plugin path.

+	# 

+	# The extension .rb is not included.

+	def all_plugin_names

+		Dir[path_to('*')].map do |file|

+			File.basename file, '.rb'

 		end

+	end

 

-		# Loads all plugins using all_plugin_names and load.

-		def load_all

-			for plugin in all_plugin_names

-				load_plugin plugin

-			end

+	# Loads all plugins using all_plugin_names and load.

+	def load_all

+		for plugin in all_plugin_names

+			load_plugin plugin

 		end

+	end

 

 

-		# Returns the Plugin for +id+.

-		# 

-		# Example:

-		#  yaml_plugin = MyPluginHost[:yaml]

-		def [] id, *args, &blk

-			plugin_hash.[] validate_id(id), *args, &blk

-		end

+	# Returns the Plugin for +id+.

+	# 

+	# Example:

+	#  yaml_plugin = MyPluginHost[:yaml]

+	def [] id, *args, &blk

+		plugin_hash.[] validate_id(id), *args, &blk

+	end

 

-		# Alias for +[]+.

-		alias load_plugin []

+	# Alias for +[]+.

+	alias load_plugin []

 

-		# Returns the Plugin for +id+.

-		# Use it like Hash#fetch.

-		# 

-		# Example:

-		#  yaml_plugin = MyPluginHost[:yaml, :default]

-		def fetch id, *args, &blk

-			plugin_hash.fetch validate_id(id), *args, &blk

-		end

+	# Returns the Plugin for +id+.

+	# Use it like Hash#fetch.

+	# 

+	# Example:

+	#  yaml_plugin = MyPluginHost[:yaml, :default]

+	def fetch id, *args, &blk

+		plugin_hash.fetch validate_id(id), *args, &blk

+	end

 

-		# Returns the path to the encoder for format.

-		def path_to plugin_id

-			File.join plugin_path, "#{plugin_id}.rb"

-		end

+	# Returns the path to the encoder for format.

+	def path_to plugin_id

+		File.join plugin_path, "#{plugin_id}.rb"

+	end

 

-		# Converts +id+ to a downcase Symbol if it is a String,

-		# or returns +id+ if it already is a Symbol.

-		#

-		# Raises +ArgumentError+ for all other objects, or if the

-		# given String includes non-alphanumeric characters (\W).

-		def validate_id id

-			if id.is_a? Symbol

-				id

-			elsif id.is_a? String

-				if id[/\w+/] == id

-					id.downcase.to_sym

-				else

-					raise ArgumentError, "Invalid id: '#{id}' given."

-				end

+	# Converts +id+ to a downcase Symbol if it is a String,

+	# or returns +id+ if it already is a Symbol.

+	#

+	# Raises +ArgumentError+ for all other objects, or if the

+	# given String includes non-alphanumeric characters (\W).

+	def validate_id id

+		if id.is_a? Symbol

+			id

+		elsif id.is_a? String

+			if id[/\w+/] == id

+				id.downcase.to_sym

 			else

-				raise ArgumentError,

-					"String or Symbol expected, but #{id.class} given."

+				raise ArgumentError, "Invalid id: '#{id}' given."

 			end

+		else

+			raise ArgumentError,

+				"String or Symbol expected, but #{id.class} given."

 		end

-

-	#end

-

+	end

 

 end

 

@@ -239,8 +235,11 @@ end
 # 

 # Returns the loaded plugin.

 def require_plugin path

-	host, plugin_id = path.split '/', 2

-	PluginHost.host_by_id(host).load_plugin plugin_id

+	host_id, plugin_id = path.split '/', 2

+	host = PluginHost.host_by_id(host_id)

+	raise PluginHost::HostNotFound,

+		"No host for #{host_id.inspect} found." unless host

+	host.load_plugin plugin_id

 end

 

 

diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index 433726b..72e59bd 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -72,7 +72,8 @@ module CodeRay module Scanners
 							tokens << [modifiers, :modifier] unless modifiers.empty?

 							if parse_regexp

 								extended = modifiers.index ?x

-								tokens, regexp = saved_tokens, tokens

+								tokens = saved_tokens

+								regexp = tokens

 								for text, type in regexp

 									if text.is_a? String

 										case type

@@ -125,7 +126,8 @@ module CodeRay module Scanners
 						when ?{

 							states.push [state, depth, heredocs]

 							fancy_allowed = regexp_allowed = true

-							state, depth = :initial, 1

+							state = :initial

+							depth = 1

 							tokens << [match + getch, :escape]

 						when ?$, ?@

 							tokens << [match, :escape]

@@ -192,7 +194,7 @@ module CodeRay module Scanners
 								when '}'

 									depth -= 1

 									if depth == 0

-										state, depth, heredocs = *states.pop

+										state, depth, heredocs = states.pop

 										type = :escape

 									end

 								end

@@ -214,7 +216,7 @@ module CodeRay module Scanners
 						elsif match = scan(/ ['"] /mx)

 							tokens << [:open, :string]

 							type = :delimiter

-							state = StringState.new :string, match != '\'', match.dup  # important for streaming

+							state = StringState.new :string, match != '\'', match  # important for streaming

 							

 						elsif match = scan(/#{INSTANCE_VARIABLE}/o)

 							type = :instance_variable

@@ -223,13 +225,14 @@ module CodeRay module Scanners
 							tokens << [:open, :regexp]

 							type = :delimiter

 							interpreted = true

-							state = StringState.new :regexp, interpreted, match.dup

+							state = StringState.new :regexp, interpreted, match

 							if parse_regexp

-								tokens, saved_tokens = [], tokens

+								tokens = []

+								saved_tokens = tokens

 							end

 							

 						elsif match = scan(/#{NUMERIC}/o)

-							type = if match[/#{FLOAT}/o] then :float else :integer end							

+							type = if match[/#{FLOAT}/o] then :float else :integer end

 

 						elsif fancy_allowed and match = scan(/#{SYMBOL}/o)

 							case match[1]

@@ -240,7 +243,8 @@ module CodeRay module Scanners
 							type = :symbol

 							

 						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)

-							indented, quote = self[1] == '-', self[3]

+							indented = self[1] == '-'

+							quote = self[3]

 							delim = self[quote ? 4 : 2]

 							type = QUOTE_TO_TYPE[quote]

 							tokens << [:open, type]

@@ -271,7 +275,7 @@ module CodeRay module Scanners
 							else

 								tokens << [:open, :shell]

 								type = :delimiter

-								state = StringState.new :shell, true, '`'

+								state = StringState.new :shell, true, match

 							end

 							

 						elsif match = scan(/#{GLOBAL_VARIABLE}/o)

diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 988008e..9318844 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -1,5 +1,7 @@
 module CodeRay

 

+	# = Tokens

+	#

 	# The Tokens class represents a list of tokens returnd from

 	# a Scanner.

 	#

@@ -137,6 +139,15 @@ module CodeRay
 			encoder.encode_tokens self, options

 		end

 

+

+		# Turn into a string using Encoders::Text.

+		#

+		# +options+ are passed to the encoder if given.

+		def to_s options = {}

+			encode :text, options

+		end

+

+

 		# Redirects unknown methods to encoder calls.

 		#

 		# For example, if you call +tokens.html+, the HTML encoder

@@ -152,13 +163,13 @@ module CodeRay
 		# in most Encoders.  It basically makes the output smaller.

 		#

 		# Combined with dump, it saves space for the cost

-    # calculating time.

-    #

-    # If the scanner is written carefully, this is not required - 

-    # for example, consecutive //-comment lines can already be 

-    # joined in one token by the Scanner.

+		# calculating time.

+		#

+		# If the scanner is written carefully, this is not required - 

+		# for example, consecutive //-comment lines can already be 

+		# joined in one token by the Scanner.

 		def optimize

-      print ' Tokens#optimize: before: %d - ' % size if $DEBUG

+			print ' Tokens#optimize: before: %d - ' % size if $DEBUG

 			last_kind = last_text = nil

 			new = self.class.new

 			each do |text, kind|

@@ -177,8 +188,8 @@ module CodeRay
 				end

 			end

 			new << [last_text, last_kind] if last_kind

-      print 'after: %d (%d saved = %2.0f%%)' % 

-        [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG

+			print 'after: %d (%d saved = %2.0f%%)' % 

+				[new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG

 			new

 		end

 

@@ -241,6 +252,8 @@ module CodeRay
 	end

 

 

+	# = TokenStream

+	#

 	# The TokenStream class is a fake Array without elements.

 	# 

 	# It redirects the method << to a block given at creation.

@@ -283,9 +296,12 @@ module CodeRay
 		end

 

 		# Calls +block+ with +token+ and increments size.

+		#

+		# Returns self.

 		def << token

 			@callback.call token

 			@size += 1

+			self

 		end

 

 		# This method is not implemented due to speed reasons. Use Tokens.

@@ -306,5 +322,3 @@ module CodeRay
 	end

 

 end

-

-# vim:sw=2:ts=2:et:tw=78