After merge with Plugin branch.

3 files changed, 472 insertions, 369 deletions

diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index c5295be..74e1582 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -8,65 +8,8 @@ module CodeRay
 	# mechanism and the [] method that returns the Encoder class

 	# belonging to the given format.

 	module Encoders

-

-		# Raised if Encoders::[] fails because:

-		# * a file could not be found

-		# * the requested Encoder is not registered

-		EncoderNotFound = Class.new Exception

-

-		def Encoders.create_encoders_hash

-			Hash.new do |h, lang|

-				path = Encoders.path_to lang

-				lang = lang.to_sym

-				begin

-					require path

-				rescue LoadError

-					raise EncoderNotFound, "#{path} not found."

-				else

-					# Encoder should have registered by now

-					unless h[lang]

-						raise EncoderNotFound,

-							"No Encoder for #{lang} found in #{path}."

-					end

-				end

-				h[lang]

-			end

-		end

-

-		# Loaded Encoders are saved here.

-		ENCODERS = create_encoders_hash

-

-		class << self

-

-			# Every Encoder class must register itself for one or more

-			# +formats+ by calling register_for, which calls this method.

-			#

-			# See CodeRay::Encoder.register_for.

-			def register encoder_class, *formats

-				for format in formats

-					ENCODERS[format.to_sym] = encoder_class

-				end

-			end

-

-			# Returns the Encoder for +lang+.

-			# 

-			# Example:

-			#  require 'coderay'

-			#  yaml_encoder = CodeRay::Encoders[:yaml]

-			def [] lang

-				ENCODERS[lang]

-			end

-

-			# Alias for +[]+.

-			alias load []

-

-			# Returns the path to the encoder for format.

-			def path_to plugin

-				File.join 'coderay', 'encoders', "#{plugin}.rb"

-			end

-

-		end

-

+		extend PluginHost

+		plugin_path 'coderay/encoders'

 

 		# = Encoder

 		#

@@ -81,24 +24,13 @@ module CodeRay
 		# If you want the highlighted code in a div or a span instead,

 		# use its subclasses Div and Span. 

 		class Encoder

+			extend Plugin

+			plugin_host Encoders

 

 			attr_reader :token_stream

 

 			class << self

 

-				# Register this class for the given langs.

-				#

-				# Example:

-				#   class MyEncoder < CodeRay::Encoders:Encoder

-				#     register_for :myenc

-				#     ...

-				#   end

-				#

-				# See Encoder.register.

-				def register_for *args

-					Encoders.register self, *args

-				end

-

 				# Returns if the Encoder can be used in streaming mode.

 				def streamable?

 					is_a? Streamable

@@ -131,12 +63,12 @@ module CodeRay
 			# - encode_tokens expects a +tokens+ object instead

 			# - encode_stream is like encode, but uses streaming mode.

 			# 

-			# Each method has an optional +options+ parameter. These are added

-			# to the options you passed at creation.

+			# Each method has an optional +options+ parameter. These are

+			# added to the options you passed at creation.

 			def initialize options = {}

 				@options = self.class::DEFAULT_OPTIONS.merge options

-				raise "I am only the basic Encoder class. I can't encode anything. :(\n"\

-					"Use my subclasses." if self.class == Encoder

+				raise "I am only the basic Encoder class. I can't encode "\

+					"anything. :( Use my subclasses." if self.class == Encoder

 			end

 

 			# Encode a Tokens object.

@@ -147,8 +79,8 @@ module CodeRay
 				finish options

 			end

 

-			# Encode the given +code+ after tokenizing it using the Scanner for

-			# +lang+.

+			# Encode the given +code+ after tokenizing it using the Scanner

+			# for +lang+.

 			def encode code, lang, options = {}

 				options = @options.merge options

 				scanner_options = CodeRay.get_scanner_options(options)

@@ -160,8 +92,8 @@ module CodeRay
 			# more clear to you.

 			alias highlight encode

 

-			# Encode the given +code+ using the Scanner for +lang+ in streaming

-			# mode.

+			# Encode the given +code+ using the Scanner for +lang+ in

+			# streaming mode.

 			def encode_stream code, lang, options = {}

 				raise NotStreamableError, self unless kind_of? Streamable

 				options = @options.merge options

@@ -177,7 +109,7 @@ module CodeRay
 				method(:token).to_proc

 			end

 

-			protected

+		protected

 

 			# Called with merged options before encoding starts.

 			# Sets @out to an empty string.

@@ -193,7 +125,8 @@ module CodeRay
 			# Raises a NotImplementedError exception if it is not overwritten

 			# in subclass.

 			def token text, kind

-				raise NotImplementedError, "#{self.class}#token not implemented."

+				raise NotImplementedError,

+					"#{self.class}#token not implemented."

 			end

 

 			# Called with merged options after encoding starts.

diff --git a/lib/coderay/helpers/plugin.rb b/lib/coderay/helpers/plugin.rb
new file mode 100644
index 0000000..b0bb49e
--- /dev/null
+++ b/lib/coderay/helpers/plugin.rb
@@ -0,0 +1,287 @@
+

+# = PluginHost

+#

+# $Id$

+#

+# A simple subclass plugin system.

+# 

+#	Example:		

+#		class Generators < PluginHost

+#			plugin_path 'app/generators'

+#		end

+#		

+#	  class Generator

+#	  	extend Plugin

+#	  	PLUGIN_HOST = Generators

+#	  end

+#   

+#	  class FancyGenerator < Generator

+#	  	register_for :fancy

+#	  end

+#

+#	  Generators[:fancy]  #-> FancyGenerator

+#	  # or

+#	  require_plugin 'Generators/fancy'

+module PluginHost

+

+	# Raised if Encoders::[] fails because:

+	# * a file could not be found

+	# * the requested Encoder is not registered

+	PluginNotFound = Class.new Exception

+

+	PLUGIN_HOSTS = []

+	PLUGIN_HOSTS_BY_ID = {}  # dummy hash

+

+	class << self

+

+		def extended mod

+			PLUGIN_HOSTS << mod

+		end

+

+		def included mod

+			warn "#{name} should not be included. Use extend."

+		end

+

+		# Find the PluginHost for host_id.

+		def host_by_id host_id

+			unless PLUGIN_HOSTS_BY_ID.default_proc

+				ph = Hash.new do |h, _host_id|

+					for host in PLUGIN_HOSTS

+						h[host.host_id] = host

+					end

+					h.fetch _host_id, nil

+				end				

+				PLUGIN_HOSTS_BY_ID.replace ph

+			end

+			PLUGIN_HOSTS_BY_ID[host_id]

+		end

+

+	end

+	

+		def plugin_host_id host_id

+			if host_id.is_a? String

+				raise ArgumentError,

+					"String or Symbol expected, but #{lang.class} given."

+			end

+			

+		end

+

+		# The path where the plugins can be found.

+		def plugin_path *args

+			unless args.empty?

+				@plugin_path = File.join(*args)

+			end

+			@plugin_path

+		end

+

+		# The host's ID.

+		#

+		# If PLUGIN_HOST_ID is not set, it is simply the class name.

+		def host_id

+			if self.const_defined? :PLUGIN_HOST_ID

+				self::PLUGIN_HOST_ID

+			else

+				name

+			end

+		end

+

+		def create_plugin_hash

+			@plugin_hash =

+				Hash.new do |h, plugin_id|

+					id = validate_id(plugin_id)

+					path = path_to id

+					begin

+						puts 'Loading plugin: ' + path if $DEBUG

+						require path

+					rescue LoadError

+						raise PluginNotFound, "#{path} not found."

+					else

+						# Plugin should have registered by now

+						unless h.has_key? id

+							raise PluginNotFound,

+								"No #{self.name} plugin for #{id} found in #{path}."

+						end

+					end

+					h[id]

+				end

+		end

+

+		def plugin_hash

+			@plugin_hash ||= create_plugin_hash

+		end

+

+

+		# Every plugin must register itself for one or more

+		# +ids+ by calling register_for, which calls this method.

+		#

+		# See Plugin#register_for.

+		def register plugin, *ids

+			for id in ids

+				unless id.is_a? Symbol

+					raise ArgumentError,

+						"id must be a Symbol, but it was a #{id.class}" 

+				end

+				plugin_hash[validate_id(id)] = plugin

+			end

+		end

+

+

+		# Returns an array of all .rb files in the plugin path.

+		# 

+		# The extension .rb is not included.

+		def all_plugin_names

+			Dir[path_to('*')].map do |file|

+				File.basename file, '.rb'

+			end

+		end

+

+		# Loads all plugins using all_plugin_names and load.

+		def load_all

+			for plugin in all_plugin_names

+				load_plugin plugin

+			end

+		end

+

+

+		# Returns the Plugin for +id+.

+		# 

+		# Example:

+		#  yaml_plugin = MyPluginHost[:yaml]

+		def [] id, *args, &blk

+			plugin_hash.[] validate_id(id), *args, &blk

+		end

+

+		# Alias for +[]+.

+		alias load_plugin []

+

+		# Returns the Plugin for +id+.

+		# Use it like Hash#fetch.

+		# 

+		# Example:

+		#  yaml_plugin = MyPluginHost[:yaml, :default]

+		def fetch id, *args, &blk

+			plugin_hash.fetch validate_id(id), *args, &blk

+		end

+

+		# Returns the path to the encoder for format.

+		def path_to plugin_id

+			File.join plugin_path, "#{plugin_id}.rb"

+		end

+

+		# Converts +id+ to a downcase Symbol if it is a String,

+		# or returns +id+ if it already is a Symbol.

+		#

+		# Raises +ArgumentError+ for all other objects, or if the

+		# given String includes non-alphanumeric characters (\W).

+		def validate_id id

+			if id.is_a? Symbol

+				id

+			elsif id.is_a? String

+				if id[/\w+/] == id

+					id.downcase.to_sym

+				else

+					raise ArgumentError, "Invalid id: '#{id}' given."

+				end

+			else

+				raise ArgumentError,

+					"String or Symbol expected, but #{id.class} given."

+			end

+		end

+

+	#end

+

+

+end

+

+

+# = Plugin

+# 

+#	Plugins have to include this module.

+#

+#	IMPORTANT: use extend for this module.

+#

+#	Example: see PluginHost.

+module Plugin

+

+	def included mod

+		warn "#{name} should not be included. Use extend."

+	end

+

+	# Register this class for the given langs.

+	# Example:

+	#   class MyPlugin < PluginHost::BaseClass

+	#     register_for :my_id

+	#     ...

+	#   end

+	#

+	# See PluginHost.register.

+	def register_for *ids

+		plugin_host.register self, *ids

+	end

+

+	# The host for this Plugin class.

+	def plugin_host host = nil

+		if host and not host.is_a? PluginHost

+			raise ArgumentError,

+				"PluginHost expected, but #{host.class} given."

+		end

+		self.const_set :PLUGIN_HOST, host if host

+		self::PLUGIN_HOST

+	end

+

+end

+

+

+# Convenience method for plugin loading.

+# The syntax used is:

+#

+#  require_plugin '<Host ID>/<Plugin ID>'

+# 

+# Returns the loaded plugin.

+def require_plugin path

+	host, plugin_id = path.split '/', 2

+	PluginHost.host_by_id(host).load_plugin plugin_id

+end

+

+

+if $0 == __FILE__

+	$VERBOSE = $DEBUG = true

+	eval DATA.read, nil, $0, __LINE__+4

+end

+

+__END__

+

+require 'test/unit'

+

+class TC_PLUGINS < Test::Unit::TestCase

+

+	class Generators

+		extend PluginHost

+		plugin_path '.'

+	end

+

+	class Generator

+		extend Plugin

+		plugin_host Generators

+	end

+

+	class FancyGenerator < Generator

+		register_for :plugin_host

+	end

+

+	def test_plugin

+		assert_nothing_raised do

+			Generators[:plugin_host]

+		end	

+		assert_equal FancyGenerator, Generators[:plugin_host]

+	end

+	

+	def test_require

+		assert_nothing_raised do

+			require_plugin('TC_PLUGINS::Generators/plugin_host')

+		end

+		assert_equal FancyGenerator,

+			require_plugin('TC_PLUGINS::Generators/plugin_host')

+	end

+

+end

diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 58a92f7..cf4f3c6 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -1,283 +1,166 @@
 module CodeRay

-

-  # This module holds class Scanner and its subclasses.

-  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby

-  # can be found in coderay/scanners/ruby.

-  # 

-  # Scanner also provides methods and constants for the register

-  # mechanism and the [] method that returns the Scanner class

-  # belonging to the given lang.

-  module Scanners

-

-    # Raised if Scanners[] fails because:

-    # * a file could not be found

-    # * the requested Scanner is not registered

-    ScannerNotFound = Class.new(Exception)

-

-    # Loaded Scanners are saved here.

-    SCANNERS = Hash.new { |h, lang|

-      raise ScannerNotFound, "No scanner for #{lang} found."

-    }

-

-    class << self

-

-      # Registers a scanner class by setting SCANNERS[lang].

-      #

-      # Typically used in Scanners, for example in the Ruby scanner:

-      #

-      #   register_for :ruby

-      def register scanner_class, *langs

-        for lang in langs

-          unless lang.is_a? Symbol

-            raise ArgumentError,

-              "lang must be a Symbol, but it was a #{lang.class}"

-          end

-          SCANNERS[lang] = scanner_class

-        end

-      end

-

-      # Loads the scanner class for +lang+ and returns it.

-      #

-      # Example:

-      #

-      #   Scanners[:xml].new

-      #

-      # +lang+ is converted using +normalize+ and must be

-      # * a String containing only alphanumeric characters (\w+)

-      # * a Symbol

-      #

-      # Strings are converted to lowercase symbols (so +'C'+ and +'c'+

-      # load the same scanner, namely the one registered for +:c+.)

-      # 

-      # If the scanner isn't registered yet, it is searched.

-      # CodeRay expects that the scanner class is defined in

-      #

-      #   <install-dir>/coderay/scanners/<lang>.rb

-      #

-      # (See path_to.)

-      #

-      # If the file isn't found, a ScannerNotFound exception is raised

-      #

-      # The scanner should register itself using +register+. If the

-      # scanner is still not found (because has not registered or

-      # registered under another

-      # lang), a ScannerNotFound exception is raised.

-      def [] lang

-        lang = normalize lang

-

-        SCANNERS.fetch lang do

-          scanner_file = path_to lang

-

-          begin

-            require scanner_file

-          rescue LoadError

-            raise ScannerNotFound, "File #{scanner_file} not found."

-          end

-

-          SCANNERS.fetch lang do

-            raise ScannerNotFound, <<-ERR

-No scanner for #{lang} found in #{scanner_file}.

-Known scanners: #{SCANNERS}

-            ERR

-          end

-        end

-      end

-

-      # Alias for +[]+.

-      alias load []

-

-      # Calculates the path where a scanner for +lang+

-      # is expected to be. This is:

-      # 

-      #   <install-dir>/coderay/scanners/<lang>.rb

-      def path_to lang

-        File.join 'coderay', 'scanners', "#{lang}.rb"

-      end

-

-      # Returns an array of all filenames in the scanners/ folder.

-      # The extension +.rb+ is not included.

-      def languages

-        scanners = File.join File.dirname(__FILE__), 'scanners', '*.rb'

-        Dir[scanners].map do |file|

-          File.basename file, '.rb'

-        end

-      end

-

-      # Loads all scanners that +languages+ finds using +load+.

-      def load_all

-        for lang in languages

-          load lang

-        end

-      end

-

-      # Converts +lang+ to a downcase Symbol if it is a String,

-      # or returns +lang+ if it already is a Symbol.

-      #

-      # Raises +ArgumentError+ for all other objects, or if the

-      # given String includes non-alphanumeric characters (\W).

-      def normalize lang

-        if lang.is_a? Symbol

-          lang

-        elsif lang.is_a? String

-          if lang[/\w+/] == lang

-            lang[/\w+/].downcase.to_sym

-          else

-            raise ArgumentError, "Invalid lang: '#{lang}' given."

-          end

-        elsif lang.nil?

-          :plaintext

-        else

-          raise ArgumentError,

-            "String or Symbol expected, but #{lang.class} given."

-        end

-      end

-

-    end

-

-

-    require 'strscan'

-    # = Scanner

-    #

-    # The base class for all Scanners.

-    #

-    # It is a subclass of Ruby's great +StringScanner+, which

-    # makes it easy to access the scanning methods inside.

-    #

-    # It is also +Enumerable+, so you can use it like an Array of Tokens:

-    #

-    #   require 'coderay'

-    #   

-    #   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"

-    #   

-    #   for text, kind in c_scanner

-    #     puts text if kind == :operator

-    #   end

-    #   

-    #   # prints: (*==)++;

-    # 

-    # OK, this is a very simple example :)

-    # You can also use +map+, +any?+, +find+ and even +sort_by+,

-    # if you want.

-    class Scanner < StringScanner

-

-      # Raised if a Scanner fails while scanning

-      ScanError = Class.new(Exception)

-

-      require 'coderay/helpers/scanner_helper'

-

-      # The default options for all scanner classes.

-      # 

-      # Define @default_options for subclasses.

-      DEFAULT_OPTIONS = { :stream => false }

-

-      class << self

-        # Register the scanner class for all

-        # +langs+.

-        #

-        # See Scanners.register.

-        def register_for *langs

-          Scanners.register self, *langs

-        end

-

-        # Returns if the Scanner can be used in streaming mode.

-        def streamable?

-          is_a? Streamable

-        end

-

-      end

+	

+	require 'coderay/helpers/plugin'

+

+	# = Scanners

+	# 

+	# $Id$

+	# 

+	# This module holds the Scanner class and its subclasses.

+	# For example, the Ruby scanner is named CodeRay::Scanners::Ruby

+	# can be found in coderay/scanners/ruby.

+	# 

+	# Scanner also provides methods and constants for the register

+	# mechanism and the [] method that returns the Scanner class

+	# belonging to the given lang.

+	#

+	# See PluginHost.

+	module Scanners

+		extend PluginHost

+		plugin_path 'coderay/scanners'

+

+		require 'strscan'

+

+		# = Scanner

+		#

+		# The base class for all Scanners.

+		#

+		# It is a subclass of Ruby's great +StringScanner+, which

+		# makes it easy to access the scanning methods inside.

+		#

+		# It is also +Enumerable+, so you can use it like an Array of

+		# Tokens:

+		# 

+		#   require 'coderay'

+		#   

+		#   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"

+		#   

+		#   for text, kind in c_scanner

+		#     puts text if kind == :operator

+		#   end

+		#   

+		#   # prints: (*==)++;

+		# 

+		# OK, this is a very simple example :)

+		# You can also use +map+, +any?+, +find+ and even +sort_by+,

+		# if you want.

+		class Scanner < StringScanner

+			extend Plugin

+			plugin_host Scanners

+

+			# Raised if a Scanner fails while scanning

+			ScanError = Class.new(Exception)

+

+			require 'coderay/helpers/scanner_helper'

+

+			# The default options for all scanner classes.

+			# 

+			# Define @default_options for subclasses.

+			DEFAULT_OPTIONS = { :stream => false }

+

+			class << self

+

+				# Returns if the Scanner can be used in streaming mode.

+				def streamable?

+					is_a? Streamable

+				end

+

+			end

 

 =begin

-      ## Excluded for speed reasons; protected seems to make methods slow.

-

-      # Save the StringScanner methods from being called.

-      # This would not be useful for highlighting.

-strscan_public_methods =

-  StringScanner.instance_methods - StringScanner.ancestors[1].instance_methods

-protected(*strscan_public_methods)

+## Excluded for speed reasons; protected seems to make methods slow.

+

+# Save the StringScanner methods from being called.

+# This would not be useful for highlighting.

+	strscan_public_methods =

+		StringScanner.instance_methods -

+		StringScanner.ancestors[1].instance_methods

+	protected(*strscan_public_methods)

 =end

 

-      # Creates a new Scanner.

-      #

-      # * +code+ is the input String and is handled by the superclass

-      #   StringScanner.

-      # * +options+ is a Hash with Symbols as keys.

-      #   It is merged with the default options of the class (you can

-      #   overwrite default options here.)

-      # * +block+ is the callback for streamed highlighting.

-      #

-      # If you set :stream to +true+ in the options, the Scanner uses a

-      # TokenStream with the +block+ as callback to handle the tokens.

-      #

-      # Else, a Tokens object is used.

-      def initialize code, options = {}, &block

-        @options = self.class::DEFAULT_OPTIONS.merge options

-        raise "I am only the basic Scanner class. I can't scan anything. :(\n" + 

-          "Use my subclasses." if self.class == Scanner

-

-        # I love this hack. It seems to silence

-        # all dos/unix/mac newline problems.

-        super code.gsub(/\r\n?/, "\n")

-

-        if @options[:stream]

-          warn "warning in CodeRay::Scanner.new: :stream is set, "\

-            "but no block was given" unless block_given?

-          raise NotStreamableError, self unless kind_of? Streamable

-          @tokens = TokenStream.new(&block)

-        else

-          warn "warning in CodeRay::Scanner.new: Block given, "\

-            "but :stream is #{@options[:stream]}" if block_given?

-          @tokens = Tokens.new

-        end

-      end

-

-      # More mnemonic accessor name for the input string.

-      alias code string

-

-      # Scans the code and returns all tokens in a Tokens object.

-      def tokenize options = {}

-        options = @options.merge({}) #options

-        if @options[:stream]  # :stream must have been set already

-          reset ## what is this for?

-          scan_tokens @tokens, options

-          @tokens

-        else

-          @cached_tokens ||= scan_tokens @tokens, options

-        end

-      end

-

-      # you can also see this as a read-only attribute

-      alias tokens tokenize

-

-      # Traverses the tokens.

-      def each &block

-        raise ArgumentError, 

-          'Cannot traverse TokenStream.' if @options[:stream]

-        tokens.each(&block)

-      end

-      include Enumerable

-

-      # The current line position of the scanner.

-      #

-      # Beware, this is implemented inefficiently. It should be used

-      # for debugging only.

-      def line

-        string[0..pos].count("\n") + 1

-      end

-

-      protected

-

-      # This is the central method, and commonly the only one a subclass

-      # implements.

-      # 

-      # Subclasses must implement this method; it must return +tokens+

-      # and must only use Tokens#<< for storing scanned tokens!

-      def scan_tokens tokens, options

-        raise NotImplementedError, "#{self.class}#scan_tokens not implemented."

-      end

-

-      # Scanner error with additional status information

-      def raise_inspect msg, tokens, ambit = 30

-        raise ScanError, <<-EOE % [

+			# Create a new Scanner.

+			#

+			# * +code+ is the input String and is handled by the superclass

+			#   StringScanner.

+			# * +options+ is a Hash with Symbols as keys.

+			#   It is merged with the default options of the class (you can

+			#   overwrite default options here.)

+			# * +block+ is the callback for streamed highlighting.

+			#

+			# If you set :stream to +true+ in the options, the Scanner uses a

+			# TokenStream with the +block+ as callback to handle the tokens.

+			#

+			# Else, a Tokens object is used.

+			def initialize code, options = {}, &block

+				@options = self.class::DEFAULT_OPTIONS.merge options

+				raise "I am only the basic Scanner class. I can't scan "\

+					"anything. :( Use my subclasses." if self.class == Scanner

+

+				# I love this hack. It seems to silence

+				# all dos/unix/mac newline problems.

+				super code.gsub(/\r\n?/, "\n")

+

+				if @options[:stream]

+					warn "warning in CodeRay::Scanner.new: :stream is set, "\

+						"but no block was given" unless block_given?

+					raise NotStreamableError, self unless kind_of? Streamable

+					@tokens = TokenStream.new(&block)

+				else

+					warn "warning in CodeRay::Scanner.new: Block given, "\

+						"but :stream is #{@options[:stream]}" if block_given?

+					@tokens = Tokens.new

+				end

+			end

+

+			# More mnemonic accessor name for the input string.

+			alias code string

+

+			# Scans the code and returns all tokens in a Tokens object.

+			def tokenize options = {}

+				options = @options.merge({}) #options

+				if @options[:stream]  # :stream must have been set already

+					reset ## what is this for?

+					scan_tokens @tokens, options

+					@tokens

+				else

+					@cached_tokens ||= scan_tokens @tokens, options

+				end

+			end

+

+			# You can also see tokenize as a read-only attribute

+			alias tokens tokenize

+

+			# Traverses the tokens.

+			def each &block

+				raise ArgumentError, 

+					'Cannot traverse TokenStream.' if @options[:stream]

+				tokens.each(&block)

+			end

+			include Enumerable

+

+			# The current line position of the scanner.

+			#

+			# Beware, this is implemented inefficiently. It should be used

+			# for debugging only.

+			def line

+				string[0..pos].count("\n") + 1

+			end

+

+		protected

+

+			# This is the central method, and commonly the only one a

+			# subclass implements.

+			# 

+			# Subclasses must implement this method; it must return +tokens+

+			# and must only use Tokens#<< for storing scanned tokens!

+			def scan_tokens tokens, options

+				raise NotImplementedError,

+					"#{self.class}#scan_tokens not implemented."

+			end

+

+			# Scanner error with additional status information

+			def raise_inspect msg, tokens, ambit = 30

+				raise ScanError, <<-EOE % [

 

 

 ***ERROR in %s: %s

@@ -295,20 +178,20 @@ surrounding code:
 

 ***ERROR***

 

-        EOE

-        File.basename(caller[0]),

-        msg,

-        tokens.last(10).map { |t| t.inspect }.join("\n"),

-        line, pos,

-        matched, bol?, eos?,

-        string[pos-ambit,ambit],

-        string[pos,ambit],

-        ]

-      end

+				EOE

+				File.basename(caller[0]),

+				msg,

+				tokens.last(10).map { |t| t.inspect }.join("\n"),

+				line, pos,

+				matched, bol?, eos?,

+				string[pos-ambit,ambit],

+				string[pos,ambit],

+				]

+			end

 

-    end

+		end

 

-  end

+	end

 end

 

 # vim:sw=2:ts=2:noet:tw=78