1 files changed, 302 insertions, 0 deletions
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
new file mode 100644
index 0000000..71ad33a
--- /dev/null
+++ b/lib/coderay/tokens.rb
@@ -0,0 +1,302 @@
+module CodeRay
+
+	# The Tokens class represents a list of tokens returnd from
+	# a Scanner.
+	#
+	# A token is not a special object, just a two-element Array
+	# consisting of
+	# * the _token_ _kind_ (a Symbol representing the type of the token)
+	# * the _token_ _text_ (the original source of the token in a String)
+	#
+	# A token looks like this:
+	# 
+	#   [:comment, '# It looks like this']
+	#   [:float, '3.1415926']
+	#   [:error, 'äöü']
+	# 
+	# Some scanners also yield some kind of sub-tokens, represented by special
+	# token texts, namely :open and :close .
+	# 
+	# The Ruby scanner, for example, splits "a string" into:
+	# 
+	#  [
+	#   [:open, :string],
+	#   [:delimiter, '"'],
+	#   [:content, 'a string'],
+	#   [:delimiter, '"'],
+	#   [:close, :string]
+	#  ]
+	# 
+	# Tokens is also the interface between Scanners and Encoders:
+	# The input is split and saved into a Tokens object. The Encoder
+	# then builds the output from this object.
+	# 
+	# Thus, the syntax below becomes clear:
+	#
+	#   CodeRay.scan('price = 2.59', :ruby).html
+	#   # the Tokens object is here -------^
+	# 
+	# See how small it is? ;)
+	# 
+	# Tokens gives you the power to handle pre-scanned code very easily:
+	# You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
+	# that you put in your DB.
+	#
+	# Tokens' subclass TokenStream allows streaming to save memory.
+	class Tokens < Array
+
+		class << self
+
+			# Convert the token to a string.
+			# 
+			# This format is used by Encoders.Tokens.
+			# It can be reverted using read_token.
+			def write_token text, type
+				if text.is_a? String
+					"#{type}\t#{escape(text)}\n"
+				else
+					":#{text}\t#{type}\t\n"
+				end
+			end
+
+			# Read a token from the string.
+			# 
+			# Inversion of write_token.
+			#
+			# TODO Test this!
+			def read_token token
+				type, text = token.split("\t", 2)
+				if type[0] == ?:
+					[text.to_sym, type[1..-1].to_sym]
+				else
+					[type.to_sym, unescape(text)]
+				end
+			end
+
+			# Escapes a string for use in write_token.
+			def escape text
+				text.gsub(/[\n\\]/, '\\\\\&')
+			end
+
+			# Unescapes a string created by escape.
+			def unescape text
+				text.gsub(/\\[\n\\]/) { |m| m[1,1] }
+			end
+
+		end
+
+		# Whether the object is a TokenStream.
+		#
+		# Returns false.
+		def stream?
+			false
+		end
+
+		alias :orig_each :each
+		# Iterates over all tokens.
+		# 
+		# If a filter is given, only tokens of that kind are yielded.
+		def each kind_filter = nil, &block
+			unless kind_filter
+				orig_each(&block)
+			else
+				orig_each do |text, kind|
+					next unless kind == kind_filter
+					yield text, kind
+				end
+			end
+		end
+
+		# Iterates over all text tokens.
+		# Range tokens like [:open, :string] are left out.
+		#
+		# Example:
+		#   tokens.each_text_token { |text, kind| text.replace html_escape(text) }
+		def each_text_token
+			orig_each do |text, kind|
+				next unless text.respond_to? :to_str
+				yield text, kind
+			end
+		end
+
+		# Encode the tokens using encoder.
+		#
+		# encoder can be
+		# * a symbol like :html oder :statistic
+		# * an Encoder class
+		# * an Encoder object
+		# 
+		# options are passed to the encoder.
+		def encode encoder, options = {}
+			unless encoder.is_a? Encoders::Encoder
+				unless encoder.is_a? Class
+					encoder_class = Encoders[encoder]
+				end
+				encoder = encoder_class.new options
+			end
+			encoder.encode_tokens self, options
+		end
+
+		# Redirects unknown methods to encoder calls.
+		#
+		# For example, if you call +tokens.html+, the HTML encoder
+		# is used to highlight the tokens.
+		def method_missing meth, options = {}
+			Encoders[meth].new(options).encode_tokens self
+		end
+
+		# Returns the tokens compressed by joining consecutive
+		# tokens of the same kind.
+		# 
+		# This can not be undone, but should yield the same output
+		# in most Encoders.  It basically makes the output smaller.
+		#
+		# Combined with dump, it saves database space.
+		def optimize
+			last_kind, last_text = nil, nil
+			new = self.class.new
+			each do |text, kind|
+				if text.is_a? String
+					if kind == last_kind
+						last_text << text
+					else
+						new << [last_text, last_kind] if last_kind
+						last_text = text
+						last_kind = kind
+					end
+				else
+					new << [last_text, last_kind] if last_kind
+					last_kind, last_text = nil, nil
+					new << [text, kind]
+				end
+			end
+			new << [last_text, last_kind] if last_kind
+			new
+		end
+
+		# Compact the object itself; see compact.
+		def optimize!
+			replace optimize
+		end
+
+		# Dumps the object into a String that can be saved
+		# in files or databases.
+		#
+		# The dump is created with Marshal.dump;
+		# In addition, it is gzipped using GZip.gzip.
+		#
+		# The returned String object includes Undumping
+		# so it has an #undump method. See Tokens.load.
+		#
+		# You can configure the level of compression,
+		# but the default value 7 should be what you want
+		# in most cases as it is a good comprimise between
+		# speed and compression rate.
+		# 
+		# See GZip module.
+		def dump gzip_level = 7
+			require 'coderay/helpers/gzip_simple'
+			dump = Marshal.dump self
+			dump = dump.gzip gzip_level
+			dump.extend Undumping
+		end
+
+		# The total size of the tokens;
+		# Should be equal to the input size before
+		# scanning.
+		def text_size
+			map { |t, k| t }.join.size
+		end
+
+		# Include this module to give an object an #undump
+		# method.
+		#
+		# The string returned by Tokens.dump includes Undumping.
+		module Undumping
+			# Calls Tokens.load with itself.
+			def undump
+				Tokens.load self
+			end
+		end
+
+		# Undump the object using Marshal.load, then
+		# unzip it using GZip.gunzip.
+		# 
+		# The result is commonly a Tokens object, but
+		# this is not guaranteed.
+		def Tokens.load dump
+			require 'coderay/helpers/gzip_simple'
+			dump = dump.gunzip
+			@dump = Marshal.load dump
+		end
+
+	end
+
+
+	# The TokenStream class is a fake Array without elements.
+	# 
+	# It redirects the method << to a block given at creation.
+	#
+	# This allows scanners and Encoders to use streaming (no
+	# tokens are saved, the input is highlighted the same time it
+	# is scanned) with the same code.
+	#
+	# See CodeRay.encode_stream and CodeRay.scan_stream
+	class TokenStream < Tokens
+
+		# Whether the object is a TokenStream.
+		#
+		# Returns true.
+		def stream?
+			true
+		end
+
+		# The Array is empty, but size counts the tokens given by <<.
+		attr_reader :size
+
+		# Creates a new TokenStream that calls +block+ whenever
+		# its << method is called.
+		#
+		# Example:
+		#
+		#   require 'coderay'
+		# 	
+		#   token_stream = CodeRay::TokenStream.new do |kind, text|
+		#     puts 'kind: %s, text size: %d.' % [kind, text.size]
+		#   end
+		#   
+		#   token_stream << [:regexp, '/\d+/']
+		#   #-> kind: rexpexp, text size: 5.
+		#
+		def initialize &block
+			raise ArgumentError, 'Block expected for streaming.' unless block
+			@callback = block
+			@size = 0
+		end
+
+		# Calls +block+ with +token+ and increments size.
+		def << token
+			@callback.call token
+			@size += 1
+		end
+
+		# This method is not implemented due to speed reasons. Use Tokens.
+		def text_size
+			raise NotImplementedError, 'This method is not implemented due to speed reasons.'
+		end
+
+		# A TokenStream cannot be dumped. Use Tokens.
+		def dump
+			raise NotImplementedError, 'A TokenStream cannot be dumped.'
+		end
+
+		# A TokenStream cannot be compacted. Use Tokens.
+		def compact
+			raise NotImplementedError, 'A TokenStream cannot be compacted.'
+		end
+
+	end
+
+end
+
+# vim:sw=2:ts=2:et:tw=78