17 files changed, 876 insertions, 0 deletions
diff --git a/demo/demo_count.rb b/demo/demo_count.rb
new file mode 100644
index 0000000..bcb7c2d
--- /dev/null
+++ b/demo/demo_count.rb
@@ -0,0 +1,10 @@
+require 'coderay'
+
+stats = CodeRay.encoder(:statistic)
+stats.encode("puts 17 + 4\n", :ruby)
+
+puts '%d out of %d tokens have the kind :integer.' % [
+	stats.type_stats[:integer].count,
+	stats.real_token_count
+]
+#-> 2 out of 4 tokens have the kind :integer.
diff --git a/demo/demo_css.rb b/demo/demo_css.rb
new file mode 100644
index 0000000..972bbfa
--- /dev/null
+++ b/demo/demo_css.rb
@@ -0,0 +1,6 @@
+require 'coderay'
+
+data = File.read 'L:\bench\strange.ruby'
+page = CodeRay.scan(data, :ruby).optimize.html(:css => :style, :debug => $DEBUG).page
+
+puts page
diff --git a/demo/demo_div.rb b/demo/demo_div.rb
new file mode 100644
index 0000000..27b6f32
--- /dev/null
+++ b/demo/demo_div.rb
@@ -0,0 +1,19 @@
+require 'coderay'
+
+puts CodeRay.scan(DATA.read, :ruby).div
+
+__END__
+for a in 0..255
+	a = a.chr
+	begin
+		x = eval("?\\#{a}")
+		if x == a[0]
+			next
+		else
+			print "#{a}: #{x}"
+		end
+	rescue SyntaxError => boom
+		print "#{a}: error"
+	end
+	puts
+end
diff --git a/demo/demo_dump.rb b/demo/demo_dump.rb
new file mode 100644
index 0000000..b848dcd
--- /dev/null
+++ b/demo/demo_dump.rb
@@ -0,0 +1,8 @@
+require 'coderay'
+
+puts CodeRay.
+	scan("puts 'Hello, world!'", :ruby).
+	compact.
+	dump.
+	undump.
+	html(:wrap => :div)
diff --git a/demo/demo_encoder.rb b/demo/demo_encoder.rb
new file mode 100644
index 0000000..267676b
--- /dev/null
+++ b/demo/demo_encoder.rb
@@ -0,0 +1,39 @@
+require 'coderay'
+
+SAMPLE = "puts 17 + 4\n"
+puts 'Encoders Demo: ' + SAMPLE
+scanner = CodeRay::Scanners[:ruby].new SAMPLE
+encoder = CodeRay::Encoders[:statistic].new
+
+tokens = scanner.tokenize
+stats = encoder.encode_tokens tokens
+
+puts
+puts 'Statistic:'
+puts stats
+
+# alternative 1
+tokens = CodeRay.scan SAMPLE, :ruby
+encoder = CodeRay.encoder(:tokens)
+textual = encoder.encode_tokens tokens
+puts
+puts 'Original text:'
+puts textual
+
+# alternative 2
+yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby
+puts
+puts 'YAML:'
+puts yaml
+
+# alternative 3
+BIGSAMPLE = SAMPLE * 100
+dump = CodeRay.scan(BIGSAMPLE, :ruby).dump
+puts
+puts 'Dump:'
+p dump
+puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size]
+
+puts
+puts 'Undump:'
+puts dump.undump.statistic
diff --git a/demo/demo_global_vars.rb b/demo/demo_global_vars.rb
new file mode 100644
index 0000000..2bacfe5
--- /dev/null
+++ b/demo/demo_global_vars.rb
@@ -0,0 +1,15 @@
+code = <<'CODE'
+$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
+oder
+text = $bla.test(...) 
+CODE
+
+require 'coderay'
+require 'erb'
+include ERB::Util
+
+tokens = CodeRay.scan code, :ruby
+tokens.each_text_token { |text, kind| text.replace h(text) }
+tokens.each(:global_variable) { |text, kind| text.replace '<span class="glob-var">%s</span>' % text }
+
+puts tokens.text
diff --git a/demo/demo_global_vars2.rb b/demo/demo_global_vars2.rb
new file mode 100644
index 0000000..7646890
--- /dev/null
+++ b/demo/demo_global_vars2.rb
@@ -0,0 +1,28 @@
+require 'coderay'
+require 'erb'
+include ERB::Util
+
+code = <<'CODE'
+$ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
+oder
+text = $bla.test(...) 
+CODE
+puts <<HTML
+<html>
+<head>
+<style>span.glob-var { color: green; font-weight: bold; }</style>
+</head>
+<body>
+HTML
+
+CodeRay.scan_stream code, :ruby do |text, kind|
+	next if text.is_a? Symbol
+	text = h(text)
+	text = '<span class="glob-var">%s</span>' % text if kind == :global_variable
+	print text
+end
+
+puts <<HTML
+</body>
+</html>
+HTML
diff --git a/demo/demo_html.rb b/demo/demo_html.rb
new file mode 100644
index 0000000..d2d25a1
--- /dev/null
+++ b/demo/demo_html.rb
@@ -0,0 +1,395 @@
+$: << '..'
+require 'coderay'
+
+tokens = CodeRay.scan DATA.read, :ruby
+html = tokens.html(:tab_width => 2, :line_numbers => :table)
+
+puts html.page
+
+__END__
+require 'scanner'
+
+module CodeRay
+	
+	class RubyScanner < Scanner
+		
+		RESERVED_WORDS = [
+			'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
+			'defined?', 'ensure', 'module', 'redo', 'super', 'until',
+			'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
+			'when', 'END', 'case', 'else', 'for', 'retry',
+			'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
+			'undef', 'yield',
+		]
+
+		DEF_KEYWORDS = ['def']
+		MODULE_KEYWORDS = ['class', 'module']
+		DEF_NEW_STATE = WordList.new(:initial).
+			add(DEF_KEYWORDS, :def_expected).
+			add(MODULE_KEYWORDS, :module_expected)
+
+		WORDS_ALLOWING_REGEXP = [
+			'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
+		]
+		REGEXP_ALLOWED = WordList.new(false).
+			add(WORDS_ALLOWING_REGEXP, :set)
+		
+		PREDEFINED_CONSTANTS = [
+			'nil', 'true', 'false', 'self',
+			'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
+		]
+
+		IDENT_KIND = WordList.new(:ident).
+			add(RESERVED_WORDS, :reserved).
+			add(PREDEFINED_CONSTANTS, :pre_constant)
+
+		METHOD_NAME = / #{IDENT} [?!]? /xo
+		METHOD_NAME_EX = /
+		 #{METHOD_NAME}  # common methods: split, foo=, empty?, gsub!
+		 | \*\*?         # multiplication and power
+		 | [-+~]@?       # plus, minus
+		 | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
+		 | \[\]=?        # array getter and setter
+		 | <=?>? | >=?   # comparison, rocket operator
+		 | << | >>       # append or shift left, shift right
+		 | ===?          # simple equality and case equality
+		/ox
+		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
+
+		DOUBLEQ = / "  [^"\#\\]*  (?: (?: \#\{.*?\} | \#(?:$")?  | \\. ) [^"\#\\]*  )* "?  /ox
+		SINGLEQ = / '  [^'\\]*    (?:                              \\.   [^'\\]*    )* '?  /ox
+		STRING  = / #{SINGLEQ} | #{DOUBLEQ} /ox
+		SHELL   = / `  [^`\#\\]*  (?: (?: \#\{.*?\} | \#(?:$`)?  | \\. ) [^`\#\\]*  )* `?  /ox
+		REGEXP  = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
+		
+		DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
+		OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+		HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+		BINARY = /0b[01]+(?:_[01]+)*/
+
+		EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+		FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
+		INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
+		
+		def reset
+			super
+			@regexp_allowed = false
+		end
+		
+		def next_token
+			return if @scanner.eos?
+
+			kind = :error
+			if @scanner.scan(/\s+/)  # in every state
+				kind = :space
+				@regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n)  # delayed flag setting
+
+			elsif @state == :def_expected
+				if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
+					kind = :method
+					@state = :initial
+				else
+					@scanner.scan(/./)
+					kind = :error
+				end
+				@state = :initial
+				
+			elsif @state == :module_expected
+				if @scanner.scan(/<</)
+					kind = :operator
+				else
+					if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
+						kind = :method
+					else
+						@scanner.scan(/./)
+						kind = :error
+					end
+					@state = :initial
+				end
+				
+			elsif # state == :initial
+				# IDENTIFIERS, KEYWORDS
+				if @scanner.scan(GLOBAL_VARIABLE)
+					kind = :global_variable
+				elsif @scanner.scan(/ @@ #{IDENT} /ox)
+					kind = :class_variable
+				elsif @scanner.scan(/ @ #{IDENT} /ox)
+					kind = :instance_variable
+				elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
+					kind = :comment
+				elsif @scanner.scan(METHOD_NAME)
+					if @last_token_dot
+						kind = :ident
+					else
+						matched = @scanner.matched
+						kind = IDENT_KIND[matched]
+						if kind == :ident and matched =~ /^[A-Z]/
+							kind = :constant
+						elsif kind == :reserved
+							@state = DEF_NEW_STATE[matched]
+							@regexp_allowed = REGEXP_ALLOWED[matched]
+						end
+					end
+					
+				elsif @scanner.scan(STRING)
+					kind = :string
+				elsif @scanner.scan(SHELL)
+					kind = :shell
+				## HEREDOCS
+				elsif @scanner.scan(/\//) and @regexp_allowed
+				 	@scanner.unscan
+				 	@scanner.scan(REGEXP)
+					kind = :regexp
+				## %strings
+				elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
+					kind = :global_variable
+				elsif @scanner.scan(/
+				  \? (?:
+				    [^\s\\]
+				  | 
+				    \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
+				  )
+				/ox)
+					kind = :integer
+					
+				elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
+					kind = :operator
+					@regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
+				elsif @scanner.scan(FLOAT)
+					kind = :float
+				elsif @scanner.scan(INTEGER)
+					kind = :integer
+				elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
+					kind = :global_variable
+				else
+					@scanner.scan(/./m)
+				end
+			end
+			
+			token = Token.new @scanner.matched, kind
+
+			if kind == :regexp
+				token.text << @scanner.scan(/[eimnosux]*/)
+			end
+			
+			@regexp_allowed = (@regexp_allowed == :set)  # delayed flag setting
+
+			token
+		end
+	end
+	
+	ScannerList.register RubyScanner, 'ruby'
+
+end
+
+module CodeRay
+	require 'scanner'
+
+	class Highlighter
+
+		def initialize lang
+			@scanner = Scanner[lang].new
+		end
+
+		def highlight code
+			@scanner.feed code
+			@scanner.all_tokens.map { |t| t.inspect }.join "\n"
+		end
+
+	end
+
+	class HTMLHighlighter < Highlighter
+		
+		ClassOfKind = {
+			:attribute_name => 'an',
+			:attribute_name_fat => 'af',
+			:attribute_value => 'av',
+			:attribute_value_fat => 'aw',
+			:bin => 'bi',
+	 		:char => 'ch',
+			:class => 'cl',
+			:class_variable => 'cv',
+			:color => 'cr',
+			:comment => 'c',
+			:constant => 'co',
+			:definition => 'df',
+			:directive => 'di',
+			:doc => 'do',
+			:doc_string => 'ds',
+			:exception => 'ex',
+			:error => 'er',
+			:float => 'fl',
+			:function => 'fu',
+			:global_variable => 'gv',
+			:hex => 'hx',
+			:include => 'ic',
+			:instance_variable => 'iv',
+			:integer => 'i',
+			:interpreted => 'in',
+			:label => 'la',
+			:local_variable => 'lv',
+			:oct => 'oc',
+			:operator_name => 'on',
+			:pre_constant => 'pc',
+			:pre_type => 'pt',
+			:predefined => 'pd',
+			:preprocessor => 'pp',
+			:regexp => 'rx',
+			:reserved => 'r',
+			:shell => 'sh',
+			:string => 's',
+			:symbol => 'sy',
+			:tag => 'ta',
+			:tag_fat => 'tf',
+			:tag_special => 'ts',
+			:type => 'ty',
+			:variable => 'v',
+			:xml_text => 'xt',
+
+			:ident => :NO_HIGHLIGHT,
+			:operator => :NO_HIGHLIGHT,
+			:space => :NO_HIGHLIGHT,
+		}
+		ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+		ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
+		
+		def initialize lang, options = {}
+			super lang
+			
+			@HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
+			case level = options.fetch(:level, 'xhtml')
+				when 'html'
+					@HTML_BR = "<BR>\n"
+				when 'xhtml'
+					@HTML_BR = "<br />\n"
+			else
+				raise "Unknown HTML level: #{level}"
+			end
+		end
+
+		def highlight code
+			@scanner.feed code
+			
+			out = ''
+			while t = @scanner.next_token
+				warn t.inspect if t.text.nil?
+				out << to_html(t)
+			end
+			TEMPLATE =~ /<%CONTENT%>/
+			$` + out + $'
+		end
+		
+	private
+		def to_html token
+			css_class = ClassOfKind[token.kind]
+			if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
+				warn "no token class found for :#{token.kind}"
+			end
+				
+			text = text_to_html token.text
+			if css_class == :NO_HIGHLIGHT
+				text
+			else
+				"<span class=\"#{css_class}\">#{text}</span>"
+			end
+		end
+		
+		def text_to_html text
+			return '' if text.empty?
+			text = text.dup  # important
+			if text.index(/["><&]/)
+				text.gsub!('&', '&amp;')
+				text.gsub!('"', '&quot;')
+				text.gsub!('>', '&gt;')
+				text.gsub!('<', '&lt;')
+			end
+			if text.index(/\s/)
+				text.gsub!("\n", @HTML_BR)
+				text.gsub!("\t", @HTML_TAB)
+				text.gsub!(/^ /, '&nbsp;')
+				text.gsub!('  ', ' &nbsp;')
+			end
+			text
+		end
+		
+		TEMPLATE = <<-'TEMPLATE'
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html dir="ltr">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta http-equiv="Content-Style-Type" content="text/css">
+
+<title>RubyBB BBCode</title>
+<style type="text/css">
+.code {
+	width: 100%;
+	background-color: #FAFAFA;
+	border: 1px solid #D1D7DC;
+	font-family: 'Courier New', 'Terminal', monospace;
+	font-size: 10pt;
+	color: black;
+	vertical-align: top;
+	text-align: left;
+}
+.code .af { color:#00C; }
+.code .an { color:#007; }
+.code .av { color:#700; }
+.code .aw { color:#C00; }
+.code .bi { color:#509; font-weight:bold; }
+.code .c  { color:#888; }
+.code .ch { color:#C28; font-weight:bold; }
+.code .cl { color:#B06; font-weight:bold; }
+.code .co { color:#036; font-weight:bold; }
+.code .cr { color:#0A0; }
+.code .cv { color:#369; }
+.code .df { color:#099; font-weight:bold; }
+.code .di { color:#088; font-weight:bold; }
+.code .do { color:#970; }
+.code .ds { color:#D42; font-weight:bold; }
+.code .er { color:#F00; background-color:#FAA; }
+.code .ex { color:#F00; font-weight:bold; }
+.code .fl { color:#60E; font-weight:bold; }
+.code .fu { color:#06B; font-weight:bold; }
+.code .gv { color:#800; font-weight:bold; }
+.code .hx { color:#058; font-weight:bold; }
+.code .i  { color:#00D; font-weight:bold; }
+.code .ic { color:#B44; font-weight:bold; }
+.code .in { color:#B2B; font-weight:bold; }
+.code .iv { color:#33B; }
+.code .la { color:#970; font-weight:bold; }
+.code .lv { color:#963; }
+.code .oc { color:#40E; font-weight:bold; }
+.code .on { color:#000; font-weight:bold; }
+.code .pc { color:#038; font-weight:bold; }
+.code .pd { color:#369; font-weight:bold; }
+.code .pp { color:#579; }
+.code .pt { color:#339; font-weight:bold; }
+.code .r  { color:#080; font-weight:bold; }
+.code .rx { color:#927; font-weight:bold; }
+.code .s  { color:#D42; font-weight:bold; }
+.code .sh { color:#B2B; font-weight:bold; }
+.code .sy { color:#A60; }
+.code .ta { color:#070; }
+.code .tf { color:#070; font-weight:bold; }
+.code .ts { color:#D70; font-weight:bold; }
+.code .ty { color:#339; font-weight:bold; }
+.code .v  { color:#036; }
+.code .xt { color:#444; }
+</style>
+</head>
+<body>
+<div class="code">
+<%CONTENT%>
+</div>
+<div class="validators">
+<a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
+<img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
+</div>    
+</body>
+</html>
+		TEMPLATE
+
+	end
+
+end
+
diff --git a/demo/demo_html2.rb b/demo/demo_html2.rb
new file mode 100644
index 0000000..0982ad8
--- /dev/null
+++ b/demo/demo_html2.rb
@@ -0,0 +1,4 @@
+require 'coderay'
+require 'coderay/encoders/html'
+
+puts CodeRay["puts CodeRay['...', :ruby]", :ruby].div
diff --git a/demo/demo_load_encoder.rb b/demo/demo_load_encoder.rb
new file mode 100644
index 0000000..3c85463
--- /dev/null
+++ b/demo/demo_load_encoder.rb
@@ -0,0 +1,13 @@
+require 'coderay'
+
+begin
+	CodeRay::Encoders::YAML
+rescue
+	puts 'CodeRay::Encoders::YAML is not defined; you must load it first.'
+end
+
+yaml_encoder = CodeRay::Encoders[:yaml] 
+puts 'Now it is loaded.'
+
+p yaml_encoder == CodeRay::Encoders::YAML  #-> true
+puts 'See?'
diff --git a/demo/demo_more.rb b/demo/demo_more.rb
new file mode 100644
index 0000000..7ebf5c3
--- /dev/null
+++ b/demo/demo_more.rb
@@ -0,0 +1,206 @@
+require 'rubygems'
+$: << '..'
+require 'coderay'
+require 'benchmark'
+
+c, ruby = DATA.read.split(/^---$/)
+DATA.rewind
+me = DATA.read[/.*^__END__$/m]
+$input = c + ruby + me
+
+time = Benchmark.realtime do
+
+	# here CodeRay comes to play
+	hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div)
+	c = hl.highlight c, :c
+	ruby = hl.highlight ruby, :ruby
+	me = hl.highlight me, :ruby
+
+	body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code|
+		"<h1>#{title}</h1>\n#{code}"
+	end.join
+	body = hl.class::Output.new(body, :div).page!
+
+	# CodeRay also provides a simple page generator
+	$output = body #hl.class.wrap_in_page body
+end
+
+File.open('test.html', 'w') do |f|
+	f.write $output
+end
+puts 'Input: %dB, Output: %dB' % [$input.size, $output.size]
+puts 'Created "test.html" in %0.3f seconds (%d KB/s). Take a look with your browser.' % [time, $input.size / 1024.0 / time]
+
+__END__
+/**********************************************************************
+
+  version.c -
+
+  $Author: nobu $
+  $Date: 2004/03/25 12:01:40 $
+  created at: Thu Sep 30 20:08:01 JST 1993
+
+  Copyright (C) 1993-2003 Yukihiro Matsumoto
+
+**********************************************************************/
+
+#include "ruby.h"
+#include "version.h"
+#include <stdio.h>
+
+const char ruby_version[] = RUBY_VERSION;
+const char ruby_release_date[] = RUBY_RELEASE_DATE;
+const char ruby_platform[] = RUBY_PLATFORM;
+
+void
+Init_version()
+{
+    VALUE v = rb_obj_freeze(rb_str_new2(ruby_version));
+    VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date));
+    VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform));
+
+    rb_define_global_const("RUBY_VERSION", v);
+    rb_define_global_const("RUBY_RELEASE_DATE", d);
+    rb_define_global_const("RUBY_PLATFORM", p);
+}
+
+void
+ruby_show_version()
+{
+    printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM);
+}
+
+void
+ruby_show_copyright()
+{
+    printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR);
+    exit(0);
+}
+---
+#
+# = ostruct.rb: OpenStruct implementation
+#
+# Author:: Yukihiro Matsumoto
+# Documentation:: Gavin Sinclair
+#
+# OpenStruct allows the creation of data objects with arbitrary attributes.
+# See OpenStruct for an example.
+#
+
+#
+# OpenStruct allows you to create data objects and set arbitrary attributes.
+# For example:
+#
+#   require 'ostruct' 
+#
+#   record = OpenStruct.new
+#   record.name    = "John Smith"
+#   record.age     = 70
+#   record.pension = 300
+#   
+#   puts record.name     # -> "John Smith"
+#   puts record.address  # -> nil
+#
+# It is like a hash with a different way to access the data.  In fact, it is
+# implemented with a hash, and you can initialize it with one.
+#
+#   hash = { "country" => "Australia", :population => 20_000_000 }
+#   data = OpenStruct.new(hash)
+#
+#   p data        # -> <OpenStruct country="Australia" population=20000000>
+#
+class OpenStruct
+  #
+  # Create a new OpenStruct object.  The optional +hash+, if given, will
+  # generate attributes and values.  For example.
+  #
+  #   require 'ostruct'
+  #   hash = { "country" => "Australia", :population => 20_000_000 }
+  #   data = OpenStruct.new(hash)
+  #
+  #   p data        # -> <OpenStruct country="Australia" population=20000000>
+  #
+  # By default, the resulting OpenStruct object will have no attributes. 
+  #
+  def initialize(hash=nil)
+    @table = {}
+    if hash
+      for k,v in hash
+	@table[k.to_sym] = v
+        new_ostruct_member(k)
+      end
+    end
+  end
+
+  # Duplicate an OpenStruct object members. 
+  def initialize_copy(orig)
+    super
+    @table = @table.dup
+  end
+
+  def marshal_dump
+    @table
+  end
+  def marshal_load(x)
+    @table = x
+    @table.each_key{|key| new_ostruct_member(key)}
+  end
+
+  def new_ostruct_member(name)
+    unless self.respond_to?(name)
+      self.instance_eval %{
+        def #{name}; @table[:#{name}]; end
+        def #{name}=(x); @table[:#{name}] = x; end
+      }
+    end
+  end
+
+  def method_missing(mid, *args) # :nodoc:
+    mname = mid.id2name
+    len = args.length
+    if mname =~ /=$/
+      if len != 1
+	raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1)
+      end
+      if self.frozen?
+	raise TypeError, "can't modify frozen #{self.class}", caller(1)
+      end
+      mname.chop!
+      @table[mname.intern] = args[0]
+      self.new_ostruct_member(mname)
+    elsif len == 0
+      @table[mid]
+    else
+      raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1)
+    end
+  end
+
+  #
+  # Remove the named field from the object.
+  #
+  def delete_field(name)
+    @table.delete name.to_sym
+  end
+
+  #
+  # Returns a string containing a detailed summary of the keys and values.
+  #
+  def inspect
+    str = "<#{self.class}"
+    for k,v in @table
+      str << " #{k}=#{v.inspect}"
+    end
+    str << ">"
+  end
+
+  attr_reader :table # :nodoc:
+  protected :table
+
+  #
+  # Compare this object and +other+ for equality.
+  #
+  def ==(other)
+    return false unless(other.kind_of?(OpenStruct))
+    return @table == other.table
+  end
+end
diff --git a/demo/demo_scanner.rb b/demo/demo_scanner.rb
new file mode 100644
index 0000000..a250f91
--- /dev/null
+++ b/demo/demo_scanner.rb
@@ -0,0 +1,12 @@
+require 'coderay'
+c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
+for text, kind in c_scanner
+	print text if kind == :operator
+end
+puts
+
+ruby_scanner = CodeRay::Scanners[:ruby].new %q<c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;">
+
+puts ruby_scanner.any? { |text, kind| kind == :string and text == :open}
+puts ruby_scanner.find { |text, kind| kind == :regexp }
+puts ruby_scanner.map { |text, kind| text if kind != :space }.compact.join(' ')
diff --git a/demo/demo_server.rb b/demo/demo_server.rb
new file mode 100644
index 0000000..44485f0
--- /dev/null
+++ b/demo/demo_server.rb
@@ -0,0 +1,92 @@
+# CodeRay dynamic highlighter
+#
+# Usage: start this and your browser.
+# 
+# Go to http://localhost:49374/?<path to the file>
+# (mnemonic: 49374 = Four-Nine-Three-Seven-Four = For No Token Shall Fall)
+# and you should get the highlighted version.
+
+require 'webrick'
+require 'pathname'
+
+class << File
+	alias dir? directory?
+end
+
+require 'erb'
+include ERB::Util
+def url_decode s
+	s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' }
+end
+
+class String
+	def to_link name = File.basename(self)
+		"<a href=\"?path=#{url_encode self}\">#{name}</a>"
+	end
+end
+
+require 'coderay'
+class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
+
+	STYLE = 'style="font-family: sans-serif; color: navy;"'
+	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="HIghlighted by CodeRay"/></p>'
+
+	def do_GET req, res
+		q = req.query_string || ''
+		args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) }
+		path = args.fetch 'path', '.'
+		
+		backlinks = '<p>current path: %s<br />' % html_escape(path) +
+			(Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' +
+			'.'.to_link('current') + '</p>'
+		
+		res.body = 
+			if File.dir? path
+				path = Pathname.new(path).cleanpath.to_s
+				dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p }
+
+				page = "<html><head></head><body #{STYLE}>"
+				page << backlinks
+				
+				page << '<dl>'
+				page << "<dt>Directories</dt>\n" + dirs.map do |p|
+					"<dd>#{p.to_link}</dd>\n"
+				end.join << "\n"
+				page << "<dt>Files</dt>\n" + files.map do |p|
+					"<dd>#{p.to_link}</dd>\n"
+				end.join << "\n"
+				page << "</dl>\n"
+				page << "#{BANNER}</body></html>"
+			
+			elsif File.exist? path
+				div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div
+				div.replace <<-DIV
+	<div #{STYLE}>
+		#{backlinks}
+#{div}
+	</div>
+	#{BANNER}
+				DIV
+				div.page
+			end
+
+		res['Content-Type'] = 'text/html'
+	end
+end
+
+# 0xCODE = 49374 
+module CodeRay
+	PORT = 0xC0DE
+end
+
+server = WEBrick::HTTPServer.new :Port => CodeRay::PORT
+
+server.mount '/', CodeRayServlet
+
+server.mount_proc '/version' do |req, res|
+	res.body = 'CodeRay::Version = ' + CodeRay::Version
+	res['Content-Type'] = "text/plain"
+end
+
+trap("INT") { server.shutdown }
+server.start
diff --git a/demo/demo_simple.rb b/demo/demo_simple.rb
new file mode 100644
index 0000000..a3129b0
--- /dev/null
+++ b/demo/demo_simple.rb
@@ -0,0 +1,10 @@
+
+# Load CodeRay
+# If this doesn't work, try ruby -rubygems.
+require 'coderay'
+
+# Generate HTML page for Ruby code.
+page = CodeRay.scan("puts 'Hello, world!'", :ruby).span
+
+# Print it
+puts page
diff --git a/demo/demo_stream.rb b/demo/demo_stream.rb
new file mode 100644
index 0000000..b1d8560
--- /dev/null
+++ b/demo/demo_stream.rb
@@ -0,0 +1,8 @@
+$: << '..'
+require 'coderay'
+
+e = CodeRay.encoder(:html)
+t = e.encode_stream('a LOT of :code', :ruby)
+
+puts t
+p t.class
diff --git a/demo/demo_stream2.rb b/demo/demo_stream2.rb
new file mode 100644
index 0000000..8a6bec7
--- /dev/null
+++ b/demo/demo_stream2.rb
@@ -0,0 +1,8 @@
+require 'coderay'
+
+token_stream = CodeRay::TokenStream.new do |kind, text|
+  puts 'kind: %s, text size: %d.' % [kind, text.size]
+end
+
+token_stream << [:regexp, '/\d+/']
+#-> kind: rexpexp, text size: 5.
diff --git a/demo/demo_tokens.rb b/demo/demo_tokens.rb
new file mode 100644
index 0000000..eb8d448
--- /dev/null
+++ b/demo/demo_tokens.rb
@@ -0,0 +1,3 @@
+require 'coderay'
+
+puts CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby).tokens