Added lots of unit tests.

Theses actually come from the library files; now they are included. Also, rake test and test:all don't test the scanners now; you have to start them using rake test:scanners.

17 files changed, 842 insertions, 0 deletions

diff --git a/test/unit/comment_filter.rb b/test/unit/comment_filter.rb
new file mode 100644
index 0000000..e255d07
--- /dev/null
+++ b/test/unit/comment_filter.rb
@@ -0,0 +1,53 @@
+require 'test/unit'
+require 'coderay'
+
+class CommentFilterTest < Test::Unit::TestCase
+  
+  def test_filtering_comments
+    tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
+#!/usr/bin/env ruby
+
+puts "Hello world!"
+    RUBY_FILTERED
+  end
+  
+  def test_filtering_docstrings
+    tokens = CodeRay.scan <<-PYTHON, :python
+'''
+Assuming this is file mymodule.py then this string, being the
+first statement in the file will become the mymodule modules
+docstring when the file is imported
+'''
+
+class Myclass():
+    """
+    The class's docstring
+    """
+
+    def mymethod(self):
+        '''The method's docstring'''
+
+def myfunction():
+    """The function's docstring"""
+    PYTHON
+    assert_equal <<-PYTHON_FILTERED.chomp, tokens.comment_filter.text
+
+
+class Myclass():
+    
+
+    def mymethod(self):
+        
+
+def myfunction():
+    
+
+PYTHON_FILTERED
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/count.rb b/test/unit/count.rb
new file mode 100644
index 0000000..ad61291
--- /dev/null
+++ b/test/unit/count.rb
@@ -0,0 +1,15 @@
+require 'test/unit'
+require 'coderay'
+
+class CountTest < Test::Unit::TestCase
+  
+  def test_count
+    tokens = CodeRay.scan <<-RUBY.strip, :ruby
+#!/usr/bin/env ruby
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal 11, tokens.encode_with(:count)
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/debug.rb b/test/unit/debug.rb
new file mode 100644
index 0000000..8bafcf5
--- /dev/null
+++ b/test/unit/debug.rb
@@ -0,0 +1,76 @@
+require 'test/unit'
+require 'coderay'
+
+class DebugEncoderTest < Test::Unit::TestCase
+  
+  def test_creation
+    debug = nil
+    assert_nothing_raised do
+      debug = CodeRay.encoder :debug
+    end
+    assert CodeRay::Encoders::Debug < CodeRay::Encoders::Encoder
+    assert_kind_of CodeRay::Encoders::Encoder, debug
+  end
+  
+  TEST_INPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n", :space],
+    ["\n  \t", :space],
+    ["   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  TEST_OUTPUT = <<-'DEBUG'.chomp
+integer(10)operator((\\\))string<content(test)>test[
+
+  	   
+method([])]
+  DEBUG
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Encoders::Debug.new.encode_tokens(TEST_INPUT)
+    assert_equal TEST_OUTPUT, TEST_INPUT.debug
+  end
+  
+end
+
+class DebugScannerTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Scanners::Debug < CodeRay::Scanners::Scanner
+    debug = nil
+    assert_nothing_raised do
+      debug = CodeRay.scanner :debug
+    end
+    assert_kind_of CodeRay::Scanners::Scanner, debug
+  end
+  
+  TEST_INPUT = <<-'DEBUG'.chomp
+integer(10)operator((\\\))string<content(test)>test[
+
+  	   
+method([])]
+  DEBUG
+  TEST_OUTPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n\n  \t   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Scanners::Debug.new.tokenize(TEST_INPUT)
+    assert_equal TEST_OUTPUT, CodeRay.scan(TEST_INPUT, :debug)
+  end
+  
+end
diff --git a/test/unit/duo.rb b/test/unit/duo.rb
new file mode 100644
index 0000000..a433c3e
--- /dev/null
+++ b/test/unit/duo.rb
@@ -0,0 +1,45 @@
+require 'test/unit'
+require 'coderay'
+
+class DuoTest < Test::Unit::TestCase
+  
+  def test_two_arguments
+    duo = CodeRay::Duo[:ruby, :html]
+    assert_kind_of CodeRay::Scanners[:ruby], duo.scanner
+    assert_kind_of CodeRay::Encoders[:html], duo.encoder
+  end
+  
+  def test_two_hash
+    duo = CodeRay::Duo[:ruby => :html]
+    assert_kind_of CodeRay::Scanners[:ruby], duo.scanner
+    assert_kind_of CodeRay::Encoders[:html], duo.encoder
+  end
+  
+  def test_call
+    duo = CodeRay::Duo[:python => :yaml]
+    assert_equal <<-'YAML', duo.call('def test: "pass"')
+--- 
+- - def
+  - :keyword
+- - " "
+  - :space
+- - test
+  - :method
+- - ":"
+  - :operator
+- - " "
+  - :space
+- - :begin_group
+  - :string
+- - "\""
+  - :delimiter
+- - pass
+  - :content
+- - "\""
+  - :delimiter
+- - :end_group
+  - :string
+    YAML
+  end
+  
+end
diff --git a/test/unit/file_type.rb b/test/unit/file_type.rb
new file mode 100644
index 0000000..4558850
--- /dev/null
+++ b/test/unit/file_type.rb
@@ -0,0 +1,122 @@
+require 'test/unit'
+require 'coderay/helpers/file_type'
+
+class FileTypeTests < Test::Unit::TestCase
+  
+  include CodeRay
+  
+  def test_fetch
+    assert_raise FileType::UnknownFileType do
+      FileType.fetch ''
+    end
+
+    assert_throws :not_found do
+      FileType.fetch '.' do
+        throw :not_found
+      end
+    end
+
+    assert_equal :default, FileType.fetch('c', :default)
+  end
+  
+  def test_block_supersedes_default_warning
+    if defined? JRUBY_VERSION
+      $stderr.puts 'Skipped test because of JRUBY bug.'
+      return
+    end
+    stderr, fake_stderr = $stderr, Object.new
+    begin
+      $err = ''
+      def fake_stderr.write x
+        $err << x
+      end
+      $stderr = fake_stderr
+      FileType.fetch('c', :default) { }
+      assert_equal "Block supersedes default value argument; use either.\n", $err
+    ensure
+      $stderr = stderr
+    end
+  end
+
+  def test_ruby
+    assert_equal :ruby, FileType['test.rb']
+    assert_equal :ruby, FileType['test.java.rb']
+    assert_equal :java, FileType['test.rb.java']
+    assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
+    assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
+    assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
+    assert_equal :ruby, FileType['./lib/tasks\repository.rake']
+    assert_not_equal :ruby, FileType['test_rb']
+    assert_not_equal :ruby, FileType['Makefile']
+    assert_not_equal :ruby, FileType['set.rb/set']
+    assert_not_equal :ruby, FileType['~/projects/blabla/rb']
+  end
+
+  def test_c
+    assert_equal :c, FileType['test.c']
+    assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
+    assert_not_equal :c, FileType['test_c']
+    assert_not_equal :c, FileType['Makefile']
+    assert_not_equal :c, FileType['set.h/set']
+    assert_not_equal :c, FileType['~/projects/blabla/c']
+  end
+
+  def test_cpp
+    assert_equal :cpp, FileType['test.c++']
+    assert_equal :cpp, FileType['test.cxx']
+    assert_equal :cpp, FileType['test.hh']
+    assert_equal :cpp, FileType['test.hpp']
+    assert_equal :cpp, FileType['test.cu']
+    assert_equal :cpp, FileType['test.C']
+    assert_not_equal :cpp, FileType['test.c']
+    assert_not_equal :cpp, FileType['test.h']
+  end
+
+  def test_html
+    assert_equal :html, FileType['test.htm']
+    assert_equal :xhtml, FileType['test.xhtml']
+    assert_equal :xhtml, FileType['test.html.xhtml']
+    assert_equal :rhtml, FileType['_form.rhtml']
+    assert_equal :rhtml, FileType['_form.html.erb']
+  end
+
+  def test_yaml
+    assert_equal :yaml, FileType['test.yml']
+    assert_equal :yaml, FileType['test.yaml']
+    assert_equal :yaml, FileType['my.html.yaml']
+    assert_not_equal :yaml, FileType['YAML']
+  end
+
+  def test_pathname
+    require 'pathname'
+    pn = Pathname.new 'test.rb'
+    assert_equal :ruby, FileType[pn]
+    dir = Pathname.new '/etc/var/blubb'
+    assert_equal :ruby, FileType[dir + pn]
+    assert_equal :cpp, FileType[dir + 'test.cpp']
+  end
+
+  def test_no_shebang
+    dir = './test'
+    if File.directory? dir
+      Dir.chdir dir do
+        assert_equal :c, FileType['test.c']
+      end
+    end
+  end
+  
+  def test_shebang_empty_file
+    require 'tmpdir'
+    tmpfile = File.join(Dir.tmpdir, 'bla')
+    File.open(tmpfile, 'w') { }  # touch
+    assert_equal nil, FileType[tmpfile]
+  end
+  
+  def test_shebang
+    require 'tmpdir'
+    tmpfile = File.join(Dir.tmpdir, 'bla')
+    File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
+    assert_equal :ruby, FileType[tmpfile, true]
+  end
+
+end
diff --git a/test/unit/filter.rb b/test/unit/filter.rb
new file mode 100644
index 0000000..25dff77
--- /dev/null
+++ b/test/unit/filter.rb
@@ -0,0 +1,38 @@
+require 'test/unit'
+require 'coderay'
+
+class FilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :filter
+    end
+    assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
+    assert_kind_of CodeRay::Encoders::Encoder, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens.text_token i.to_s, :index
+    end
+    assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
+    assert_equal tokens, tokens.filter
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens.begin_group :index
+      tokens.text_token i.to_s, :content
+      tokens.end_group :index
+      tokens.begin_line :index
+      tokens.text_token i.to_s, :content
+      tokens.end_line :index
+    end
+    assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
+    assert_equal tokens, tokens.filter
+  end
+  
+end
diff --git a/test/unit/json.rb b/test/unit/json.rb
new file mode 100644
index 0000000..4e44a64
--- /dev/null
+++ b/test/unit/json.rb
@@ -0,0 +1,28 @@
+require 'test/unit'
+require 'coderay'
+
+class JSONEncoderTest < Test::Unit::TestCase
+  
+  def test_json_output
+    old_load_paths = $:.dup
+    begin
+      $:.delete '.'
+      $:.delete File.dirname(__FILE__)
+      json = CodeRay.scan('puts "Hello world!"', :ruby).json
+      assert_equal [
+        {"type"=>"text", "text"=>"puts", "kind"=>"ident"},
+        {"type"=>"text", "text"=>" ", "kind"=>"space"},
+        {"type"=>"block", "action"=>"open", "kind"=>"string"},
+        {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+        {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
+        {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+        {"type"=>"block", "action"=>"close", "kind"=>"string"},
+      ], JSON.load(json)
+    ensure
+      for path in old_load_paths - $:
+        $: << path
+      end
+    end
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/lines_of_code.rb b/test/unit/lines_of_code.rb
new file mode 100644
index 0000000..a76fed6
--- /dev/null
+++ b/test/unit/lines_of_code.rb
@@ -0,0 +1,63 @@
+require 'test/unit'
+require 'coderay'
+$VERBOSE = true
+
+class LinesOfCodeTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :loc
+    end
+    assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+    assert_nothing_raised do
+      filter = CodeRay.encoder :lines_of_code
+    end
+    assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+  end
+  
+  def test_lines_of_code
+    tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+    assert_equal 1, tokens.lines_of_code
+    assert_equal 1, tokens.loc
+  end
+  
+  class ScannerMockup
+    KINDS_NOT_LOC = [:space]
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    tokens.concat ["Hello\n", :world]
+    tokens.concat ["\n", :space]
+    tokens.concat ["Hello\n", :comment]
+    
+    if defined? JRUBY_VERSION
+      $stderr.puts 'Skipped test because of JRUBY bug.'
+    else
+      stderr, fake_stderr = $stderr, Object.new
+      begin
+        $err = ''
+        def fake_stderr.write x
+          $err << x
+        end
+        $stderr = fake_stderr
+        assert_equal 1, tokens.lines_of_code
+        assert_equal "Tokens have no associated scanner, counting all nonempty lines.\n", $err
+      ensure
+        $stderr = stderr
+      end
+    end
+    
+    tokens.scanner = ScannerMockup.new
+    assert_equal 2, tokens.lines_of_code
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/null.rb b/test/unit/null.rb
new file mode 100644
index 0000000..ea516d8
--- /dev/null
+++ b/test/unit/null.rb
@@ -0,0 +1,14 @@
+require 'test/unit'
+require 'coderay'
+
+class NullTest < Test::Unit::TestCase
+  
+  def test_null
+    ruby = <<-RUBY
+puts "Hello world!"
+    RUBY
+    tokens = CodeRay.scan ruby, :ruby
+    assert_equal '', tokens.encode_with(:null)
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/plugin.rb b/test/unit/plugin.rb
new file mode 100755
index 0000000..25bbc93
--- /dev/null
+++ b/test/unit/plugin.rb
@@ -0,0 +1,11 @@
+require 'test/unit'
+require 'coderay'
+
+class PluginScannerTest < Test::Unit::TestCase
+  
+  def test_load
+    require File.join(File.dirname(__FILE__), 'vhdl')
+    assert_equal 'VHDL', CodeRay.scanner(:vhdl).class.name
+  end
+  
+end
diff --git a/test/unit/statistic.rb b/test/unit/statistic.rb
new file mode 100644
index 0000000..36385b0
--- /dev/null
+++ b/test/unit/statistic.rb
@@ -0,0 +1,57 @@
+require 'test/unit'
+require 'coderay'
+
+class StatisticEncoderTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder
+    stats = nil
+    assert_nothing_raised do
+      stats = CodeRay.encoder :statistic
+    end
+    assert_kind_of CodeRay::Encoders::Encoder, stats
+  end
+  
+  TEST_INPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n", :space],
+    ["\n  \t", :space],
+    ["   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  TEST_OUTPUT = <<-'DEBUG'
+
+Code Statistics
+
+Tokens                  11
+  Non-Whitespace         4
+Bytes Total             20
+
+Token Types (5):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                       11  100.00 %     1.8
+  space                        3   27.27 %     3.0
+  begin_group                  1    9.09 %     0.0
+  begin_line                   1    9.09 %     0.0
+  content                      1    9.09 %     4.0
+  end_group                    1    9.09 %     0.0
+  end_line                     1    9.09 %     0.0
+  integer                      1    9.09 %     2.0
+  method                       1    9.09 %     2.0
+  operator                     1    9.09 %     3.0
+
+  DEBUG
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT)
+    assert_equal TEST_OUTPUT, TEST_INPUT.statistic
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/suite.rb b/test/unit/suite.rb
new file mode 100755
index 0000000..845c5b2
--- /dev/null
+++ b/test/unit/suite.rb
@@ -0,0 +1,13 @@
+require 'test/unit'
+$:.unshift 'lib'
+
+MYDIR = File.dirname(__FILE__)
+suite = Dir[File.join(MYDIR, '*.rb')].
+  map { |tc| File.basename(tc).sub(/\.rb$/, '') } - %w'suite vhdl'
+
+puts "Running CodeRay unit tests: #{suite.join(', ')}"
+
+helpers = %w(file_type word_list tokens)
+for test_case in helpers + (suite - helpers)
+  load File.join(MYDIR, test_case + '.rb')
+end
diff --git a/test/unit/text.rb b/test/unit/text.rb
new file mode 100644
index 0000000..025881e
--- /dev/null
+++ b/test/unit/text.rb
@@ -0,0 +1,14 @@
+require 'test/unit'
+require 'coderay'
+
+class TextTest < Test::Unit::TestCase
+  
+  def test_count
+    ruby = <<-RUBY
+puts "Hello world!"
+    RUBY
+    tokens = CodeRay.scan ruby, :ruby
+    assert_equal ruby, tokens.encode_with(:text)
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/token_kind_filter.rb b/test/unit/token_kind_filter.rb
new file mode 100644
index 0000000..13bae52
--- /dev/null
+++ b/test/unit/token_kind_filter.rb
@@ -0,0 +1,50 @@
+require 'test/unit'
+require 'coderay'
+
+class TokenKindFilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :token_kind_filter
+    end
+    assert_instance_of CodeRay::Encoders::TokenKindFilter, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    for i in 1..10
+      tokens.text_token i.to_s, :index
+      tokens.text_token ' ', :space if i < 10
+    end
+    assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).count
+    assert_equal 10, tokens.token_kind_filter(:exclude => :space).count
+    assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).count
+    assert_equal 9, tokens.token_kind_filter(:include => :space).count
+    assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).count
+    assert_equal 0, tokens.token_kind_filter(:exclude => :all).count
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens.begin_group :index
+      tokens.text_token i.to_s, :content
+      tokens.end_group :index
+      tokens.begin_group :naught if i == 5
+      tokens.end_group :naught if i == 7
+      tokens.begin_line :blubb
+      tokens.text_token i.to_s, :content
+      tokens.end_line :blubb
+    end
+    assert_equal 16, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).count
+    assert_equal 16, tokens.token_kind_filter(:include => :blubb).count
+    assert_equal 24, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => [:blubb, :content]).count
+    assert_equal 24, tokens.token_kind_filter(:include => [:blubb, :content]).count
+    assert_equal 32, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).count
+    assert_equal 32, tokens.token_kind_filter(:exclude => :index).count
+  end
+  
+end
diff --git a/test/unit/tokens.rb b/test/unit/tokens.rb
new file mode 100644
index 0000000..ae6d25a
--- /dev/null
+++ b/test/unit/tokens.rb
@@ -0,0 +1,38 @@
+require 'test/unit'
+require 'coderay/tokens'
+
+class TokensTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Tokens < Array
+    tokens = nil
+    assert_nothing_raised do
+      tokens = CodeRay::Tokens.new
+    end
+    assert_kind_of Array, tokens
+  end
+  
+  def test_adding_tokens
+    tokens = CodeRay::Tokens.new
+    assert_nothing_raised do
+      tokens.text_token 'string', :type
+      tokens.text_token '()', :operator
+    end
+    assert_equal tokens.size, 4
+    assert_equal tokens.count, 2
+  end
+  
+  def test_dump_undump
+    tokens = CodeRay::Tokens.new
+    assert_nothing_raised do
+      tokens.text_token 'string', :type
+      tokens.text_token '()', :operator
+    end
+    tokens2 = nil
+    assert_nothing_raised do
+      tokens2 = tokens.dump.undump
+    end
+    assert_equal tokens, tokens2
+  end
+  
+end
\ No newline at end of file
diff --git a/test/unit/vhdl.rb b/test/unit/vhdl.rb
new file mode 100644
index 0000000..3b8262b
--- /dev/null
+++ b/test/unit/vhdl.rb
@@ -0,0 +1,126 @@
+class VHDL < CodeRay::Scanners::Scanner
+
+  register_for :vhdl
+
+  RESERVED_WORDS = [
+    'access','after','alias','all','assert','architecture','begin',
+    'block','body','buffer','bus','case','component','configuration','constant',
+    'disconnect','downto','else','elsif','end','entity','exit','file','for',
+    'function','generate','generic','group','guarded','if','impure','in',
+    'inertial','inout','is','label','library','linkage','literal','loop',
+    'map','new','next','null','of','on','open','others','out','package',
+    'port','postponed','procedure','process','pure','range','record','register',
+    'reject','report','return','select','severity','signal','shared','subtype',
+    'then','to','transport','type','unaffected','units','until','use','variable',
+    'wait','when','while','with','note','warning','error','failure','and',
+    'or','xor','not','nor',
+    'array'
+  ]
+
+  PREDEFINED_TYPES = [
+    'bit','bit_vector','character','boolean','integer','real','time','string',
+    'severity_level','positive','natural','signed','unsigned','line','text',
+    'std_logic','std_logic_vector','std_ulogic','std_ulogic_vector','qsim_state',
+    'qsim_state_vector','qsim_12state','qsim_12state_vector','qsim_strength',
+    'mux_bit','mux_vector','reg_bit','reg_vector','wor_bit','wor_vector'
+  ]
+
+  PREDEFINED_CONSTANTS = [
+
+  ]
+
+  IDENT_KIND = CodeRay::CaseIgnoringWordList.new(:ident).
+    add(RESERVED_WORDS, :reserved).
+    add(PREDEFINED_TYPES, :pre_type).
+    add(PREDEFINED_CONSTANTS, :pre_constant)
+
+  ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+  UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+  def scan_tokens tokens, options
+
+    state = :initial
+
+    until eos?
+
+      kind = nil
+      match = nil
+
+      case state
+
+      when :initial
+
+        if scan(/ \s+ | \\\n /x)
+          kind = :space
+
+        elsif scan(/-- .*/x)
+          kind = :comment
+
+        elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
+          kind = :operator
+
+        elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+          kind = IDENT_KIND[match.downcase]
+
+        elsif match = scan(/[a-z]?"/i)
+          tokens << [:open, :string]
+          state = :string
+          kind = :delimiter
+
+        elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+          kind = :char
+
+        elsif scan(/(?:\d+)(?![.eEfF])/)
+          kind = :integer
+
+        elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+          kind = :float
+
+        else
+          getch
+          kind = :error
+
+        end
+
+      when :string
+        if scan(/[^\\\n"]+/)
+          kind = :content
+        elsif scan(/"/)
+          tokens << ['"', :delimiter]
+          tokens << [:close, :string]
+          state = :initial
+          next
+        elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+          kind = :char
+        elsif scan(/ \\ | $ /x)
+          tokens << [:close, :string]
+          kind = :error
+          state = :initial
+        else
+          raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+        end
+
+      else
+        raise_inspect 'Unknown state', tokens
+
+      end
+
+      match ||= matched
+      if $CODERAY_DEBUG and not kind
+        raise_inspect 'Error token %p in line %d' %
+          [[match, kind], line], tokens
+      end
+      raise_inspect 'Empty token', tokens unless match
+
+      tokens << [match, kind]
+
+    end
+
+    if state == :string
+      tokens << [:close, :string]
+    end
+
+    tokens
+  end
+
+end
diff --git a/test/unit/word_list.rb b/test/unit/word_list.rb
new file mode 100644
index 0000000..449c9dc
--- /dev/null
+++ b/test/unit/word_list.rb
@@ -0,0 +1,79 @@
+require 'test/unit'
+require 'coderay/helpers/word_list'
+
+class WordListTest < Test::Unit::TestCase
+  
+  include CodeRay
+  
+  # define word arrays
+  RESERVED_WORDS = %w[
+    asm break case continue default do else
+    ...
+  ]
+  
+  PREDEFINED_TYPES = %w[
+    int long short char void
+    ...
+  ]
+  
+  PREDEFINED_CONSTANTS = %w[
+    EOF NULL ...
+  ]
+  
+  # make a WordList
+  IDENT_KIND = WordList.new(:ident).
+    add(RESERVED_WORDS, :reserved).
+    add(PREDEFINED_TYPES, :pre_type).
+    add(PREDEFINED_CONSTANTS, :pre_constant)
+
+  def test_word_list_example
+    assert_equal :pre_type, IDENT_KIND['void']
+    # assert_equal :pre_constant, IDENT_KIND['...']  # not specified
+  end
+  
+  def test_word_list
+    list = WordList.new(:ident).add(['foobar'], :reserved)
+    assert_equal :reserved, list['foobar']
+    assert_equal :ident, list['FooBar']
+  end
+
+  def test_word_list_cached
+    list = WordList.new(:ident, true).add(['foobar'], :reserved)
+    assert_equal :reserved, list['foobar']
+    assert_equal :ident, list['FooBar']
+  end
+
+  def test_case_ignoring_word_list
+    list = CaseIgnoringWordList.new(:ident).add(['foobar'], :reserved)
+    assert_equal :ident, list['foo']
+    assert_equal :reserved, list['foobar']
+    assert_equal :reserved, list['FooBar']
+
+    list = CaseIgnoringWordList.new(:ident).add(['FooBar'], :reserved)
+    assert_equal :ident, list['foo']
+    assert_equal :reserved, list['foobar']
+    assert_equal :reserved, list['FooBar']
+  end
+
+  def test_case_ignoring_word_list_cached
+    list = CaseIgnoringWordList.new(:ident, true).add(['foobar'], :reserved)
+    assert_equal :ident, list['foo']
+    assert_equal :reserved, list['foobar']
+    assert_equal :reserved, list['FooBar']
+
+    list = CaseIgnoringWordList.new(:ident, true).add(['FooBar'], :reserved)
+    assert_equal :ident, list['foo']
+    assert_equal :reserved, list['foobar']
+    assert_equal :reserved, list['FooBar']
+  end
+
+  def test_dup
+    list = WordList.new(:ident).add(['foobar'], :reserved)
+    assert_equal :reserved, list['foobar']
+    list2 = list.dup
+    list2.add(%w[foobar], :keyword)
+    assert_equal :keyword, list2['foobar']
+    assert_equal :reserved, list['foobar']
+  end
+
+end
\ No newline at end of file