Lib:

- Encoder: removed a warning
- Encoders::HTML: don't shadow outer variable
- Plugin: move require_plugin into class namespace
- Ruby Scanner:
  - "alias" keyword recognition
  - better regexp/division distinction
  - recognize ~, !, !=, and !~ as method names (partly Ruby 1.9 only)
  - reordered states for speed

Tests:
- updated coderay-suite to use gem instead of require_gem
- general improvements (more colors!, new parameter: new, new syntax lang.test for only and new)
- fixed ruby suite
- adjusted a lot of Ruby tests (alias uses methods now)
- new tests: ruby/operators, ruby/regexp

Samples:
- fixed/updated ('bout time)

Rake tasks:
- updated to use new rubygems API

9 files changed, 236 insertions, 71 deletions

diff --git a/test/scanners/coderay_suite.rb b/test/scanners/coderay_suite.rb
index 122d9cd..04b385a 100644
--- a/test/scanners/coderay_suite.rb
+++ b/test/scanners/coderay_suite.rb
@@ -12,7 +12,8 @@ begin
 rescue LoadError
   begin
     require 'rubygems'
-    require_gem 'term-ansicolor'
+    gem 'term-ansicolor'
+    require 'term/ansicolor'
   rescue LoadError
     # ignore
   end
@@ -21,6 +22,9 @@ end unless ENV['nocolor']
 if defined? Term::ANSIColor
   class String
     include Term::ANSIColor
+    def green_or_red result
+      result ? green : red
+    end
   end
 else
   class String
@@ -31,6 +35,9 @@ else
         end
       END
     end
+    def green_or_red result
+      result ? upcase : downcase
+    end
   end
 end
 $DEBUG = debug
@@ -91,15 +98,15 @@ module Enumerable
 end
 
 module CodeRay
-
+  
   require 'test/unit'
-
+  
   class TestCase < Test::Unit::TestCase
     
     if ENV['deluxe']
-      MAX_CODE_SIZE_TO_HIGHLIGHT = 200_000
-      MAX_CODE_SIZE_TO_TEST = 1_000_000
-      DEFAULT_MAX = 512
+      MAX_CODE_SIZE_TO_HIGHLIGHT = 5_000_000
+      MAX_CODE_SIZE_TO_TEST = 5_000_000
+      DEFAULT_MAX = 1024
     else
       MAX_CODE_SIZE_TO_HIGHLIGHT = 20_000
       MAX_CODE_SIZE_TO_TEST = 100_000
@@ -110,7 +117,7 @@ module CodeRay
       def inherited child
         CodeRay::TestSuite << child.suite
       end
-
+      
       # Calls its block with the working directory set to the examples
       # for this test case.
       def dir
@@ -119,11 +126,11 @@ module CodeRay
           yield
         end
       end
-
+      
       def lang
         @lang ||= name.downcase.to_sym
       end
-
+      
       def extension extension = nil
         if extension
           @extension = extension.to_s
@@ -155,13 +162,13 @@ module CodeRay
       
       puts 'Finished in '.green + '%0.2fs'.white % time_for_lang + '.'.green
     end
-
+    
     def examples_test scanner, max
       self.class.dir do
         extension = 'in.' + self.class.extension
         for example_filename in Dir["*.#{extension}"]
           name = File.basename(example_filename, ".#{extension}")
-          next if ENV['only'] and ENV['only'] != name
+          next if ENV['lang'] && ENV['only'] && ENV['only'] != name
           print name_and_size = ('%15s'.cyan + ' %4.0fK: '.yellow) %
             [ name, File.size(example_filename) / 1024.0 ]
           time_for_file = Benchmark.realtime do
@@ -179,16 +186,16 @@ module CodeRay
         return
       end
       code = File.open(example_filename, 'rb') { |f| break f.read }
-    
+      
       incremental_test scanner, code, max unless ENV['noincremental']
-
+      
       unless ENV['noshuffled'] or code.size < [0].pack('Q').size
         shuffled_test scanner, code, max
       else
         print '-skipped- '.concealed
       end
-
-      tokens = compare_test scanner, code, name
+      
+      tokens = complete_test scanner, code, name
       
       identity_test scanner, tokens
       
@@ -200,51 +207,73 @@ module CodeRay
     end
     
     def random_test scanner, max
-      print "Random test...".red
+      print "Random test...".yellow
+      ok = true
       for size in (0..max).progress
         srand size + 17
         scanner.string = Array.new(size) { rand 256 }.pack 'c*'
-        scanner.tokenize
+        begin
+          scanner.tokenize
+        rescue
+          flunk "Random test failed at #{size} #{RUBY_VERSION < '1.9' ? 'bytes' : 'chars'}!" unless ENV['noassert']
+          ok = false
+          break
+        end
       end
-      print "\b\b\b"
+      print "\b" * 'Random test...'.size
+      print 'Random test'.green_or_red(ok)
       puts ' - finished'.green
     end
     
     def incremental_test scanner, code, max
-      print 'incremental...'.red
+      print 'incremental...'.yellow
+      ok = true
       for size in (0..max).progress
         break if size > code.size
         scanner.string = code[0,size]
-        scanner.tokenize
+        begin
+          scanner.tokenize
+        rescue
+          flunk "Incremental test failed at #{size} #{RUBY_VERSION < '1.9' ? 'bytes' : 'chars'}!" unless ENV['noassert']
+          ok = false
+          break
+        end
       end
-      print "\b\b\b"
-      print ', '.red
+      print "\b" * 'incremental...'.size
+      print 'incremental, '.green_or_red(ok)
     end
-
+    
     def shuffled_test scanner, code, max
-      print 'shuffled...'.red
+      print 'shuffled...'.yellow
       code_bits = code[0,max].unpack('Q*')  # split into quadwords...
+      ok = true
       for i in (0..max / 4).progress
         srand i
         code_bits.shuffle!                     # ...mix...
         scanner.string = code_bits.pack('Q*')  # ...and join again
-        scanner.tokenize
+        begin
+          scanner.tokenize
+        rescue
+          flunk 'shuffle test failed!' unless ENV['noassert']
+          ok = false
+          break
+        end
       end
-
+      
       # highlighted = highlighter.encode_tokens scanner.tokenize
       # File.open(name + '.shuffled.html', 'w') { |f| f.write highlighted }
-      print "\b\b\b"
-      print ', '.red
+      print "\b" * 'shuffled...'.size
+      print 'shuffled, '.green_or_red(ok)
     end
     
-    def compare_test scanner, code, name
-      print 'complete...'.red
+    def complete_test scanner, code, name
+      print 'complete...'.yellow
       expected_filename = name + '.expected.' + Tokenizer.file_extension
       scanner.string = code
       tokens = scanner.tokens
       result = Tokenizer.encode_tokens tokens
-
-      if File.exist? expected_filename
+      
+      if File.exist?(expected_filename) && !(ENV['lang'] && ENV['new'] && name == ENV['new'])
         expected = File.open(expected_filename, 'rb') { |f| break f.read }
         ok = expected == result
         actual_filename = expected_filename.sub('.expected.', '.actual.')
@@ -259,67 +288,91 @@ module CodeRay
         unless ENV['noassert']
           assert(ok, "Scan error: unexpected output".red)
         end
+        print "\b" * 'complete...'.size
+        print 'complete, '.green_or_red(ok)
       else
-        print "\b" * 'complete...'.size, "new test..."
         File.open(expected_filename, 'wb') { |f| f.write result }
+        print "\b" * 'complete...'.size, "new test, ".blue
       end
       
-      print "\b\b\b"
-      print ', '.red
-      
       tokens
     end
     
     def identity_test scanner, tokens
-      print 'identity...'.red
-      unless scanner.instance_of? CodeRay::Scanners[:debug]
-        assert_equal scanner.code, tokens.text
+      print 'identity...'.yellow
+      if scanner.instance_of? CodeRay::Scanners[:debug]
+        ok = true
+      else
+        ok = scanner.code == tokens.text
+        unless ok
+          flunk 'identity test failed!' unless ENV['noassert']
+        end
       end
-      print "\b\b\b"
-      print ', '.red
+      print "\b" * 'identity...'.size
+      print 'identity, '.green_or_red(ok)
     end
     
     Highlighter = CodeRay::Encoders[:html].new(
       :tab_width => 2,
-      :line_numbers => :inline,
+      :line_numbers => :table,
       :wrap => :page,
       :hint => :debug,
       :css => :class
     )
-
+    
     def highlight_test tokens, name
-      print 'highlighting, '.red
-      highlighted = Highlighter.encode_tokens tokens
-      File.open(name + '.actual.html', 'w') { |f| f.write highlighted }      
+      print 'highlighting...'.yellow
+      ok = true
+      begin
+        highlighted = Highlighter.encode_tokens tokens
+      rescue
+        flunk 'highlighting test failed!' unless ENV['noassert']
+        ok = false
+        break
+      end
+      File.open(name + '.actual.html', 'w') { |f| f.write highlighted }
+      print "\b" * 'highlighting...'.size
+      print 'highlighting, '.green_or_red(ok)
     end
-
+    
   end
   
   require 'test/unit/testsuite'
-
+  
   class TestSuite
     @suite = Test::Unit::TestSuite.new 'CodeRay::Scanners'
     class << self
-
+      
       def << sub_suite
         @suite << sub_suite
       end
-
+      
       def load_suite name
         begin
           suite = File.join($mydir, name, 'suite.rb')
           require suite
         rescue LoadError
           $stderr.puts <<-ERR
-
+          
       !! Suite #{suite} not found
-
+          
           ERR
           false
         end
       end
-
+      
+      def check_env_lang
+        for key in %w(only new)
+          if ENV[key] && ENV[key][/^(\w+)\.([\w_]+)$/]
+            ENV['lang'] = $1
+            ENV[key] = $2
+          end
+        end
+      end
+      
       def load
+        ENV['only'] = ENV['new'] if ENV['new']
+        check_env_lang
         subsuite = ARGV.find { |a| break $& if a[/^[^-].*/] } || ENV['lang']
         if subsuite
           load_suite(subsuite) or exit
@@ -329,7 +382,7 @@ module CodeRay
           end
         end
       end
-
+      
       def run
         load
         $VERBOSE = true
@@ -339,5 +392,5 @@ module CodeRay
       end
     end
   end
-
+  
 end
\ No newline at end of file
diff --git a/test/scanners/ruby/evil.expected.raydebug b/test/scanners/ruby/evil.expected.raydebug
index cca15dd..84ac855 100644
--- a/test/scanners/ruby/evil.expected.raydebug
+++ b/test/scanners/ruby/evil.expected.raydebug
@@ -44,10 +44,10 @@ ident(p)operator(()pre_constant(false) operator(::) ident(to_s)operator(\))
 reserved(class) class(C2)
 reserved(class) operator(<<)class(self)
   reserved(def) pre_constant(self)operator(.)ident(p8)operator(;) ident(p) integer(8) reserved(end)
-  reserved(alias) ident(p?) ident(p)
-  reserved(alias) ident(P?) ident(p)
-  reserved(alias) operator([)operator(]) ident(p)
-  reserved(alias) operator(<=>) ident(p)
+  reserved(alias) method(p?) method(p)
+  reserved(alias) method(P?) method(p)
+  reserved(alias) method([]) method(p)
+  reserved(alias) method(<=>) method(p)
 reserved(end)
 
 ident(q)operator(=)integer(9)
@@ -881,7 +881,7 @@ reserved(class) operator(<<)class(a)
   reserved(def) method(foobar)
      pre_constant(self)operator(*)integer(101)
   reserved(end)
-  reserved(alias)    ident(eql?)    operator(==)
+  reserved(alias)    method(eql?)    method(==)
 reserved(end)
 
 ident(p) ident(a)operator(.)ident(foobar)
diff --git a/test/scanners/ruby/example.expected.raydebug b/test/scanners/ruby/example.expected.raydebug
index 492a67a..5a684e6 100644
--- a/test/scanners/ruby/example.expected.raydebug
+++ b/test/scanners/ruby/example.expected.raydebug
@@ -217,7 +217,7 @@ reserved(class) class(Set)
   reserved(def) method(size)
     instance_variable(@hash)operator(.)ident(size)
   reserved(end)
-  reserved(alias) ident(length) ident(size)
+  reserved(alias) method(length) method(size)
 
   comment(# Returns true if the set contains no elements.)
   reserved(def) method(empty?)
@@ -288,7 +288,7 @@ reserved(class) class(Set)
   reserved(def) method(include?)operator(()ident(o)operator(\))
     instance_variable(@hash)operator(.)ident(include?)operator(()ident(o)operator(\))
   reserved(end)
-  reserved(alias) ident(member?) ident(include?)
+  reserved(alias) method(member?) method(include?)
 
   comment(# Returns true if the set is a superset of the given set.)
   reserved(def) method(superset?)operator(()ident(set)operator(\))
@@ -331,7 +331,7 @@ reserved(class) class(Set)
     instance_variable(@hash)operator([)ident(o)operator(]) operator(=) pre_constant(true)
     pre_constant(self)
   reserved(end)
-  reserved(alias) operator(<<) ident(add)
+  reserved(alias) method(<<) method(add)
 
   comment(# Adds the given object to the set and returns self.  If the)
   comment(# object is already in the set, returns nil.)
@@ -373,7 +373,7 @@ reserved(class) class(Set)
     ident(each) operator({) operator(|)ident(o)operator(|) ident(set) operator(<<) reserved(yield)operator(()ident(o)operator(\)) operator(})
     ident(replace)operator(()ident(set)operator(\))
   reserved(end)
-  reserved(alias) ident(map!) ident(collect!)
+  reserved(alias) method(map!) method(collect!)
 
   comment(# Equivalent to Set#delete_if, but returns nil if no changes were)
   comment(# made.)
@@ -410,8 +410,8 @@ reserved(class) class(Set)
     ident(enum)operator(.)ident(is_a?)operator(()constant(Enumerable)operator(\)) reserved(or) ident(raise) constant(ArgumentError)operator(,) string<delimiter(")content(value must be enumerable)delimiter(")>
     ident(dup)operator(.)ident(merge)operator(()ident(enum)operator(\))
   reserved(end)
-  reserved(alias) operator(+) operator(|)		comment(##)
-  reserved(alias) ident(union) operator(|)		comment(##)
+  reserved(alias) method(+) method(|)		comment(##)
+  reserved(alias) method(union) method(|)		comment(##)
 
   comment(# Returns a new set built by duplicating the set, removing every)
   comment(# element that appears in the given enumerable object.)
@@ -419,7 +419,7 @@ reserved(class) class(Set)
     ident(enum)operator(.)ident(is_a?)operator(()constant(Enumerable)operator(\)) reserved(or) ident(raise) constant(ArgumentError)operator(,) string<delimiter(")content(value must be enumerable)delimiter(")>
     ident(dup)operator(.)ident(subtract)operator(()ident(enum)operator(\))
   reserved(end)
-  reserved(alias) ident(difference) operator(-)	comment(##)
+  reserved(alias) method(difference) method(-)	comment(##)
 
   comment(# Returns a new array containing elements common to the set and the)
   comment(# given enumerable object.)
@@ -429,7 +429,7 @@ reserved(class) class(Set)
     ident(enum)operator(.)ident(each) operator({) operator(|)ident(o)operator(|) ident(n)operator(.)ident(add)operator(()ident(o)operator(\)) reserved(if) ident(include?)operator(()ident(o)operator(\)) operator(})
     ident(n)
   reserved(end)
-  reserved(alias) ident(intersection) operator(&)	comment(##)
+  reserved(alias) method(intersection) method(&)	comment(##)
 
   comment(# Returns a new array containing elements exclusive between the set)
   comment(# and the given enumerable object.  (set ^ enum\) is equivalent to)
@@ -508,7 +508,7 @@ reserved(class) class(Set)
       reserved(class) operator(<<) class(dig) operator(=) operator({)operator(})		comment(# :nodoc:)
 	ident(include) constant(TSort)
 
-	reserved(alias) ident(tsort_each_node) ident(each_key)
+	reserved(alias) method(tsort_each_node) method(each_key)
 	reserved(def) method(tsort_each_child)operator(()ident(node)operator(,) operator(&)ident(block)operator(\))
 	  ident(fetch)operator(()ident(node)operator(\))operator(.)ident(each)operator(()operator(&)ident(block)operator(\))
 	reserved(end)
diff --git a/test/scanners/ruby/operators.expected.raydebug b/test/scanners/ruby/operators.expected.raydebug
new file mode 100644
index 0000000..56d9ac3
--- /dev/null
+++ b/test/scanners/ruby/operators.expected.raydebug
@@ -0,0 +1,51 @@
+reserved(class) class(Feeling)
+  
+  reserved(def) method(~)
+    ident(p) symbol(:drunk)
+  reserved(end)
+  
+  reserved(def) method(!)
+    ident(p) symbol(:alert)
+  reserved(end)
+  
+  reserved(alias) method(not) method(!@)
+  reserved(alias) method(tilde) method(~@)
+  
+  reserved(def) method(-@)
+    ident(p) symbol(:bad)
+  reserved(end)
+  
+  reserved(def) method(+@)
+    ident(p) symbol(:good)
+  reserved(end)
+  
+reserved(end)
+
+ident(feeling) operator(=) constant(Feeling)operator(.)ident(new)
+
+operator(-)ident(feeling)  comment(# => :bad)
+operator(+)ident(feeling)  comment(# => :good)
+operator(!)ident(feeling)  comment(# => :alert)
+operator(~)ident(feeling)  comment(# => :drunk)
+
+reserved(def) method(=~) ident(other)
+  ident(bla)
+reserved(end)
+
+ident(feeling)operator(.)ident(!)  comment(# => :alert)
+ident(feeling)operator(.)ident(~)  comment(# => :drunk)
+ident(feeling)operator(.)ident(!@)  comment(# => :alert)
+ident(feeling)operator(.)ident(~@)  comment(# => :drunk)
+ident(feeling)operator(.)ident(-@)operator(()operator(\))  comment(# => :bad)
+ident(feeling)operator(.)ident(+@)operator(()operator(\))  comment(# => :good)
+
+comment(# >> :bad)
+comment(# >> :good)
+comment(# >> :alert)
+comment(# >> :drunk)
+comment(# >> :alert)
+comment(# >> :drunk)
+comment(# >> :alert)
+comment(# >> :drunk)
+comment(# >> :bad)
+comment(# >> :good)
diff --git a/test/scanners/ruby/operators.in.rb b/test/scanners/ruby/operators.in.rb
new file mode 100644
index 0000000..8fa7090
--- /dev/null
+++ b/test/scanners/ruby/operators.in.rb
@@ -0,0 +1,51 @@
+class Feeling
+  
+  def ~
+    p :drunk
+  end
+  
+  def !
+    p :alert
+  end
+  
+  alias not !@
+  alias tilde ~@
+  
+  def -@
+    p :bad
+  end
+  
+  def +@
+    p :good
+  end
+  
+end
+
+feeling = Feeling.new
+
+-feeling  # => :bad
++feeling  # => :good
+!feeling  # => :alert
+~feeling  # => :drunk
+
+def =~ other
+  bla
+end
+
+feeling.!  # => :alert
+feeling.~  # => :drunk
+feeling.!@  # => :alert
+feeling.~@  # => :drunk
+feeling.-@()  # => :bad
+feeling.+@()  # => :good
+
+# >> :bad
+# >> :good
+# >> :alert
+# >> :drunk
+# >> :alert
+# >> :drunk
+# >> :alert
+# >> :drunk
+# >> :bad
+# >> :good
diff --git a/test/scanners/ruby/pleac.expected.raydebug b/test/scanners/ruby/pleac.expected.raydebug
index b1e7762..0a9a6ac 100644
--- a/test/scanners/ruby/pleac.expected.raydebug
+++ b/test/scanners/ruby/pleac.expected.raydebug
@@ -4021,7 +4021,7 @@ comment(#=> bar)
 comment(# You can also take a reference to an existing method before)
 comment(# redefining a new one, using the `alias' keyword)
 reserved(def) method(foo)operator(;) ident(puts) string<delimiter(')content(foo)delimiter(')>operator(;) reserved(end)
-reserved(alias) ident(foo_orig) ident(foo)
+reserved(alias) method(foo_orig) method(foo)
 reserved(def) method(foo)operator(;) ident(puts) string<delimiter(')content(bar)delimiter(')>operator(;) reserved(end)
 ident(foo_orig)
 ident(foo)
diff --git a/test/scanners/ruby/regexp.expected.raydebug b/test/scanners/ruby/regexp.expected.raydebug
new file mode 100644
index 0000000..43f1d91
--- /dev/null
+++ b/test/scanners/ruby/regexp.expected.raydebug
@@ -0,0 +1,5 @@
+comment(# Regexp or division?)
+ident(some_string)operator(.)ident(to_i) regexp<delimiter(/)char(\\s)content(+)delimiter(/)>
+ident(some_string)operator(.)ident(split) operator(/) operator(+)regexp<delimiter(/)content(  this is a regexp after a division )delimiter(/)>
+ident(some_string)operator(.)ident(split) operator(/) operator(+) regexp<delimiter(/)content(  this one, too )delimiter(/)>
+ident(some_string)operator(.)ident(split) regexp<delimiter(/)content(- )delimiter(/)>  comment(# and this one is a regexp without division)
\ No newline at end of file
diff --git a/test/scanners/ruby/regexp.in.rb b/test/scanners/ruby/regexp.in.rb
new file mode 100644
index 0000000..956e6b8
--- /dev/null
+++ b/test/scanners/ruby/regexp.in.rb
@@ -0,0 +1,5 @@
+# Regexp or division?
+some_string.to_i /\s+/
+some_string.split / +/  this is a regexp after a division /
+some_string.split / + /  this one, too /
+some_string.split /- /  # and this one is a regexp without division
\ No newline at end of file
diff --git a/test/scanners/ruby/suite.rb b/test/scanners/ruby/suite.rb
index ca390f5..bcc5702 100644
--- a/test/scanners/ruby/suite.rb
+++ b/test/scanners/ruby/suite.rb
@@ -1,2 +1,2 @@
-class Scheme < CodeRay::TestCase
+class Ruby < CodeRay::TestCase
 end