Improved UTF-8 support for Ruby Scanner. Also fixed a minor bug.

* closes #108 (new Ruby 1.9 call operator syntax sugar)
* Added an example for unicode code.
* automatic UTF-8 detection (experimental)
* Still problems with different Ruby versions; new unicode test fails in
  Ruby 1.9 and JRuby.

4 files changed, 68 insertions, 0 deletions

diff --git a/test/scanners/ruby/ruby19.expected.raydebug b/test/scanners/ruby/ruby19.expected.raydebug
new file mode 100644
index 0000000..f3e40d5
--- /dev/null
+++ b/test/scanners/ruby/ruby19.expected.raydebug
@@ -0,0 +1,4 @@
+ident(block)operator(.)ident(()operator(*)ident(arguments)operator(\))  comment(# bovi's example)
+
+reserved(def) operator(()ident(foo)operator(\))operator(.)ident(bar)
+reserved(end)
\ No newline at end of file
diff --git a/test/scanners/ruby/ruby19.in.rb b/test/scanners/ruby/ruby19.in.rb
new file mode 100644
index 0000000..45ec5f9
--- /dev/null
+++ b/test/scanners/ruby/ruby19.in.rb
@@ -0,0 +1,4 @@
+block.(*arguments)  # bovi's example
+
+def (foo).bar
+end
\ No newline at end of file
diff --git a/test/scanners/ruby/unicode.expected.raydebug b/test/scanners/ruby/unicode.expected.raydebug
new file mode 100644
index 0000000..dad2a74
--- /dev/null
+++ b/test/scanners/ruby/unicode.expected.raydebug
@@ -0,0 +1,30 @@
+ident(ä) operator(=) integer(42)
+ident(print) ident(ä)
+
+reserved(def) method(straße)operator(()ident(frühstück)operator(\))
+  ident(höhle)operator(()ident(frühstück)operator(\))
+reserved(end)
+
+reserved(alias) method(λ) method(lambda)
+ident(×) operator(=) ident(λ)operator({) operator(|)ident(x)operator(,)ident(y)operator(|) ident(x)operator(*)ident(y)operator(})
+ident(×)operator([)integer(2)operator(,)integer(3)operator(])  comment(# => 6)
+
+comment(# Summe der ersten 10 Quadratzahlen)
+reserved(def) method(∑) ident(enum)
+  ident(enum)operator(.)ident(inject)operator(()integer(0)operator(\)) operator({) operator(|)ident(sum)operator(,) ident(x)operator(|) ident(sum) operator(+) reserved(yield)operator(()ident(x)operator(\)) operator(})
+reserved(end)
+
+ident(∑)operator(()integer(1)operator(..)integer(10)operator(\)) operator({) operator(|)ident(x)operator(|) ident(x)operator(**)integer(2) operator(})  comment(# => 385)
+
+comment(# mehr Mathematische Zeichen)
+reserved(def) method(∞)operator(;) float(1.0) operator(/) float(0.0)operator(;) reserved(end)
+reserved(def) method(π)operator(;) constant(Math)operator(::)constant(PI)operator(;) reserved(end)
+
+operator(-)ident(∞) operator(..) integer(2)operator(*)ident(π)  comment(# => -Infinity..6.28318530717959)
+
+comment(# Azumanga Daioh Insider)
+reserved(class) operator(<<) class(Osaka) operator(=) constant(Object)operator(.)ident(new)
+  reserved(def) method(ぁ!)
+    ident(sleep) ident(∞)
+  reserved(end)
+reserved(end)
\ No newline at end of file
diff --git a/test/scanners/ruby/unicode.in.rb b/test/scanners/ruby/unicode.in.rb
new file mode 100644
index 0000000..5474072
--- /dev/null
+++ b/test/scanners/ruby/unicode.in.rb
@@ -0,0 +1,30 @@
+ä = 42
+print ä
+
+def straße(frühstück)
+  höhle(frühstück)
+end
+
+alias λ lambda
+× = λ{ |x,y| x*y}
+×[2,3]  # => 6
+
+# Summe der ersten 10 Quadratzahlen
+def ∑ enum
+  enum.inject(0) { |sum, x| sum + yield(x) }
+end
+
+∑(1..10) { |x| x**2 }  # => 385
+
+# mehr Mathematische Zeichen
+def ∞; 1.0 / 0.0; end
+def π; Math::PI; end
+
+-∞ .. 2*π  # => -Infinity..6.28318530717959
+
+# Azumanga Daioh Insider
+class << Osaka = Object.new
+  def ぁ!
+    sleep ∞
+  end
+end
\ No newline at end of file