multicore benchmarks are at https://github.com/rubychan/stringscanner-benchmarks

10 files changed, 21 insertions, 6763 deletions

diff --git a/etc/CodeRay.tmproj b/etc/CodeRay.tmproj
index 5da0eeb..af6886f 100644
--- a/etc/CodeRay.tmproj
+++ b/etc/CodeRay.tmproj
@@ -26,7 +26,7 @@
 			<key>filename</key>
 			<string>../diff</string>
 			<key>lastUsed</key>
-			<date>2011-05-08T07:53:59Z</date>
+			<date>2011-05-21T05:54:27Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
@@ -38,7 +38,7 @@
 			<key>filename</key>
 			<string>../Changes.textile</string>
 			<key>lastUsed</key>
-			<date>2011-05-08T09:37:51Z</date>
+			<date>2011-05-21T04:50:20Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
@@ -91,6 +91,8 @@
 			<string>../rake_helpers</string>
 		</dict>
 		<dict>
+			<key>expanded</key>
+			<true/>
 			<key>name</key>
 			<string>rake_tasks</string>
 			<key>regexFolderFilter</key>
@@ -102,7 +104,7 @@
 			<key>filename</key>
 			<string>../Rakefile</string>
 			<key>lastUsed</key>
-			<date>2010-11-21T14:08:49Z</date>
+			<date>2011-05-23T23:35:00Z</date>
 		</dict>
 		<dict>
 			<key>name</key>
@@ -157,7 +159,22 @@
 	<key>fileHierarchyDrawerWidth</key>
 	<integer>204</integer>
 	<key>metaData</key>
-	<dict/>
+	<dict>
+		<key>../diff</key>
+		<dict>
+			<key>caret</key>
+			<dict>
+				<key>column</key>
+				<integer>0</integer>
+				<key>line</key>
+				<integer>0</integer>
+			</dict>
+			<key>firstVisibleColumn</key>
+			<integer>0</integer>
+			<key>firstVisibleLine</key>
+			<integer>3</integer>
+		</dict>
+	</dict>
 	<key>showFileHierarchyDrawer</key>
 	<true/>
 	<key>windowFrame</key>
diff --git a/etc/multicore/multi-pos.rb b/etc/multicore/multi-pos.rb
deleted file mode 100644
index 0772e65..0000000
--- a/etc/multicore/multi-pos.rb
+++ /dev/null
@@ -1,164 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code, range = 0...code.bytesize
-    super code
-    @start = range.begin
-    @stop  = range.end
-  end
-  
-  def tokenize encoder = Tokens.new
-    self.pos = @start
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if matched = scan(/ +/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/\n/)
-        if pos >= @stop
-          # puts "stopped @ #{pos}"
-          return
-        end
-        encoder.text_token matched, :space
-      elsif matched = scan(/!/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/=/)  #/
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/%/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/\d+/)
-        encoder.text_token matched, :number
-      elsif matched = scan(/\w+/)
-        encoder.text_token matched, :word
-      elsif matched = scan(/[,.]/)
-        encoder.text_token matched, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Tokens < Array
-  alias token push
-  alias text_token push
-  alias block_token push
-  def begin_group kind; push :begin_group, kind end
-  def end_group kind; push :end_group, kind end
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-    @opened = []
-  end
-  
-  def finish
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  # def token content, kind
-  #   if content.is_a? ::String
-  #     text_token content, kind
-  #   elsif content.is_a? ::Symbol
-  #     block_token content, kind
-  #   else
-  #     raise 'Unknown token content type: %p' % [content]
-  #   end
-  # end
-  
-  def text_token text, kind
-    # @out <<
-    #   if kind == :space
-    #     text
-    #   else
-    #     text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-    #     "#{kind}(#{text})"
-    #   end
-  end
-  
-  # def block_token action, kind
-  #   case action
-  #   when :begin_group
-  #     begin_group kind
-  #   when :end_group
-  #     end_group kind
-  #   else
-  #     raise
-  #   end
-  # end
-  
-  def begin_group kind
-    # @opened << kind
-    # @out << "#{kind}<"
-  end
-  
-  def end_group kind
-    # @opened.pop
-    # @out << '>'
-  end
-  
-end
-
-size = (ARGV.first || 1).to_i * 1_000_000  # size in MB
-
-# generate string
-code = "2011 alpha, beta, (gamma), delta.\n"
-code *= (size.to_f / code.size).ceil
-code.slice! size..-1
-
-slice_size = (ARGV[1] || 100).to_i
-
-3.times do
-2.times do
-  threads = []
-  seconds = Benchmark.realtime do
-    chunk_offsets = [0]
-    code.lines.each_slice slice_size do |lines|
-      chunk_offsets << chunk_offsets.last + lines.join.bytesize
-    end
-    # p chunk_offsets.size - 1
-    chunk_offsets.each_cons(2) do |this_chunk, next_chunk|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode Scanner.new(code, this_chunk...next_chunk)
-      end
-    end
-    threads.each(&:join)
-    out = threads.map { |t| t[:out] }.join
-  end
-  
-  mb = size / 1_000_000.0
-  puts 'Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads' % [mb, seconds, mb / seconds, threads.size]
-end
-
-2.times do
-  seconds = Benchmark.realtime do
-    out = Encoder.new.encode(Scanner.new(code))
-  end
-  
-  mb = size / 1_000_000.0
-  puts 'Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s' % [mb, seconds, mb / seconds]
-end
-end
\ No newline at end of file
diff --git a/etc/multicore/multi-pos.rbc b/etc/multicore/multi-pos.rbc
deleted file mode 100644
index dc60d2e..0000000
--- a/etc/multicore/multi-pos.rbc
+++ /dev/null
@@ -1,2989 +0,0 @@
-!RBIX
-2959558172506603432
-x
-M
-1
-n
-n
-x
-10
-__script__
-i
-214
-5
-7
-0
-64
-47
-49
-1
-1
-15
-5
-7
-2
-64
-47
-49
-1
-1
-15
-5
-7
-3
-64
-47
-49
-1
-1
-15
-99
-7
-4
-45
-5
-6
-65
-49
-7
-3
-13
-99
-12
-7
-8
-12
-7
-9
-12
-65
-12
-49
-10
-4
-15
-49
-8
-0
-15
-99
-7
-11
-45
-12
-13
-65
-49
-7
-3
-13
-99
-12
-7
-8
-12
-7
-14
-12
-65
-12
-49
-10
-4
-15
-49
-8
-0
-15
-99
-7
-15
-1
-65
-49
-7
-3
-13
-99
-12
-7
-8
-12
-7
-16
-12
-65
-12
-49
-10
-4
-15
-49
-8
-0
-15
-45
-17
-18
-49
-19
-0
-13
-10
-123
-15
-79
-49
-20
-0
-7
-21
-49
-22
-1
-19
-0
-15
-7
-23
-64
-19
-1
-15
-20
-1
-20
-0
-49
-24
-0
-20
-1
-49
-25
-0
-49
-26
-1
-49
-27
-0
-49
-22
-1
-19
-1
-15
-20
-1
-44
-43
-28
-20
-0
-77
-49
-29
-2
-49
-30
-1
-15
-45
-17
-31
-79
-49
-32
-1
-13
-10
-192
-15
-4
-100
-49
-20
-0
-19
-2
-15
-80
-56
-33
-50
-34
-0
-15
-80
-56
-35
-50
-34
-0
-15
-2
-11
-I
-9
-I
-3
-I
-0
-I
-0
-n
-p
-36
-s
-7
-strscan
-x
-7
-require
-s
-9
-benchmark
-s
-6
-thread
-x
-7
-Scanner
-x
-13
-StringScanner
-n
-x
-10
-open_class
-x
-14
-__class_init__
-M
-1
-n
-n
-x
-7
-Scanner
-i
-48
-5
-66
-99
-7
-0
-7
-1
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-4
-7
-5
-65
-67
-49
-2
-0
-49
-3
-4
-15
-5
-48
-6
-15
-99
-7
-7
-7
-8
-65
-67
-49
-2
-0
-49
-3
-4
-11
-I
-5
-I
-0
-I
-0
-I
-0
-n
-p
-9
-x
-10
-initialize
-M
-1
-n
-n
-x
-10
-initialize
-i
-43
-23
-1
-10
-20
-44
-43
-0
-78
-20
-0
-49
-1
-0
-2
-49
-2
-3
-19
-1
-15
-20
-0
-54
-52
-3
-1
-15
-20
-1
-49
-4
-0
-38
-5
-15
-20
-1
-49
-6
-0
-38
-7
-11
-I
-6
-I
-2
-I
-1
-I
-2
-n
-p
-8
-x
-5
-Range
-x
-8
-bytesize
-x
-3
-new
-x
-10
-initialize
-x
-5
-begin
-x
-6
-@start
-x
-3
-end
-x
-5
-@stop
-p
-9
-I
--1
-I
-7
-I
-14
-I
-8
-I
-1b
-I
-9
-I
-23
-I
-a
-I
-2b
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-2
-x
-4
-code
-x
-5
-range
-x
-17
-method_visibility
-x
-15
-add_defn_method
-x
-8
-tokenize
-M
-1
-n
-n
-x
-8
-tokenize
-i
-54
-23
-0
-10
-31
-45
-0
-1
-13
-71
-2
-47
-9
-25
-47
-49
-3
-0
-13
-47
-49
-4
-0
-15
-8
-28
-49
-2
-0
-19
-0
-15
-5
-39
-5
-13
-18
-2
-47
-49
-6
-1
-15
-15
-5
-20
-0
-47
-49
-7
-1
-15
-20
-0
-11
-I
-4
-I
-1
-I
-0
-I
-1
-n
-p
-8
-x
-6
-Tokens
-n
-x
-3
-new
-x
-8
-allocate
-x
-10
-initialize
-x
-6
-@start
-x
-4
-pos=
-x
-11
-scan_tokens
-p
-11
-I
--1
-I
-d
-I
-1f
-I
-9c
-I
-20
-I
-e
-I
-2b
-I
-f
-I
-33
-I
-10
-I
-36
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-7
-encoder
-x
-9
-protected
-x
-11
-scan_tokens
-M
-1
-n
-n
-x
-11
-scan_tokens
-i
-404
-5
-47
-49
-0
-0
-10
-402
-5
-7
-1
-13
-70
-9
-26
-15
-44
-43
-2
-7
-3
-78
-49
-4
-2
-6
-1
-47
-49
-5
-1
-19
-1
-9
-45
-20
-0
-20
-1
-7
-6
-49
-7
-2
-8
-398
-5
-7
-8
-13
-70
-9
-64
-15
-44
-43
-2
-7
-9
-78
-49
-4
-2
-6
-8
-47
-49
-5
-1
-19
-1
-9
-99
-20
-0
-20
-1
-7
-6
-49
-7
-2
-15
-5
-48
-10
-39
-11
-49
-12
-1
-9
-96
-1
-11
-8
-97
-1
-8
-398
-5
-7
-13
-13
-70
-9
-118
-15
-44
-43
-2
-7
-14
-78
-49
-4
-2
-6
-13
-47
-49
-5
-1
-19
-1
-9
-137
-20
-0
-20
-1
-7
-15
-49
-7
-2
-8
-398
-5
-7
-16
-13
-70
-9
-156
-15
-44
-43
-2
-7
-17
-78
-49
-4
-2
-6
-16
-47
-49
-5
-1
-19
-1
-9
-175
-20
-0
-20
-1
-7
-15
-49
-7
-2
-8
-398
-5
-7
-18
-13
-70
-9
-194
-15
-44
-43
-2
-7
-19
-78
-49
-4
-2
-6
-18
-47
-49
-5
-1
-19
-1
-9
-213
-20
-0
-20
-1
-7
-15
-49
-7
-2
-8
-398
-5
-7
-20
-13
-70
-9
-232
-15
-44
-43
-2
-7
-21
-78
-49
-4
-2
-6
-20
-47
-49
-5
-1
-19
-1
-9
-251
-20
-0
-20
-1
-7
-22
-49
-7
-2
-8
-398
-5
-7
-23
-13
-70
-9
-270
-15
-44
-43
-2
-7
-24
-78
-49
-4
-2
-6
-23
-47
-49
-5
-1
-19
-1
-9
-289
-20
-0
-20
-1
-7
-25
-49
-7
-2
-8
-398
-5
-7
-26
-13
-70
-9
-308
-15
-44
-43
-2
-7
-27
-78
-49
-4
-2
-6
-26
-47
-49
-5
-1
-19
-1
-9
-327
-20
-0
-20
-1
-7
-28
-49
-7
-2
-8
-398
-5
-7
-29
-13
-70
-9
-346
-15
-44
-43
-2
-7
-30
-78
-49
-4
-2
-6
-29
-47
-49
-5
-1
-9
-361
-20
-0
-7
-31
-49
-32
-1
-8
-398
-5
-7
-33
-13
-70
-9
-380
-15
-44
-43
-2
-7
-34
-78
-49
-4
-2
-6
-33
-47
-49
-5
-1
-9
-395
-20
-0
-7
-31
-49
-35
-1
-8
-398
-5
-48
-36
-15
-68
-8
-0
-1
-11
-I
-6
-I
-2
-I
-1
-I
-1
-n
-p
-37
-x
-4
-eos?
-n
-x
-6
-Regexp
-s
-2
- +
-x
-3
-new
-x
-4
-scan
-x
-5
-space
-x
-10
-text_token
-n
-s
-2
-\n
-x
-3
-pos
-x
-5
-@stop
-x
-2
->=
-n
-s
-1
-!
-x
-19
-not_going_to_happen
-n
-s
-1
-=
-n
-s
-1
-%
-n
-s
-3
-\d+
-x
-6
-number
-n
-s
-3
-\w+
-x
-4
-word
-n
-s
-4
-[,.]
-x
-2
-op
-n
-s
-2
-\(
-x
-3
-par
-x
-11
-begin_group
-n
-s
-2
-\)
-x
-9
-end_group
-x
-5
-raise
-p
-53
-I
--1
-I
-15
-I
-0
-I
-16
-I
-7
-I
-17
-I
-22
-I
-18
-I
-2d
-I
-19
-I
-48
-I
-1a
-I
-52
-I
-1b
-I
-5c
-I
-1d
-I
-60
-I
-1b
-I
-63
-I
-1f
-I
-7e
-I
-20
-I
-89
-I
-21
-I
-a4
-I
-22
-I
-af
-I
-23
-I
-ca
-I
-24
-I
-d5
-I
-25
-I
-f0
-I
-26
-I
-fb
-I
-27
-I
-116
-I
-28
-I
-121
-I
-29
-I
-13c
-I
-2a
-I
-147
-I
-2b
-I
-160
-I
-2c
-I
-169
-I
-2d
-I
-182
-I
-2e
-I
-18b
-I
-30
-I
-194
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-2
-x
-7
-encoder
-x
-7
-matched
-p
-9
-I
-2
-I
-7
-I
-10
-I
-d
-I
-1e
-I
-13
-I
-22
-I
-15
-I
-30
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-0
-x
-13
-attach_method
-x
-6
-Tokens
-x
-5
-Array
-n
-M
-1
-n
-n
-x
-6
-Tokens
-i
-60
-5
-66
-65
-7
-0
-7
-1
-47
-49
-2
-2
-15
-65
-7
-3
-7
-1
-47
-49
-2
-2
-15
-65
-7
-4
-7
-1
-47
-49
-2
-2
-15
-99
-7
-5
-7
-6
-65
-67
-49
-7
-0
-49
-8
-4
-15
-99
-7
-9
-7
-10
-65
-67
-49
-7
-0
-49
-8
-4
-11
-I
-5
-I
-0
-I
-0
-I
-0
-n
-p
-11
-x
-5
-token
-x
-4
-push
-x
-12
-alias_method
-x
-10
-text_token
-x
-11
-block_token
-x
-11
-begin_group
-M
-1
-n
-n
-x
-11
-begin_group
-i
-10
-5
-7
-0
-20
-0
-47
-49
-1
-2
-11
-I
-4
-I
-1
-I
-1
-I
-1
-n
-p
-2
-x
-11
-begin_group
-x
-4
-push
-p
-3
-I
--1
-I
-3c
-I
-a
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-4
-kind
-x
-17
-method_visibility
-x
-15
-add_defn_method
-x
-9
-end_group
-M
-1
-n
-n
-x
-9
-end_group
-i
-10
-5
-7
-0
-20
-0
-47
-49
-1
-2
-11
-I
-4
-I
-1
-I
-1
-I
-1
-n
-p
-2
-x
-9
-end_group
-x
-4
-push
-p
-3
-I
--1
-I
-3d
-I
-a
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-4
-kind
-p
-11
-I
-2
-I
-39
-I
-c
-I
-3a
-I
-16
-I
-3b
-I
-20
-I
-3c
-I
-2e
-I
-3d
-I
-3c
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-0
-x
-7
-Encoder
-M
-1
-n
-n
-x
-7
-Encoder
-i
-86
-5
-66
-99
-7
-0
-7
-1
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-4
-7
-5
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-6
-7
-7
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-8
-7
-9
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-10
-7
-11
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-12
-7
-13
-65
-67
-49
-2
-0
-49
-3
-4
-11
-I
-5
-I
-0
-I
-0
-I
-0
-n
-p
-14
-x
-5
-setup
-M
-1
-n
-n
-x
-5
-setup
-i
-11
-7
-0
-64
-38
-1
-15
-35
-0
-38
-2
-11
-I
-1
-I
-0
-I
-0
-I
-0
-n
-p
-3
-s
-0
-
-x
-4
-@out
-x
-7
-@opened
-p
-7
-I
--1
-I
-43
-I
-0
-I
-44
-I
-6
-I
-45
-I
-b
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-0
-x
-17
-method_visibility
-x
-15
-add_defn_method
-x
-6
-finish
-M
-1
-n
-n
-x
-6
-finish
-i
-3
-39
-0
-11
-I
-1
-I
-0
-I
-0
-I
-0
-n
-p
-1
-x
-4
-@out
-p
-5
-I
--1
-I
-48
-I
-0
-I
-49
-I
-3
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-0
-x
-6
-encode
-M
-1
-n
-n
-x
-6
-encode
-i
-15
-5
-48
-0
-15
-20
-0
-5
-49
-1
-1
-15
-5
-48
-2
-11
-I
-3
-I
-1
-I
-1
-I
-1
-n
-p
-3
-x
-5
-setup
-x
-8
-tokenize
-x
-6
-finish
-p
-9
-I
--1
-I
-4c
-I
-0
-I
-4d
-I
-4
-I
-4e
-I
-b
-I
-4f
-I
-f
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-7
-scanner
-x
-10
-text_token
-M
-1
-n
-n
-x
-10
-text_token
-i
-2
-1
-11
-I
-3
-I
-2
-I
-2
-I
-2
-n
-p
-0
-p
-5
-I
--1
-I
-5c
-I
-0
-I
-64
-I
-2
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-2
-x
-4
-text
-x
-4
-kind
-x
-11
-begin_group
-M
-1
-n
-n
-x
-11
-begin_group
-i
-2
-1
-11
-I
-2
-I
-1
-I
-1
-I
-1
-n
-p
-0
-p
-5
-I
--1
-I
-71
-I
-0
-I
-74
-I
-2
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-4
-kind
-x
-9
-end_group
-M
-1
-n
-n
-x
-9
-end_group
-i
-2
-1
-11
-I
-2
-I
-1
-I
-1
-I
-1
-n
-p
-0
-p
-5
-I
--1
-I
-76
-I
-0
-I
-79
-I
-2
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-4
-kind
-p
-13
-I
-2
-I
-43
-I
-10
-I
-48
-I
-1e
-I
-4c
-I
-2c
-I
-5c
-I
-3a
-I
-71
-I
-48
-I
-76
-I
-56
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-0
-x
-4
-ARGV
-n
-x
-5
-first
-x
-4
-to_i
-I
-f4240
-x
-1
-*
-s
-34
-2011 alpha, beta, (gamma), delta.
-
-x
-4
-to_f
-x
-4
-size
-x
-1
-/
-x
-4
-ceil
-x
-5
-Range
-x
-3
-new
-x
-6
-slice!
-n
-x
-2
-[]
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-45
-35
-0
-19
-0
-15
-45
-0
-1
-56
-2
-50
-3
-0
-19
-1
-15
-5
-7
-4
-64
-20
-1
-21
-1
-0
-7
-5
-49
-6
-1
-20
-0
-49
-7
-0
-35
-3
-49
-8
-1
-47
-49
-9
-1
-11
-I
-8
-I
-2
-I
-0
-I
-0
-I
--2
-p
-10
-x
-9
-Benchmark
-n
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-64
-78
-35
-1
-19
-0
-15
-21
-2
-1
-49
-0
-0
-21
-2
-2
-56
-1
-50
-2
-1
-15
-20
-0
-80
-56
-3
-50
-4
-1
-15
-21
-1
-0
-7
-5
-13
-70
-10
-46
-44
-43
-6
-12
-49
-7
-1
-50
-8
-0
-15
-21
-1
-0
-56
-9
-50
-10
-0
-49
-5
-0
-19
-1
-11
-I
-6
-I
-2
-I
-0
-I
-0
-I
--2
-p
-11
-x
-5
-lines
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-27
-57
-19
-0
-15
-21
-1
-0
-21
-1
-0
-49
-0
-0
-20
-0
-49
-1
-0
-49
-2
-0
-81
-3
-49
-4
-1
-11
-I
-5
-I
-1
-I
-1
-I
-1
-n
-p
-5
-x
-4
-last
-x
-4
-join
-x
-8
-bytesize
-x
-1
-+
-x
-2
-<<
-p
-5
-I
-0
-I
-89
-I
-4
-I
-8a
-I
-1b
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-5
-lines
-x
-10
-each_slice
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-25
-58
-37
-19
-0
-15
-37
-19
-1
-15
-15
-21
-2
-0
-45
-0
-1
-56
-2
-50
-3
-0
-49
-4
-1
-11
-I
-6
-I
-2
-I
-2
-I
-2
-n
-p
-5
-x
-6
-Thread
-n
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-99
-45
-0
-1
-49
-2
-0
-7
-3
-45
-4
-5
-13
-71
-6
-47
-9
-29
-47
-49
-7
-0
-13
-47
-49
-8
-0
-15
-8
-32
-49
-6
-0
-45
-9
-10
-13
-71
-6
-47
-9
-69
-47
-49
-7
-0
-13
-21
-4
-1
-44
-43
-11
-21
-1
-0
-21
-1
-1
-2
-49
-6
-3
-47
-49
-8
-2
-15
-8
-88
-21
-4
-1
-44
-43
-11
-21
-1
-0
-21
-1
-1
-2
-49
-6
-3
-49
-6
-2
-49
-12
-1
-13
-18
-3
-49
-13
-2
-15
-11
-I
-b
-I
-0
-I
-0
-I
-0
-I
--2
-p
-14
-x
-6
-Thread
-n
-x
-7
-current
-x
-3
-out
-x
-7
-Encoder
-n
-x
-3
-new
-x
-8
-allocate
-x
-10
-initialize
-x
-7
-Scanner
-n
-x
-5
-Range
-x
-6
-encode
-x
-3
-[]=
-p
-3
-I
-0
-I
-8e
-I
-63
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-0
-x
-3
-new
-x
-2
-<<
-p
-5
-I
-0
-I
-8c
-I
-a
-I
-8d
-I
-19
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-2
-x
-10
-this_chunk
-x
-10
-next_chunk
-x
-9
-each_cons
-x
-4
-join
-x
-4
-Proc
-x
-14
-__from_block__
-x
-4
-each
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-12
-57
-19
-0
-15
-20
-0
-7
-0
-49
-1
-1
-11
-I
-4
-I
-1
-I
-1
-I
-1
-n
-p
-2
-x
-3
-out
-x
-2
-[]
-p
-3
-I
-0
-I
-92
-I
-c
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-1
-t
-x
-3
-map
-p
-11
-I
-0
-I
-88
-I
-6
-I
-89
-I
-15
-I
-8c
-I
-1e
-I
-91
-I
-32
-I
-92
-I
-40
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-2
-x
-13
-chunk_offsets
-x
-3
-out
-x
-8
-realtime
-s
-46
-Multi-Threaded: %0.2fs -- %0.2f MB, %d threads
-d
- +0.953674316406250000000000000000000000000000000000000000    20
-x
-1
-/
-x
-4
-size
-x
-1
-%
-x
-4
-puts
-p
-7
-I
-0
-I
-86
-I
-5
-I
-87
-I
-10
-I
-94
-I
-2d
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-2
-x
-7
-threads
-x
-4
-time
-x
-5
-times
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-35
-45
-0
-1
-56
-2
-50
-3
-0
-19
-0
-15
-5
-7
-4
-64
-20
-0
-21
-1
-0
-7
-5
-49
-6
-1
-35
-2
-49
-7
-1
-47
-49
-8
-1
-11
-I
-7
-I
-1
-I
-0
-I
-0
-I
--2
-p
-9
-x
-9
-Benchmark
-n
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-60
-45
-0
-1
-13
-71
-2
-47
-9
-21
-47
-49
-3
-0
-13
-47
-49
-4
-0
-15
-8
-24
-49
-2
-0
-45
-5
-6
-13
-71
-2
-47
-9
-48
-47
-49
-3
-0
-13
-21
-2
-1
-47
-49
-4
-1
-15
-8
-54
-21
-2
-1
-49
-2
-1
-49
-7
-1
-19
-0
-11
-I
-6
-I
-1
-I
-0
-I
-0
-I
--2
-p
-8
-x
-7
-Encoder
-n
-x
-3
-new
-x
-8
-allocate
-x
-10
-initialize
-x
-7
-Scanner
-n
-x
-6
-encode
-p
-3
-I
-0
-I
-99
-I
-3c
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-3
-out
-x
-8
-realtime
-s
-27
-Current: %0.2fs -- %0.2f MB
-d
- +0.953674316406250000000000000000000000000000000000000000    20
-x
-1
-/
-x
-1
-%
-x
-4
-puts
-p
-5
-I
-0
-I
-98
-I
-b
-I
-9b
-I
-23
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-1
-x
-4
-time
-p
-27
-I
-0
-I
-1
-I
-9
-I
-2
-I
-12
-I
-3
-I
-1b
-I
-5
-I
-38
-I
-38
-I
-55
-I
-41
-I
-70
-I
-7d
-I
-86
-I
-80
-I
-8c
-I
-81
-I
-a4
-I
-82
-I
-b3
-I
-84
-I
-c6
-I
-85
-I
-cd
-I
-97
-I
-d6
-x
-52
-/Users/murphy/ruby/coderay/etc/speedup2/multi-pos.rb
-p
-3
-x
-4
-size
-x
-4
-code
-x
-10
-slice_size
diff --git a/etc/multicore/multi-slice-encoding-improved.rb b/etc/multicore/multi-slice-encoding-improved.rb
deleted file mode 100644
index 301589d..0000000
--- a/etc/multicore/multi-slice-encoding-improved.rb
+++ /dev/null
@@ -1,122 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code
-    super code
-  end
-  
-  def tokenize encoder = Tokens.new
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if match = scan(/\s+/)
-        encoder.text_token match, :space
-      elsif match = scan(/\d+/)
-        encoder.text_token match, :number
-      elsif match = scan(/\w+/)
-        encoder.text_token match, :word
-      elsif match = scan(/[,.]/)
-        encoder.text_token match, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-    @opened = []
-  end
-  
-  def finish
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  def text_token text, kind
-    if kind == :space
-      @out << text
-    else
-      text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-      @out << kind.to_s << '(' << text << ')'
-    end
-  end
-  
-  def begin_group kind
-    # @opened << kind
-    @out << kind.to_s << '<'
-  end
-  
-  def end_group kind
-    # @opened.pop
-    @out << '>'
-  end
-  
-end
-
-size = ((ARGV.first || 1).to_f * 1_000_000).to_i  # size
-
-# generate string
-code = "2011 alpha, beta, (gamma), delta.\n"
-code *= (size.to_f / code.size).ceil
-code.slice! size..-1
-
-slice_size = (ARGV[1] || 100).to_i
-N = 1
-
-1.times do
-  out = Encoder.new.encode(Scanner.new(code))
-end
-
-1.times do
-2.times do
-  threads = []
-  seconds = Benchmark.realtime do N.times do
-    chunk_offsets = [0]
-    code.lines.each_slice slice_size do |lines|
-      chunk_offsets << chunk_offsets.last + lines.join.bytesize
-    end
-    threads.clear
-    chunk_offsets.each_cons(2) do |this_chunk, next_chunk|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode Scanner.new(code[this_chunk...next_chunk])
-      end
-    end
-    threads.each(&:join)
-    # out = threads.map { |t| t[:out] }.join
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads' % [mb, seconds, mb / seconds, threads.size]
-end
-2.times do
-  seconds = Benchmark.realtime do N.times do
-    out = Encoder.new.encode(Scanner.new(code))
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s' % [mb, seconds, mb / seconds]
-end
-end
\ No newline at end of file
diff --git a/etc/multicore/multi-slice-encoding.rb b/etc/multicore/multi-slice-encoding.rb
deleted file mode 100644
index 7616bd7..0000000
--- a/etc/multicore/multi-slice-encoding.rb
+++ /dev/null
@@ -1,161 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code
-    super code
-  end
-  
-  def tokenize encoder = Tokens.new
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if matched = scan(/ +/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/\n/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/!/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/=/)  #/
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/%/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/\d+/)
-        encoder.text_token matched, :number
-      elsif matched = scan(/\w+/)
-        encoder.text_token matched, :word
-      elsif matched = scan(/[,.]/)
-        encoder.text_token matched, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Tokens < Array
-  alias token push
-  alias text_token push
-  alias block_token push
-  def begin_group kind; push :begin_group, kind end
-  def end_group kind; push :end_group, kind end
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-    @opened = []
-  end
-  
-  def finish
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  # def token content, kind
-  #   if content.is_a? ::String
-  #     text_token content, kind
-  #   elsif content.is_a? ::Symbol
-  #     block_token content, kind
-  #   else
-  #     raise 'Unknown token content type: %p' % [content]
-  #   end
-  # end
-  
-  def text_token text, kind
-    @out <<
-      if kind == :space
-        text
-      else
-        text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-        "#{kind}(#{text})"
-      end
-  end
-  
-  # def block_token action, kind
-  #   case action
-  #   when :begin_group
-  #     begin_group kind
-  #   when :end_group
-  #     end_group kind
-  #   else
-  #     raise
-  #   end
-  # end
-  
-  def begin_group kind
-    @opened << kind
-    @out << "#{kind}<"
-  end
-  
-  def end_group kind
-    @opened.pop
-    @out << '>'
-  end
-  
-end
-
-size = ((ARGV.first || 1).to_f * 1_000_000).to_i  # size
-
-# generate string
-code = "2011 alpha, beta, (gamma), delta.\n"
-code *= (size.to_f / code.size).ceil
-code.slice! size..-1
-
-slice_size = (ARGV[1] || 100).to_i
-N = 1
-
-1.times do
-  out = Encoder.new.encode(Scanner.new(code))
-end
-
-1.times do
-2.times do
-  threads = []
-  seconds = Benchmark.realtime do N.times do
-    chunk_offsets = [0]
-    code.lines.each_slice slice_size do |lines|
-      chunk_offsets << chunk_offsets.last + lines.join.bytesize
-    end
-    threads.clear
-    chunk_offsets.each_cons(2) do |this_chunk, next_chunk|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode Scanner.new(code[this_chunk...next_chunk])
-      end
-    end
-    threads.each(&:join)
-    # out = threads.map { |t| t[:out] }.join
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads' % [mb, seconds, mb / seconds, threads.size]
-end
-2.times do
-  seconds = Benchmark.realtime do N.times do
-    out = Encoder.new.encode(Scanner.new(code))
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s' % [mb, seconds, mb / seconds]
-end
-end
\ No newline at end of file
diff --git a/etc/multicore/multi-slice-improved.rb b/etc/multicore/multi-slice-improved.rb
deleted file mode 100644
index b5997ca..0000000
--- a/etc/multicore/multi-slice-improved.rb
+++ /dev/null
@@ -1,119 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code
-    super code
-  end
-  
-  def tokenize encoder = Tokens.new
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if match = scan(/\s+/)
-        encoder.text_token match, :space
-      elsif match = scan(/\d+/)
-        encoder.text_token match, :number
-      elsif match = scan(/\w+/)
-        encoder.text_token match, :word
-      elsif match = scan(/[,.]/)
-        encoder.text_token match, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-  end
-  
-  def finish
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  def text_token text, kind
-    # if kind == :space
-    #   @out << text
-    # else
-    #   text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-    #   @out << kind.to_s << '(' << text << ')'
-    # end
-  end
-  
-  def begin_group kind
-    # @out << kind.to_s << '<'
-  end
-  
-  def end_group kind
-    # @out << '>'
-  end
-  
-end
-
-size = ((ARGV.first || 1).to_f * 1_000_000).to_i  # size
-
-# generate string
-code = "2011 alpha, beta, (gamma), delta.\n"
-code *= (size.to_f / code.size).ceil
-code.slice! size..-1
-
-slice_size = (ARGV[1] || 100).to_i
-N = 1
-
-1.times do
-  out = Encoder.new.encode(Scanner.new(code))
-end
-
-1.times do
-2.times do
-  threads = []
-  seconds = Benchmark.realtime do N.times do
-    chunk_offsets = [0]
-    code.lines.each_slice slice_size do |lines|
-      chunk_offsets << chunk_offsets.last + lines.join.bytesize
-    end
-    threads.clear
-    chunk_offsets.each_cons(2) do |this_chunk, next_chunk|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode Scanner.new(code[this_chunk...next_chunk])
-      end
-    end
-    threads.each(&:join)
-    # out = threads.map { |t| t[:out] }.join
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads' % [mb, seconds, mb / seconds, threads.size]
-end
-2.times do
-  seconds = Benchmark.realtime do N.times do
-    out = Encoder.new.encode(Scanner.new(code))
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s' % [mb, seconds, mb / seconds]
-end
-end
\ No newline at end of file
diff --git a/etc/multicore/multi-slice.rb b/etc/multicore/multi-slice.rb
deleted file mode 100644
index 33b8cd6..0000000
--- a/etc/multicore/multi-slice.rb
+++ /dev/null
@@ -1,161 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code
-    super code
-  end
-  
-  def tokenize encoder = Tokens.new
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if matched = scan(/ +/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/\n/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/!/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/=/)  #/
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/%/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/\d+/)
-        encoder.text_token matched, :number
-      elsif matched = scan(/\w+/)
-        encoder.text_token matched, :word
-      elsif matched = scan(/[,.]/)
-        encoder.text_token matched, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Tokens < Array
-  alias token push
-  alias text_token push
-  alias block_token push
-  def begin_group kind; push :begin_group, kind end
-  def end_group kind; push :end_group, kind end
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-    @opened = []
-  end
-  
-  def finish
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  # def token content, kind
-  #   if content.is_a? ::String
-  #     text_token content, kind
-  #   elsif content.is_a? ::Symbol
-  #     block_token content, kind
-  #   else
-  #     raise 'Unknown token content type: %p' % [content]
-  #   end
-  # end
-  
-  def text_token text, kind
-    # @out <<
-    #   if kind == :space
-    #     text
-    #   else
-    #     text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-    #     "#{kind}(#{text})"
-    #   end
-  end
-  
-  # def block_token action, kind
-  #   case action
-  #   when :begin_group
-  #     begin_group kind
-  #   when :end_group
-  #     end_group kind
-  #   else
-  #     raise
-  #   end
-  # end
-  
-  def begin_group kind
-    # @opened << kind
-    # @out << "#{kind}<"
-  end
-  
-  def end_group kind
-    # @opened.pop
-    # @out << '>'
-  end
-  
-end
-
-size = ((ARGV.first || 1).to_f * 1_000_000).to_i  # size
-
-# generate string
-code = "2011 alpha, beta, (gamma), delta.\n"
-code *= (size.to_f / code.size).ceil
-code.slice! size..-1
-
-slice_size = (ARGV[1] || 100).to_i
-N = 1
-
-100.times do
-  out = Encoder.new.encode(Scanner.new(code))
-end
-
-1000.times do
-2.times do
-  threads = []
-  seconds = Benchmark.realtime do N.times do
-    chunk_offsets = [0]
-    code.lines.each_slice slice_size do |lines|
-      chunk_offsets << chunk_offsets.last + lines.join.bytesize
-    end
-    threads.clear
-    chunk_offsets.each_cons(2) do |this_chunk, next_chunk|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode Scanner.new(code[this_chunk...next_chunk])
-      end
-    end
-    threads.each(&:join)
-    out = threads.map { |t| t[:out] }.join
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads' % [mb, seconds, mb / seconds, threads.size]
-end
-2.times do
-  seconds = Benchmark.realtime do N.times do
-    out = Encoder.new.encode(Scanner.new(code))
-  end end
-  
-  mb = N * size / 1_000_000.0
-  puts 'Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s' % [mb, seconds, mb / seconds]
-end
-end
\ No newline at end of file
diff --git a/etc/multicore/multi.rb b/etc/multicore/multi.rb
deleted file mode 100644
index 789228e..0000000
--- a/etc/multicore/multi.rb
+++ /dev/null
@@ -1,150 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code
-    super code
-  end
-  
-  def tokenize encoder = Tokens.new
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if matched = scan(/\s+/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/!/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/=/)  #/
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/%/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/\d+/)
-        encoder.text_token matched, :number
-      elsif matched = scan(/\w+/)
-        encoder.text_token matched, :word
-      elsif matched = scan(/[,.]/)
-        encoder.text_token matched, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Tokens < Array
-  alias token push
-  alias text_token push
-  alias block_token push
-  def begin_group kind; push :begin_group, kind end
-  def end_group kind; push :end_group, kind end
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-    @opened = []
-  end
-  
-  def finish
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  # def token content, kind
-  #   if content.is_a? ::String
-  #     text_token content, kind
-  #   elsif content.is_a? ::Symbol
-  #     block_token content, kind
-  #   else
-  #     raise 'Unknown token content type: %p' % [content]
-  #   end
-  # end
-  
-  def text_token text, kind
-    # @out <<
-    #   if kind == :space
-    #     text
-    #   else
-    #     text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-    #     "#{kind}(#{text})"
-    #   end
-  end
-  
-  # def block_token action, kind
-  #   case action
-  #   when :begin_group
-  #     begin_group kind
-  #   when :end_group
-  #     end_group kind
-  #   else
-  #     raise
-  #   end
-  # end
-  
-  def begin_group kind
-    # @opened << kind
-    # @out << "#{kind}<"
-  end
-  
-  def end_group kind
-    # @opened.pop
-    # @out << '>'
-  end
-  
-end
-
-size = (ARGV.first || 1).to_i * 1_000_000  # size in MB
-
-# generate string
-code = "2011 alpha, beta, (gamma), delta.\n"
-code *= (size.to_f / code.size).ceil
-code.slice! size..-1
-
-slice_size = (ARGV[1] || 100).to_i
-
-3.times do
-2.times do
-  threads = []
-  seconds = Benchmark.realtime do
-    code.lines.each_slice slice_size do |lines|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode Scanner.new(lines.join)
-      end
-    end
-    threads.each(&:join)
-    out = threads.map { |t| t[:out] }.join
-  end
-  
-  mb = size / 1_000_000.0
-  puts 'Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads' % [mb, seconds, mb / seconds, threads.size]
-end
-
-2.times do
-  seconds = Benchmark.realtime do
-    out = Encoder.new.encode Scanner.new(code)
-  end
-  
-  mb = size / 1_000_000.0
-  puts 'Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s' % [mb, seconds, mb / seconds]
-end
-end
\ No newline at end of file
diff --git a/etc/multicore/multi.rbc b/etc/multicore/multi.rbc
deleted file mode 100644
index b23ec04..0000000
--- a/etc/multicore/multi.rbc
+++ /dev/null
@@ -1,2751 +0,0 @@
-!RBIX
-2959558172506603432
-x
-M
-1
-n
-n
-x
-10
-__script__
-i
-208
-5
-7
-0
-64
-47
-49
-1
-1
-15
-5
-7
-2
-64
-47
-49
-1
-1
-15
-5
-7
-3
-64
-47
-49
-1
-1
-15
-99
-7
-4
-45
-5
-6
-65
-49
-7
-3
-13
-99
-12
-7
-8
-12
-7
-9
-12
-65
-12
-49
-10
-4
-15
-49
-8
-0
-15
-99
-7
-11
-45
-12
-13
-65
-49
-7
-3
-13
-99
-12
-7
-8
-12
-7
-14
-12
-65
-12
-49
-10
-4
-15
-49
-8
-0
-15
-99
-7
-15
-1
-65
-49
-7
-3
-13
-99
-12
-7
-8
-12
-7
-16
-12
-65
-12
-49
-10
-4
-15
-49
-8
-0
-15
-45
-17
-18
-49
-19
-0
-13
-10
-123
-15
-79
-49
-20
-0
-7
-21
-49
-22
-1
-19
-0
-15
-7
-23
-64
-19
-1
-15
-20
-1
-20
-0
-49
-24
-0
-20
-1
-49
-25
-0
-49
-26
-1
-49
-27
-0
-49
-22
-1
-19
-1
-15
-20
-1
-44
-43
-28
-20
-0
-77
-49
-29
-2
-49
-30
-1
-15
-45
-17
-31
-79
-49
-32
-1
-13
-10
-192
-15
-4
-100
-49
-20
-0
-19
-2
-15
-4
-3
-56
-33
-50
-34
-0
-15
-2
-11
-I
-9
-I
-3
-I
-0
-I
-0
-n
-p
-35
-s
-7
-strscan
-x
-7
-require
-s
-9
-benchmark
-s
-6
-thread
-x
-7
-Scanner
-x
-13
-StringScanner
-n
-x
-10
-open_class
-x
-14
-__class_init__
-M
-1
-n
-n
-x
-7
-Scanner
-i
-48
-5
-66
-99
-7
-0
-7
-1
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-4
-7
-5
-65
-67
-49
-2
-0
-49
-3
-4
-15
-5
-48
-6
-15
-99
-7
-7
-7
-8
-65
-67
-49
-2
-0
-49
-3
-4
-11
-I
-5
-I
-0
-I
-0
-I
-0
-n
-p
-9
-x
-10
-initialize
-M
-1
-n
-n
-x
-10
-initialize
-i
-7
-20
-0
-54
-52
-0
-1
-11
-I
-3
-I
-1
-I
-1
-I
-1
-n
-p
-1
-x
-10
-initialize
-p
-5
-I
--1
-I
-7
-I
-0
-I
-8
-I
-7
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-4
-code
-x
-17
-method_visibility
-x
-15
-add_defn_method
-x
-8
-tokenize
-M
-1
-n
-n
-x
-8
-tokenize
-i
-42
-23
-0
-10
-31
-45
-0
-1
-13
-71
-2
-47
-9
-25
-47
-49
-3
-0
-13
-47
-49
-4
-0
-15
-8
-28
-49
-2
-0
-19
-0
-15
-5
-20
-0
-47
-49
-5
-1
-15
-20
-0
-11
-I
-3
-I
-1
-I
-0
-I
-1
-n
-p
-6
-x
-6
-Tokens
-n
-x
-3
-new
-x
-8
-allocate
-x
-10
-initialize
-x
-11
-scan_tokens
-p
-7
-I
--1
-I
-b
-I
-1f
-I
-c
-I
-27
-I
-d
-I
-2a
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-7
-encoder
-x
-9
-protected
-x
-11
-scan_tokens
-M
-1
-n
-n
-x
-11
-scan_tokens
-i
-350
-5
-47
-49
-0
-0
-10
-348
-5
-7
-1
-13
-70
-9
-26
-15
-44
-43
-2
-7
-3
-78
-49
-4
-2
-6
-1
-47
-49
-5
-1
-19
-1
-9
-45
-20
-0
-20
-1
-7
-6
-49
-7
-2
-8
-344
-5
-7
-8
-13
-70
-9
-64
-15
-44
-43
-2
-7
-9
-78
-49
-4
-2
-6
-8
-47
-49
-5
-1
-19
-1
-9
-83
-20
-0
-20
-1
-7
-10
-49
-7
-2
-8
-344
-5
-7
-11
-13
-70
-9
-102
-15
-44
-43
-2
-7
-12
-78
-49
-4
-2
-6
-11
-47
-49
-5
-1
-19
-1
-9
-121
-20
-0
-20
-1
-7
-10
-49
-7
-2
-8
-344
-5
-7
-13
-13
-70
-9
-140
-15
-44
-43
-2
-7
-14
-78
-49
-4
-2
-6
-13
-47
-49
-5
-1
-19
-1
-9
-159
-20
-0
-20
-1
-7
-10
-49
-7
-2
-8
-344
-5
-7
-15
-13
-70
-9
-178
-15
-44
-43
-2
-7
-16
-78
-49
-4
-2
-6
-15
-47
-49
-5
-1
-19
-1
-9
-197
-20
-0
-20
-1
-7
-17
-49
-7
-2
-8
-344
-5
-7
-18
-13
-70
-9
-216
-15
-44
-43
-2
-7
-19
-78
-49
-4
-2
-6
-18
-47
-49
-5
-1
-19
-1
-9
-235
-20
-0
-20
-1
-7
-20
-49
-7
-2
-8
-344
-5
-7
-21
-13
-70
-9
-254
-15
-44
-43
-2
-7
-22
-78
-49
-4
-2
-6
-21
-47
-49
-5
-1
-19
-1
-9
-273
-20
-0
-20
-1
-7
-23
-49
-7
-2
-8
-344
-5
-7
-24
-13
-70
-9
-292
-15
-44
-43
-2
-7
-25
-78
-49
-4
-2
-6
-24
-47
-49
-5
-1
-9
-307
-20
-0
-7
-26
-49
-27
-1
-8
-344
-5
-7
-28
-13
-70
-9
-326
-15
-44
-43
-2
-7
-29
-78
-49
-4
-2
-6
-28
-47
-49
-5
-1
-9
-341
-20
-0
-7
-26
-49
-30
-1
-8
-344
-5
-48
-31
-15
-68
-8
-0
-1
-11
-I
-6
-I
-2
-I
-1
-I
-1
-n
-p
-32
-x
-4
-eos?
-n
-x
-6
-Regexp
-s
-3
-\s+
-x
-3
-new
-x
-4
-scan
-x
-5
-space
-x
-10
-text_token
-n
-s
-1
-!
-x
-19
-not_going_to_happen
-n
-s
-1
-=
-n
-s
-1
-%
-n
-s
-3
-\d+
-x
-6
-number
-n
-s
-3
-\w+
-x
-4
-word
-n
-s
-4
-[,.]
-x
-2
-op
-n
-s
-2
-\(
-x
-3
-par
-x
-11
-begin_group
-n
-s
-2
-\)
-x
-9
-end_group
-x
-5
-raise
-p
-43
-I
--1
-I
-12
-I
-0
-I
-13
-I
-7
-I
-14
-I
-22
-I
-15
-I
-2d
-I
-16
-I
-48
-I
-17
-I
-53
-I
-18
-I
-6e
-I
-19
-I
-79
-I
-1a
-I
-94
-I
-1b
-I
-9f
-I
-1c
-I
-ba
-I
-1d
-I
-c5
-I
-1e
-I
-e0
-I
-1f
-I
-eb
-I
-20
-I
-106
-I
-21
-I
-111
-I
-22
-I
-12a
-I
-23
-I
-133
-I
-24
-I
-14c
-I
-25
-I
-155
-I
-27
-I
-15e
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-2
-x
-7
-encoder
-x
-7
-matched
-p
-9
-I
-2
-I
-7
-I
-10
-I
-b
-I
-1e
-I
-10
-I
-22
-I
-12
-I
-30
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-13
-attach_method
-x
-6
-Tokens
-x
-5
-Array
-n
-M
-1
-n
-n
-x
-6
-Tokens
-i
-60
-5
-66
-65
-7
-0
-7
-1
-47
-49
-2
-2
-15
-65
-7
-3
-7
-1
-47
-49
-2
-2
-15
-65
-7
-4
-7
-1
-47
-49
-2
-2
-15
-99
-7
-5
-7
-6
-65
-67
-49
-7
-0
-49
-8
-4
-15
-99
-7
-9
-7
-10
-65
-67
-49
-7
-0
-49
-8
-4
-11
-I
-5
-I
-0
-I
-0
-I
-0
-n
-p
-11
-x
-5
-token
-x
-4
-push
-x
-12
-alias_method
-x
-10
-text_token
-x
-11
-block_token
-x
-11
-begin_group
-M
-1
-n
-n
-x
-11
-begin_group
-i
-10
-5
-7
-0
-20
-0
-47
-49
-1
-2
-11
-I
-4
-I
-1
-I
-1
-I
-1
-n
-p
-2
-x
-11
-begin_group
-x
-4
-push
-p
-3
-I
--1
-I
-33
-I
-a
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-4
-kind
-x
-17
-method_visibility
-x
-15
-add_defn_method
-x
-9
-end_group
-M
-1
-n
-n
-x
-9
-end_group
-i
-10
-5
-7
-0
-20
-0
-47
-49
-1
-2
-11
-I
-4
-I
-1
-I
-1
-I
-1
-n
-p
-2
-x
-9
-end_group
-x
-4
-push
-p
-3
-I
--1
-I
-34
-I
-a
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-4
-kind
-p
-11
-I
-2
-I
-30
-I
-c
-I
-31
-I
-16
-I
-32
-I
-20
-I
-33
-I
-2e
-I
-34
-I
-3c
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-7
-Encoder
-M
-1
-n
-n
-x
-7
-Encoder
-i
-86
-5
-66
-99
-7
-0
-7
-1
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-4
-7
-5
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-6
-7
-7
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-8
-7
-9
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-10
-7
-11
-65
-67
-49
-2
-0
-49
-3
-4
-15
-99
-7
-12
-7
-13
-65
-67
-49
-2
-0
-49
-3
-4
-11
-I
-5
-I
-0
-I
-0
-I
-0
-n
-p
-14
-x
-5
-setup
-M
-1
-n
-n
-x
-5
-setup
-i
-11
-7
-0
-64
-38
-1
-15
-35
-0
-38
-2
-11
-I
-1
-I
-0
-I
-0
-I
-0
-n
-p
-3
-s
-0
-
-x
-4
-@out
-x
-7
-@opened
-p
-7
-I
--1
-I
-3a
-I
-0
-I
-3b
-I
-6
-I
-3c
-I
-b
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-17
-method_visibility
-x
-15
-add_defn_method
-x
-6
-finish
-M
-1
-n
-n
-x
-6
-finish
-i
-3
-39
-0
-11
-I
-1
-I
-0
-I
-0
-I
-0
-n
-p
-1
-x
-4
-@out
-p
-5
-I
--1
-I
-3f
-I
-0
-I
-40
-I
-3
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-6
-encode
-M
-1
-n
-n
-x
-6
-encode
-i
-15
-5
-48
-0
-15
-20
-0
-5
-49
-1
-1
-15
-5
-48
-2
-11
-I
-3
-I
-1
-I
-1
-I
-1
-n
-p
-3
-x
-5
-setup
-x
-8
-tokenize
-x
-6
-finish
-p
-9
-I
--1
-I
-43
-I
-0
-I
-44
-I
-4
-I
-45
-I
-b
-I
-46
-I
-f
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-7
-scanner
-x
-10
-text_token
-M
-1
-n
-n
-x
-10
-text_token
-i
-2
-1
-11
-I
-3
-I
-2
-I
-2
-I
-2
-n
-p
-0
-p
-5
-I
--1
-I
-53
-I
-0
-I
-5b
-I
-2
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-2
-x
-4
-text
-x
-4
-kind
-x
-11
-begin_group
-M
-1
-n
-n
-x
-11
-begin_group
-i
-2
-1
-11
-I
-2
-I
-1
-I
-1
-I
-1
-n
-p
-0
-p
-5
-I
--1
-I
-68
-I
-0
-I
-6b
-I
-2
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-4
-kind
-x
-9
-end_group
-M
-1
-n
-n
-x
-9
-end_group
-i
-2
-1
-11
-I
-2
-I
-1
-I
-1
-I
-1
-n
-p
-0
-p
-5
-I
--1
-I
-6d
-I
-0
-I
-70
-I
-2
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-4
-kind
-p
-13
-I
-2
-I
-3a
-I
-10
-I
-3f
-I
-1e
-I
-43
-I
-2c
-I
-53
-I
-3a
-I
-68
-I
-48
-I
-6d
-I
-56
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-4
-ARGV
-n
-x
-5
-first
-x
-4
-to_i
-I
-f4240
-x
-1
-*
-s
-34
-2011 alpha, beta, (gamma), delta.
-
-x
-4
-to_f
-x
-4
-size
-x
-1
-/
-x
-4
-ceil
-x
-5
-Range
-x
-3
-new
-x
-6
-slice!
-n
-x
-2
-[]
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-14
-80
-56
-0
-50
-1
-0
-15
-80
-56
-2
-50
-1
-0
-11
-I
-3
-I
-0
-I
-0
-I
-0
-I
--2
-p
-3
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-57
-35
-0
-19
-0
-15
-45
-0
-1
-56
-2
-50
-3
-0
-19
-1
-15
-21
-2
-0
-7
-4
-49
-5
-1
-19
-2
-15
-5
-7
-6
-64
-20
-2
-20
-1
-20
-2
-20
-1
-49
-5
-1
-20
-0
-49
-7
-0
-35
-4
-49
-8
-1
-47
-49
-9
-1
-11
-I
-a
-I
-3
-I
-0
-I
-0
-I
--2
-p
-10
-x
-9
-Benchmark
-n
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-49
-21
-3
-1
-49
-0
-0
-21
-3
-2
-56
-1
-50
-2
-1
-15
-21
-1
-0
-7
-3
-13
-70
-10
-31
-44
-43
-4
-12
-49
-5
-1
-50
-6
-0
-15
-21
-1
-0
-56
-7
-50
-8
-0
-49
-3
-0
-19
-0
-11
-I
-5
-I
-1
-I
-0
-I
-0
-I
--2
-p
-9
-x
-5
-lines
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-19
-57
-19
-0
-15
-21
-2
-0
-45
-0
-1
-56
-2
-50
-3
-0
-49
-4
-1
-11
-I
-5
-I
-1
-I
-1
-I
-1
-n
-p
-5
-x
-6
-Thread
-n
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-79
-45
-0
-1
-49
-2
-0
-7
-3
-45
-4
-5
-13
-71
-6
-47
-9
-29
-47
-49
-7
-0
-13
-47
-49
-8
-0
-15
-8
-32
-49
-6
-0
-45
-9
-10
-13
-71
-6
-47
-9
-59
-47
-49
-7
-0
-13
-21
-1
-0
-49
-11
-0
-47
-49
-8
-1
-15
-8
-68
-21
-1
-0
-49
-11
-0
-49
-6
-1
-49
-12
-1
-13
-18
-3
-49
-13
-2
-15
-11
-I
-7
-I
-0
-I
-0
-I
-0
-I
--2
-p
-14
-x
-6
-Thread
-n
-x
-7
-current
-x
-3
-out
-x
-7
-Encoder
-n
-x
-3
-new
-x
-8
-allocate
-x
-10
-initialize
-x
-7
-Scanner
-n
-x
-4
-join
-x
-6
-encode
-x
-3
-[]=
-p
-3
-I
-0
-I
-83
-I
-4f
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-3
-new
-x
-2
-<<
-p
-5
-I
-0
-I
-81
-I
-4
-I
-82
-I
-13
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-5
-lines
-x
-10
-each_slice
-x
-4
-join
-x
-4
-Proc
-x
-14
-__from_block__
-x
-4
-each
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-12
-57
-19
-0
-15
-20
-0
-7
-0
-49
-1
-1
-11
-I
-4
-I
-1
-I
-1
-I
-1
-n
-p
-2
-x
-3
-out
-x
-2
-[]
-p
-3
-I
-0
-I
-87
-I
-c
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-1
-t
-x
-3
-map
-p
-7
-I
-0
-I
-81
-I
-f
-I
-86
-I
-23
-I
-87
-I
-31
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-3
-out
-x
-8
-realtime
-d
- +0.953674316406250000000000000000000000000000000000000000    20
-x
-1
-/
-s
-60
-Multi-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s @ %d threads
-x
-4
-size
-x
-1
-%
-x
-4
-puts
-p
-9
-I
-0
-I
-7f
-I
-5
-I
-80
-I
-10
-I
-8a
-I
-1b
-I
-8b
-I
-39
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-3
-x
-7
-threads
-x
-7
-seconds
-x
-2
-mb
-x
-5
-times
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-47
-45
-0
-1
-56
-2
-50
-3
-0
-19
-0
-15
-21
-2
-0
-7
-4
-49
-5
-1
-19
-1
-15
-5
-7
-6
-64
-20
-1
-20
-0
-20
-1
-20
-0
-49
-5
-1
-35
-3
-49
-7
-1
-47
-49
-8
-1
-11
-I
-9
-I
-2
-I
-0
-I
-0
-I
--2
-p
-9
-x
-9
-Benchmark
-n
-M
-1
-p
-2
-x
-9
-for_block
-t
-n
-x
-9
-__block__
-i
-60
-45
-0
-1
-13
-71
-2
-47
-9
-21
-47
-49
-3
-0
-13
-47
-49
-4
-0
-15
-8
-24
-49
-2
-0
-45
-5
-6
-13
-71
-2
-47
-9
-48
-47
-49
-3
-0
-13
-21
-3
-1
-47
-49
-4
-1
-15
-8
-54
-21
-3
-1
-49
-2
-1
-49
-7
-1
-19
-0
-11
-I
-6
-I
-1
-I
-0
-I
-0
-I
--2
-p
-8
-x
-7
-Encoder
-n
-x
-3
-new
-x
-8
-allocate
-x
-10
-initialize
-x
-7
-Scanner
-n
-x
-6
-encode
-p
-3
-I
-0
-I
-90
-I
-3c
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-1
-x
-3
-out
-x
-8
-realtime
-d
- +0.953674316406250000000000000000000000000000000000000000    20
-x
-1
-/
-s
-48
-Single-Threaded: %0.1f MB in %0.2fs = %0.1f MB/s
-x
-1
-%
-x
-4
-puts
-p
-7
-I
-0
-I
-8f
-I
-b
-I
-93
-I
-16
-I
-94
-I
-2f
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-2
-x
-7
-seconds
-x
-2
-mb
-p
-5
-I
-0
-I
-7e
-I
-7
-I
-8e
-I
-e
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-0
-x
-5
-times
-p
-25
-I
-0
-I
-1
-I
-9
-I
-2
-I
-12
-I
-3
-I
-1b
-I
-5
-I
-38
-I
-2f
-I
-55
-I
-38
-I
-70
-I
-74
-I
-86
-I
-77
-I
-8c
-I
-78
-I
-a4
-I
-79
-I
-b3
-I
-7b
-I
-c6
-I
-7d
-I
-d0
-x
-48
-/Users/murphy/ruby/coderay/etc/speedup2/multi.rb
-p
-3
-x
-4
-size
-x
-4
-code
-x
-10
-slice_size
diff --git a/etc/speedup2/multi.rb b/etc/speedup2/multi.rb
deleted file mode 100644
index bd9551c..0000000
--- a/etc/speedup2/multi.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-require 'strscan'
-require 'benchmark'
-require 'thread'
-
-class Scanner < StringScanner
-  
-  def initialize code
-    super code
-  end
-  
-  def tokenize encoder = Tokens.new
-    scan_tokens encoder
-    encoder
-  end
-  
-protected
-  
-  def scan_tokens encoder
-    until eos?
-      if matched = scan(/\s+/)
-        encoder.text_token matched, :space
-      elsif matched = scan(/!/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/=/)  #/
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/%/)
-        encoder.text_token matched, :not_going_to_happen
-      elsif matched = scan(/\d+/)
-        encoder.text_token matched, :number
-      elsif matched = scan(/\w+/)
-        encoder.text_token matched, :word
-      elsif matched = scan(/[,.]/)
-        encoder.text_token matched, :op
-      elsif scan(/\(/)
-        encoder.begin_group :par
-      elsif scan(/\)/)
-        encoder.end_group :par
-      else
-        raise
-      end
-    end
-  end
-  
-end
-
-
-class Tokens < Array
-  alias token push
-  alias text_token push
-  alias block_token push
-  def begin_group kind; push :begin_group, kind end
-  def end_group kind; push :end_group, kind end
-end
-
-
-class Encoder
-  
-  def setup
-    @out = ''
-    @opened = []
-  end
-  
-  def finish
-    while kind = @opened.pop
-      close kind
-    end
-    @out
-  end
-  
-  def encode scanner
-    setup
-    scanner.tokenize self
-    finish
-  end
-  
-  def token content, kind
-    if content.is_a? ::String
-      text_token content, kind
-    elsif content.is_a? ::Symbol
-      block_token content, kind
-    else
-      raise 'Unknown token content type: %p' % [content]
-    end
-  end
-  
-  def text_token text, kind
-    @out <<
-      if kind == :space
-        text
-      else
-        text.gsub!(/[)\\]/, '\\\\\0')  # escape ) and \
-        "#{kind}(#{text})"
-      end
-  end
-  
-  def block_token action, kind
-    case action
-    when :begin_group
-      begin_group kind
-    when :end_group
-      end_group kind
-    else
-      raise
-    end
-  end
-  
-  def begin_group kind
-    @opened << kind
-    @out << "#{kind}<"
-  end
-  
-  def end_group kind
-    @opened.pop
-    @out << '>'
-  end
-  
-end
-
-N = (5 ** (ARGV.first || 8).to_i)
-code = (1..N).map { |n| "#{n} alpha, beta, (gamma).\n" }.join
-
-slice_size = (ARGV[1] || 100).to_i
-3.times do
-  time = Benchmark.realtime do
-    threads = []
-    code.lines.each_slice slice_size do |lines|
-      threads << Thread.new do
-        Thread.current[:out] = Encoder.new.encode(Scanner.new(lines.inject(&:+)))
-      end
-    end
-    threads.each(&:join)
-    out = threads.map { |t| t[:out] }.join
-  end
-  puts 'Multi-Threaded: %0.2fs -- %0.0f kTok/s' % [time, (N * 11 + 1) / time / 1000]
-end
-
-3.times do
-  time = Benchmark.realtime do
-    out = Encoder.new.encode(Scanner.new(code))
-  end
-  puts 'Current: %0.2fs -- %0.0f kTok/s' % [time, (N * 11 + 1) / time / 1000]
-end