1 files changed, 16 insertions, 3 deletions
diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 7d154da..64b3c83 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -147,14 +147,27 @@ module CodeRay
       end
 
       # Scans the code and returns all tokens in a Tokens object.
-      def tokenize new_string=nil, options = {}
+      def tokenize source = nil, options = {}
         options = @options.merge(options)
         @tokens = options[:tokens] || @tokens || Tokens.new
         @tokens.scanner = self if @tokens.respond_to? :scanner=
-        self.string = new_string if new_string
-        reset unless new_string
+        case source
+        when String
+          self.string = source
+        when Array
+          self.string = source.join
+        when nil
+          reset
+        else
+          raise ArgumentError, 'expected String, Array, or nil'
+        end
         scan_tokens @tokens, options
         @cached_tokens = @tokens
+        if source.is_a? Array
+          @tokens.split_into_parts(*source.map { |part| part.size })
+        else
+          @tokens
+        end
       end
       
       # Caches the result of tokenize.