Added ability to scan an Array of Strings as one with Scanner#tokenize.

See issue #222.

2 files changed, 19 insertions, 3 deletions

diff --git a/Changes.textile b/Changes.textile
index 293d12f..44daa5f 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -104,6 +104,9 @@ h3. @Scanners::Scanner@
  
 * *REMOVED* helper method @String#to_unix@.
 * *REMOVED* method @streamable?@.
+* *NEW*: The @#tokenize@ method also takes an Array of Strings as source. The
+  code is highlighted as one and split into parts of the input lengths
+  after that using @Tokens#split_into_parts@.
 
 h3. @Scanners::CSS@
  
diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 7d154da..64b3c83 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -147,14 +147,27 @@ module CodeRay
       end
 
       # Scans the code and returns all tokens in a Tokens object.
-      def tokenize new_string=nil, options = {}
+      def tokenize source = nil, options = {}
         options = @options.merge(options)
         @tokens = options[:tokens] || @tokens || Tokens.new
         @tokens.scanner = self if @tokens.respond_to? :scanner=
-        self.string = new_string if new_string
-        reset unless new_string
+        case source
+        when String
+          self.string = source
+        when Array
+          self.string = source.join
+        when nil
+          reset
+        else
+          raise ArgumentError, 'expected String, Array, or nil'
+        end
         scan_tokens @tokens, options
         @cached_tokens = @tokens
+        if source.is_a? Array
+          @tokens.split_into_parts(*source.map { |part| part.size })
+        else
+          @tokens
+        end
       end
       
       # Caches the result of tokenize.