diff options
| -rw-r--r-- | Changes.textile | 3 | ||||
| -rw-r--r-- | lib/coderay/scanner.rb | 19 |
2 files changed, 19 insertions, 3 deletions
diff --git a/Changes.textile b/Changes.textile index 293d12f..44daa5f 100644 --- a/Changes.textile +++ b/Changes.textile @@ -104,6 +104,9 @@ h3. @Scanners::Scanner@ * *REMOVED* helper method @String#to_unix@. * *REMOVED* method @streamable?@. +* *NEW*: The @#tokenize@ method also takes an Array of Strings as source. The + code is highlighted as one and split into parts of the input lengths + after that using @Tokens#split_into_parts@. h3. @Scanners::CSS@ diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb index 7d154da..64b3c83 100644 --- a/lib/coderay/scanner.rb +++ b/lib/coderay/scanner.rb @@ -147,14 +147,27 @@ module CodeRay end # Scans the code and returns all tokens in a Tokens object. - def tokenize new_string=nil, options = {} + def tokenize source = nil, options = {} options = @options.merge(options) @tokens = options[:tokens] || @tokens || Tokens.new @tokens.scanner = self if @tokens.respond_to? :scanner= - self.string = new_string if new_string - reset unless new_string + case source + when String + self.string = source + when Array + self.string = source.join + when nil + reset + else + raise ArgumentError, 'expected String, Array, or nil' + end scan_tokens @tokens, options @cached_tokens = @tokens + if source.is_a? Array + @tokens.split_into_parts(*source.map { |part| part.size }) + else + @tokens + end end # Caches the result of tokenize. |