Upporting changes from 0.9.2 (vs. 0.9.1).

6 files changed, 54 insertions, 23 deletions

diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index d43805d..56857dc 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -262,7 +262,7 @@ module Encoders
         @out << (@css_style[@opened] || '<span>')
         @opened << type
       when :close
-        if $CODERAY_DEBUG and @opened.last != type
+        if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
           warn 'Malformed token stream: Trying to close a token (%p) ' \
             'that is not open. Open are: %p.' % [type, @opened[1..-1]]
         end
@@ -283,7 +283,7 @@ module Encoders
         end
         @opened << type
       when :end_line
-        if $CODERAY_DEBUG and @opened.last != type
+        if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
           warn 'Malformed token stream: Trying to close a line (%p) ' \
             'that is not open. Open are: %p.' % [type, @opened[1..-1]]
         end
diff --git a/lib/coderay/encoders/terminal.rb b/lib/coderay/encoders/terminal.rb
index 782147a..7224218 100644
--- a/lib/coderay/encoders/terminal.rb
+++ b/lib/coderay/encoders/terminal.rb
@@ -19,6 +19,7 @@ module CodeRay
       register_for :terminal
       
       TOKEN_COLORS = {
+        :annotation => '35',
         :attribute_name => '33',
         :attribute_value => '31',
         :bin => '1;35',
@@ -29,10 +30,13 @@ module CodeRay
         :class_variable => '36',
         :color => '32',
         :comment => '37',
+        :complex => '34',
         :constant => ['34', '4'],
+        :decoration => '35',
         :definition => '1;32',
         :directive => ['32', '4'],
         :doc => '46',
+        :doctype => '1;30',
         :doc_string => ['31', '4'],
         :entity => '33',
         :error => ['1;33', '41'],
@@ -44,6 +48,7 @@ module CodeRay
         :include => '33',
         :integer => '1;34',
         :interpreted => '1;35',
+        :key => '35',
         :label => '1;15',
         :local_variable => '33',
         :oct => '1;35',
@@ -52,6 +57,7 @@ module CodeRay
         :pre_type => '1;30',
         :predefined => ['4', '1;34'],
         :preprocessor => '36',
+        :pseudo_class => '34',
         :regexp => {
           :self => '31',
           :content => '31',
@@ -60,20 +66,32 @@ module CodeRay
           :function => '1;29'
         },
         :reserved => '1;31',
-        :shell => {:self => '42', :content => '1;29'},
-        :string => '32',
+        :shell => {
+          :self => '42',
+          :content => '1;29',
+          :delimiter => '37',
+        },
+        :string => {
+          :self => '32',
+          :modifier => '1;32',
+          :escape => '1;36',
+          :delimiter => '1;32',
+        },
         :symbol => '1;32',
         :tag => '34',
         :tag_special => ['34', '4'],
         :type => '1;34',
+        :value => '36',
         :variable => '34',
         
         :insert => '42',
         :delete => '41',
         :change => '44',
-        :head => '41'
+        :head => '45'
       }
+      TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
+      TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
       TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
 
     protected
diff --git a/lib/coderay/for_redcloth.rb b/lib/coderay/for_redcloth.rb
index 43b2268..5149562 100644
--- a/lib/coderay/for_redcloth.rb
+++ b/lib/coderay/for_redcloth.rb
@@ -13,10 +13,20 @@ module CodeRay
   module ForRedCloth
     
     def self.install
-      gem 'RedCloth', '>= 4.0.3' rescue nil
+      gem 'RedCloth', '>= 4.0.3' if defined? gem
       require 'redcloth'
       unless RedCloth::VERSION.to_s >= '4.0.3'
-        raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later.'
+        if defined? gem
+          raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later. ' +
+            "You have #{RedCloth::VERSION}. Please gem install RedCloth."
+        else
+          $".delete 'redcloth.rb'  # sorry, but it works
+          require 'rubygems'
+          return install  # retry
+        end
+      end
+      unless RedCloth::VERSION.to_s >= '4.2.2'
+        warn 'CodeRay.for_redcloth works best with RedCloth version 4.2.2 or later.'
       end
       RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
       RedCloth::Formatters::HTML.module_eval do
diff --git a/lib/coderay/scanners/php.rb b/lib/coderay/scanners/php.rb
index 73c4fc8..289e795 100644
--- a/lib/coderay/scanners/php.rb
+++ b/lib/coderay/scanners/php.rb
@@ -216,6 +216,7 @@ module Scanners
         \.(?!\d)=? |      # dot that is not decimal point, string concatenation
         && | \|\| |       # logic
         :: | -> | => |    # scope, member, dictionary
+        \\(?!\n) |        # namespace
         \+\+ | -- |       # increment, decrement
         [,;?:()\[\]{}] |  # simple delimiters
         [-+*\/%&|^]=? |   # ordinary math, binary logic, assignment shortcuts
diff --git a/lib/coderay/scanners/yaml.rb b/lib/coderay/scanners/yaml.rb
index fa29701..62a6aba 100644
--- a/lib/coderay/scanners/yaml.rb
+++ b/lib/coderay/scanners/yaml.rb
@@ -125,9 +125,9 @@ module Scanners
         
         if $CODERAY_DEBUG and not kind
           raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
+            [[match, kind], line], tokens, state
         end
-        raise_inspect 'Empty token', tokens unless match
+        raise_inspect 'Empty token', tokens, state unless match
         
         tokens << [match, kind]
         
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index 0cefef2..2a0dc15 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -7,29 +7,30 @@ module CodeRay
   #
   # A token is not a special object, just a two-element Array
   # consisting of
+  # * the _token_ _text_ (the original source of the token in a String) or
+  #   a _token_ _action_ (:open, :close, :begin_line, :end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
-  # * the _token_ _text_ (the original source of the token in a String)
   #
   # A token looks like this:
   #
-  #   [:comment, '# It looks like this']
-  #   [:float, '3.1415926']
-  #   [:error, '$^']
+  #   ['# It looks like this', :comment]
+  #   ['3.1415926', :float]
+  #   ['$^', :error]
   #
-  # Some scanners also yield some kind of sub-tokens, represented by special
-  # token texts, namely :open and :close .
+  # Some scanners also yield sub-tokens, represented by special
+  # token actions, namely :open and :close.
   #
   # The Ruby scanner, for example, splits "a string" into:
   #
   #  [
   #   [:open, :string],
-  #   [:delimiter, '"'],
-  #   [:content, 'a string'],
-  #   [:delimiter, '"'],
+  #   ['"', :delimiter],
+  #   ['a string', :content],
+  #   ['"', :delimiter],
   #   [:close, :string]
   #  ]
   #
-  # Tokens is also the interface between Scanners and Encoders:
+  # Tokens is the interface between Scanners and Encoders:
   # The input is split and saved into a Tokens object. The Encoder
   # then builds the output from this object.
   #
@@ -43,6 +44,9 @@ module CodeRay
   # Tokens gives you the power to handle pre-scanned code very easily:
   # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
   # that you put in your DB.
+  # 
+  # It also allows you to generate tokens directly (without using a scanner),
+  # to load them from a file, and still use any Encoder that CodeRay provides.
   #
   # Tokens' subclass TokenStream allows streaming to save memory.
   class Tokens < Array
@@ -128,7 +132,6 @@ module CodeRay
     # for example, consecutive //-comment lines could already be
     # joined in one comment token by the Scanner.
     def optimize
-      # print ' Tokens#optimize: before: %d - ' % size
       last_kind = last_text = nil
       new = self.class.new
       for text, kind in self
@@ -147,7 +150,6 @@ module CodeRay
         end
       end
       new << [last_text, last_kind] if last_kind
-      # print 'after: %d (%d saved = %2.0f%%)' % [new.size, size - new.size, 1.0 - (new.size.to_f / size)]
       new
     end
 
@@ -302,11 +304,11 @@ module CodeRay
     #
     #   require 'coderay'
     #   
-    #   token_stream = CodeRay::TokenStream.new do |kind, text|
+    #   token_stream = CodeRay::TokenStream.new do |text, kind|
     #     puts 'kind: %s, text size: %d.' % [kind, text.size]
     #   end
     #   
-    #   token_stream << [:regexp, '/\d+/']
+    #   token_stream << ['/\d+/', :regexp]
     #   #-> kind: rexpexp, text size: 5.
     #
     def initialize &block