cleanups, documentation for TokensProxy

11 files changed, 46 insertions, 138 deletions

diff --git a/Changes-1.0.textile b/Changes-1.0.textile
index 2503f4a..5448767 100644
--- a/Changes-1.0.textile
+++ b/Changes-1.0.textile
@@ -63,6 +63,7 @@ h3. @Tokens@
  
 * *NEW* methods @count@, @begin_group@, @end_group@, @begin_line@, and @end_line@.
 * *REMOVED* methods @#stream?@, @#each_text_token@.
+* *REMOVED* methods @#optimize@, @#fix@, @#split_into_lines@ along with their bang! variants.
 * *REMOVED* @#text@ and @#text_size@ methods. Use the @Text@ encoder instead.
 * *REMOVED* special implementation of @#each@ taking a filter parameter. Use @TokenKindFilter@ instead.
 
diff --git a/etc/CodeRay.tmproj b/etc/CodeRay.tmproj
index 8a505bb..afbaf9d 100644
--- a/etc/CodeRay.tmproj
+++ b/etc/CodeRay.tmproj
@@ -5,8 +5,6 @@
 	<key>documents</key>
 	<array>
 		<dict>
-			<key>expanded</key>
-			<true/>
 			<key>name</key>
 			<string>lib</string>
 			<key>regexFolderFilter</key>
@@ -15,6 +13,8 @@
 			<string>../lib</string>
 		</dict>
 		<dict>
+			<key>expanded</key>
+			<true/>
 			<key>name</key>
 			<string>bin</string>
 			<key>regexFolderFilter</key>
@@ -26,15 +26,15 @@
 			<key>filename</key>
 			<string>../coderay.gemspec</string>
 			<key>lastUsed</key>
-			<date>2011-08-23T02:18:44Z</date>
+			<date>2011-09-20T13:58:45Z</date>
+			<key>selected</key>
+			<true/>
 		</dict>
 		<dict>
 			<key>filename</key>
 			<string>../Changes-1.0.textile</string>
 			<key>lastUsed</key>
-			<date>2011-09-14T00:53:48Z</date>
-			<key>selected</key>
-			<true/>
+			<date>2011-09-19T00:26:49Z</date>
 		</dict>
 		<dict>
 			<key>filename</key>
@@ -101,6 +101,8 @@
 			<string>../rake_helpers</string>
 		</dict>
 		<dict>
+			<key>expanded</key>
+			<true/>
 			<key>name</key>
 			<string>rake_tasks</string>
 			<key>regexFolderFilter</key>
@@ -122,7 +124,7 @@
 			<key>filename</key>
 			<string>../Rakefile</string>
 			<key>lastUsed</key>
-			<date>2011-08-23T02:18:45Z</date>
+			<date>2011-09-18T23:49:40Z</date>
 		</dict>
 		<dict>
 			<key>name</key>
diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index cc331d1..d2d6c7e 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -34,7 +34,7 @@ module CodeRay
         # downcase class name instead.
         def const_missing sym
           if sym == :FILE_EXTENSION
-            (@plugin_id || name[/\w+$/].downcase).to_s
+            (defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s
           else
             super
           end
diff --git a/lib/coderay/scanners/java/builtin_types.rb b/lib/coderay/scanners/java/builtin_types.rb
index 8087edd..d1b8b73 100644
--- a/lib/coderay/scanners/java/builtin_types.rb
+++ b/lib/coderay/scanners/java/builtin_types.rb
@@ -3,6 +3,7 @@ module Scanners
   
   module Java::BuiltinTypes  # :nodoc:
     
+    #:nocov:
     List = %w[
       AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection
       AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel
@@ -412,6 +413,7 @@ module Scanners
       XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver
       XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView
     ]
+    #:nocov:
     
   end
   
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index ee28a4e..045cf4a 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -83,97 +83,6 @@ module CodeRay
       super
     end
     
-    # Returns the tokens compressed by joining consecutive
-    # tokens of the same kind.
-    #
-    # This can not be undone, but should yield the same output
-    # in most Encoders.  It basically makes the output smaller.
-    #
-    # Combined with dump, it saves space for the cost of time.
-    #
-    # If the scanner is written carefully, this is not required -
-    # for example, consecutive //-comment lines could already be
-    # joined in one comment token by the Scanner.
-    def optimize
-      raise NotImplementedError, 'Tokens#optimize needs to be rewritten.'
-      # last_kind = last_text = nil
-      # new = self.class.new
-      # for text, kind in self
-      #   if text.is_a? String
-      #     if kind == last_kind
-      #       last_text << text
-      #     else
-      #       new << [last_text, last_kind] if last_kind
-      #       last_text = text
-      #       last_kind = kind
-      #     end
-      #   else
-      #     new << [last_text, last_kind] if last_kind
-      #     last_kind = last_text = nil
-      #     new << [text, kind]
-      #   end
-      # end
-      # new << [last_text, last_kind] if last_kind
-      # new
-    end
-    
-    # Compact the object itself; see optimize.
-    def optimize!
-      replace optimize
-    end
-    
-    # Ensure that all begin_group tokens have a correspondent end_group.
-    #
-    # TODO: Test this!
-    def fix
-      raise NotImplementedError, 'Tokens#fix needs to be rewritten.'
-      # tokens = self.class.new
-      # # Check token nesting using a stack of kinds.
-      # opened = []
-      # for type, kind in self
-      #   case type
-      #   when :begin_group
-      #     opened.push [:begin_group, kind]
-      #   when :begin_line
-      #     opened.push [:end_line, kind]
-      #   when :end_group, :end_line
-      #     expected = opened.pop
-      #     if [type, kind] != expected
-      #       # Unexpected end; decide what to do based on the kind:
-      #       # - token was never opened: delete the end (just skip it)
-      #       next unless opened.rindex expected
-      #       # - token was opened earlier: also close tokens in between
-      #       tokens << token until (token = opened.pop) == expected
-      #     end
-      #   end
-      #   tokens << [type, kind]
-      # end
-      # # Close remaining opened tokens
-      # tokens << token while token = opened.pop
-      # tokens
-    end
-    
-    def fix!
-      replace fix
-    end
-    
-    # TODO: Scanner#split_into_lines
-    # 
-    # Makes sure that:
-    # - newlines are single tokens
-    #   (which means all other token are single-line)
-    # - there are no open tokens at the end the line
-    #
-    # This makes it simple for encoders that work line-oriented,
-    # like HTML with list-style numeration.
-    def split_into_lines
-      raise NotImplementedError
-    end
-    
-    def split_into_lines!
-      replace split_into_lines
-    end
-    
     # Split the tokens into parts of the given +sizes+.
     # 
     # The result will be an Array of Tokens objects. The parts have
diff --git a/lib/coderay/tokens_proxy.rb b/lib/coderay/tokens_proxy.rb
index 2a20435..31ff39b 100644
--- a/lib/coderay/tokens_proxy.rb
+++ b/lib/coderay/tokens_proxy.rb
@@ -1,7 +1,23 @@
 module CodeRay
   
-  class TokensProxy < Struct.new :input, :lang, :options, :block
+  # The result of a scan operation is a TokensProxy, but should act like Tokens.
+  # 
+  # This proxy makes it possible to use the classic CodeRay.scan.encode API
+  # while still providing the benefits of direct streaming.
+  class TokensProxy
     
+    attr_accessor :input, :lang, :options, :block
+    
+    # Create a new TokensProxy with the arguments of CodeRay.scan.
+    def initialize input, lang, options = {}, block = nil
+      @input   = input
+      @lang    = lang
+      @options = options
+      @block   = block
+    end
+    
+    # Call CodeRay.encode if +encoder+ is a Symbol;
+    # otherwise, convert the receiver to tokens and call encoder.encode_tokens.
     def encode encoder, options = {}
       if encoder.respond_to? :to_sym
         CodeRay.encode(input, lang, encoder, options)
@@ -10,29 +26,30 @@ module CodeRay
       end
     end
     
+    # Tries to call encode;
+    # delegates to tokens otherwise.
     def method_missing method, *args, &blk
-      encode method, *args
+      encode method.to_sym, *args
     rescue PluginHost::PluginNotFound
       tokens.send(method, *args, &blk)
     end
     
+    # The (cached) result of the tokenized input; a Tokens instance.
     def tokens
       @tokens ||= scanner.tokenize(input)
     end
     
+    # A (cached) scanner instance to use for the scan task.
     def scanner
       @scanner ||= CodeRay.scanner(lang, options, &block)
     end
     
+    # Overwrite Struct#each.
     def each *args, &blk
       tokens.each(*args, &blk)
       self
     end
     
-    def count
-      tokens.count
-    end
-    
   end
   
 end
diff --git a/rake_helpers/html_coderay_generator.rb b/rake_helpers/html_coderay_generator.rb
index 4216865..3c77705 100644
--- a/rake_helpers/html_coderay_generator.rb
+++ b/rake_helpers/html_coderay_generator.rb
@@ -1087,13 +1087,12 @@ module Generators
     def markup_code(tokens)
       code = tokens.map { |t| t.text }.join
       options = {
-              :css => :class,
-              :line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1,
-              :bold_every => :no_bolding,
+        :css => :class,
+        :line_numbers_start => code[/\A.*?, line (\d+)/,1].to_i - 1,
+        :bold_every => :no_bolding,
       }
       options[:line_numbers] = nil unless Options.instance.include_line_numbers
-      out = CodeRay.scan(code, :ruby).div(options)
-      out.wrap! :div
+      CodeRay.scan(code, :ruby).div(options)
     end
 
     # we rely on the fact that the first line of a source code
diff --git a/test/functional/basic.rb b/test/functional/basic.rb
index 5d03513..bf289b3 100755
--- a/test/functional/basic.rb
+++ b/test/functional/basic.rb
@@ -148,7 +148,9 @@ more code  # and another comment, in-line.
       end
     end
     assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
-    assert_equal false, CodeRay::TokenKinds[:shibboleet]
+    assert_warning 'Undefined Token kind: :shibboleet' do
+      assert_equal false, CodeRay::TokenKinds[:shibboleet]
+    end
   end
   
   class Milk < CodeRay::Encoders::Encoder
diff --git a/test/functional/suite.rb b/test/functional/suite.rb
index 5490f98..ec23eec 100755
--- a/test/functional/suite.rb
+++ b/test/functional/suite.rb
@@ -1,5 +1,6 @@
 require 'test/unit'
 
+$VERBOSE = $CODERAY_DEBUG = true
 $:.unshift File.expand_path('../../../lib', __FILE__)
 require 'coderay'
 
diff --git a/test/unit/suite.rb b/test/unit/suite.rb
index ee568e7..417dfed 100755
--- a/test/unit/suite.rb
+++ b/test/unit/suite.rb
@@ -1,5 +1,7 @@
 require 'test/unit'
 require 'rubygems'
+
+$VERBOSE = $CODERAY_DEBUG = true
 $:.unshift 'lib'
 
 mydir = File.dirname(__FILE__)
diff --git a/test/unit/tokens.rb b/test/unit/tokens.rb
index 4fc9833..86dc632 100644
--- a/test/unit/tokens.rb
+++ b/test/unit/tokens.rb
@@ -37,33 +37,6 @@ class TokensTest < Test::Unit::TestCase
     end
   end
   
-  def test_optimize
-    assert_raise NotImplementedError do
-      make_tokens.optimize
-    end
-    assert_raise NotImplementedError do
-      make_tokens.optimize!
-    end
-  end
-  
-  def test_fix
-    assert_raise NotImplementedError do
-      make_tokens.fix
-    end
-    assert_raise NotImplementedError do
-      make_tokens.fix!
-    end
-  end
-  
-  def test_split_into_lines
-    assert_raise NotImplementedError do
-      make_tokens.split_into_lines
-    end
-    assert_raise NotImplementedError do
-      make_tokens.split_into_lines!
-    end
-  end
-  
   def test_split_into_parts
     parts_4_3 = [
       ["stri", :type],