Got rid of the old streaming system (see #142).

28 files changed, 17 insertions, 120 deletions

diff --git a/Changes.textile b/Changes.textile
index 3145fee..12fc6ae 100644
--- a/Changes.textile
+++ b/Changes.textile
@@ -29,10 +29,6 @@ h3. @Tokens@
 * *REMOVED* method @stream?@.
 * *NEW* methods @encode_with@, @count@, @begin_group@, @end_group@, @begin_line@, and @end_line@.
 
-h3. @TokenStream@
- 
-Removed.
-
 h3. *RENAMED*: @Tokens::AbbreviationForKind@
  
 Renamed from @ClassOfKind@; the term "token class" is no longer used in CodeRay. Instead, tokens have _kinds_.
diff --git a/bin/coderay b/bin/coderay
index 7f8271d..ef819f7 100644
--- a/bin/coderay
+++ b/bin/coderay
@@ -73,7 +73,7 @@ Examples:
   end
   
   if tokens == :scan
-    output = CodeRay::Duo[lang => format].highlight input, :stream => true
+    output = CodeRay::Duo[lang => format].highlight input
   else
     output = tokens.encode format
   end
diff --git a/lib/coderay.rb b/lib/coderay.rb
index ef2574a..ad0bb5e 100644
--- a/lib/coderay.rb
+++ b/lib/coderay.rb
@@ -170,27 +170,6 @@ module CodeRay
       scan file, lang, options = {}, &block
     end
 
-    # Scan the +code+ (a string) with the scanner for +lang+.
-    #
-    # Calls scan.
-    #
-    # See CodeRay.scan.
-    def scan_stream code, lang, options = {}, &block
-      options[:stream] = true
-      scan code, lang, options, &block
-    end
-
-    # Encode a string in Streaming mode.
-    #
-    # This starts scanning +code+ with the the Scanner for +lang+
-    # while encodes the output with the Encoder for +format+.
-    # +options+ will be passed to the Encoder.
-    #
-    # See CodeRay::Encoder.encode_stream
-    def encode_stream code, lang, format, options = {}
-      encoder(format, options).encode_stream code, lang, options
-    end
-
     # Encode a string.
     #
     # This scans +code+ with the the Scanner for +lang+ and then
@@ -286,12 +265,6 @@ module CodeRay
 
   end
 
-  # A dummy module that is included by subclasses of
-  # CodeRay::Scanners::Scanner and CodeRay::Encoders::Encoder
-  # to show that they are able to handle streams.
-  module Streamable
-  end
-
 end
 
 # Run a test script.
diff --git a/lib/coderay/duo.rb b/lib/coderay/duo.rb
index 5468dda..c11bd3f 100644
--- a/lib/coderay/duo.rb
+++ b/lib/coderay/duo.rb
@@ -23,9 +23,6 @@ module CodeRay
     # simple:
     #   CodeRay::Duo[:ruby, :page].highlight 'bla 42'
     # 
-    # streaming:
-    #   CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
-    # 
     # with options:
     #   CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
     # 
@@ -64,18 +61,9 @@ module CodeRay
     end
     
     # Tokenize and highlight the code using +scanner+ and +encoder+.
-    #
-    # If the :stream option is set, the Duo will go into streaming mode,
-    # saving memory for the cost of time.
-    def encode code, options = { :stream => false }
-      stream = options.delete :stream
+    def encode code, options = {}
       options = @options.merge options
-      if stream
-        encoder.encode_stream(code, @lang, options)
-      else
-        scanner.code = code
-        encoder.encode_tokens(scanner.tokenize, options)
-      end
+      encoder.encode(code, @lang, options)
     end
     alias highlight encode
 
diff --git a/lib/coderay/encoder.rb b/lib/coderay/encoder.rb
index 82545c4..716f516 100644
--- a/lib/coderay/encoder.rb
+++ b/lib/coderay/encoder.rb
@@ -27,8 +27,6 @@ module CodeRay
       extend Plugin
       plugin_host Encoders
 
-      attr_reader :token_stream
-
       class << self
 
         # If FILE_EXTENSION isn't defined, this method returns the
@@ -44,7 +42,7 @@ module CodeRay
       end
 
       # Subclasses are to store their default options in this constant.
-      DEFAULT_OPTIONS = { :stream => false }
+      DEFAULT_OPTIONS = { }
 
       # The options you gave the Encoder at creating.
       attr_accessor :options
@@ -56,7 +54,6 @@ module CodeRay
       # Encoder objects provide three encode methods:
       # - encode simply takes a +code+ string and a +lang+
       # - encode_tokens expects a +tokens+ object instead
-      # - encode_stream is like encode, but uses streaming mode.
       #
       # Each method has an optional +options+ parameter. These are
       # added to the options you passed at creation.
@@ -75,31 +72,20 @@ module CodeRay
         finish options
       end
 
-      # Encode the given +code+ after tokenizing it using the Scanner
-      # for +lang+.
+      # Encode the given +code+ using the Scanner for +lang+.
       def encode code, lang, options = {}
         options = @options.merge options
-        scanner_options = CodeRay.get_scanner_options(options)
-        tokens = CodeRay.scan code, lang, scanner_options
-        encode_tokens tokens, options
-      end
-
-      # You can use highlight instead of encode, if that seems
-      # more clear to you.
-      alias highlight encode
-
-      # Encode the given +code+ using the Scanner for +lang+ in
-      # streaming mode.
-      def encode_stream code, lang, options = {}
-        options = @options.merge options
         setup options
         scanner_options = CodeRay.get_scanner_options options
         scanner_options[:tokens] = self
-        @token_stream =
-          CodeRay.scan_stream code, lang, scanner_options
+        CodeRay.scan code, lang, scanner_options
         finish options
       end
 
+      # You can use highlight instead of encode, if that seems
+      # more clear to you.
+      alias highlight encode
+
       # Return the default file extension for outputs of this encoder.
       def file_extension
         self.class::FILE_EXTENSION
diff --git a/lib/coderay/encoders/count.rb b/lib/coderay/encoders/count.rb
index 451a7f8..f4dd0c8 100644
--- a/lib/coderay/encoders/count.rb
+++ b/lib/coderay/encoders/count.rb
@@ -7,7 +7,6 @@ module Encoders
   # Text and block tokens are counted.
   class Count < Encoder
     
-    include Streamable
     register_for :count
     
   protected
diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 89e430f..c97631b 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -15,7 +15,6 @@ module Encoders
   # See also: Scanners::Debug
   class Debug < Encoder
 
-    include Streamable
     register_for :debug
 
     FILE_EXTENSION = 'raydebug'
diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb
index 807fb42..0a55a02 100644
--- a/lib/coderay/encoders/html.rb
+++ b/lib/coderay/encoders/html.rb
@@ -89,7 +89,6 @@ module Encoders
   # Default: false
   class HTML < Encoder
 
-    include Streamable
     register_for :html
 
     FILE_EXTENSION = 'html'
diff --git a/lib/coderay/encoders/null.rb b/lib/coderay/encoders/null.rb
index 7a8899b..b815c2f 100644
--- a/lib/coderay/encoders/null.rb
+++ b/lib/coderay/encoders/null.rb
@@ -6,7 +6,6 @@ module Encoders
   # Does nothing and returns an empty string.
   class Null < Encoder
 
-    include Streamable
     register_for :null
 
     # Defined for faster processing.
diff --git a/lib/coderay/encoders/statistic.rb b/lib/coderay/encoders/statistic.rb
index d267b21..455da46 100644
--- a/lib/coderay/encoders/statistic.rb
+++ b/lib/coderay/encoders/statistic.rb
@@ -7,7 +7,6 @@ module Encoders
   # Alias: +stats+
   class Statistic < Encoder
 
-    include Streamable
     register_for :stats, :statistic
 
     attr_reader :type_stats, :real_token_count  # :nodoc:
diff --git a/lib/coderay/encoders/text.rb b/lib/coderay/encoders/text.rb
index ecbf624..2dfb224 100644
--- a/lib/coderay/encoders/text.rb
+++ b/lib/coderay/encoders/text.rb
@@ -14,7 +14,6 @@ module Encoders
   # Default: empty String
   class Text < Encoder
 
-    include Streamable
     register_for :text
 
     FILE_EXTENSION = 'txt'
diff --git a/lib/coderay/encoders/token_kind_filter.rb b/lib/coderay/encoders/token_kind_filter.rb
index fd3df44..431794d 100644
--- a/lib/coderay/encoders/token_kind_filter.rb
+++ b/lib/coderay/encoders/token_kind_filter.rb
@@ -25,7 +25,6 @@ module Encoders
   # See also: CommentFilter
   class TokenKindFilter < Filter
 
-    include Streamable
     register_for :token_kind_filter
 
     DEFAULT_OPTIONS = {
diff --git a/lib/coderay/encoders/xml.rb b/lib/coderay/encoders/xml.rb
index 0006d75..d3e8832 100644
--- a/lib/coderay/encoders/xml.rb
+++ b/lib/coderay/encoders/xml.rb
@@ -6,7 +6,6 @@ module Encoders
   # Uses REXML. Very slow.
   class XML < Encoder
 
-    include Streamable
     register_for :xml
 
     FILE_EXTENSION = 'xml'
diff --git a/lib/coderay/for_redcloth.rb b/lib/coderay/for_redcloth.rb
index e439929..2af9082 100644
--- a/lib/coderay/for_redcloth.rb
+++ b/lib/coderay/for_redcloth.rb
@@ -57,7 +57,7 @@ module CodeRay
             @in_bc ||= nil
             format = @in_bc ? :div : :span
             opts[:text] = unescape(opts[:text]) unless @in_bc
-            highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
+            highlighted_code = CodeRay.encode opts[:text], opts[:lang], format
             highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
             highlighted_code
           else
diff --git a/lib/coderay/scanner.rb b/lib/coderay/scanner.rb
index 286561d..22f1c67 100644
--- a/lib/coderay/scanner.rb
+++ b/lib/coderay/scanner.rb
@@ -55,7 +55,7 @@ module CodeRay
       # The default options for all scanner classes.
       #
       # Define @default_options for subclasses.
-      DEFAULT_OPTIONS = { :stream => false }
+      DEFAULT_OPTIONS = { }
       
       KINDS_NOT_LOC = [:comment, :doctype]
 
@@ -108,10 +108,9 @@ module CodeRay
       # * +options+ is a Hash with Symbols as keys.
       #   It is merged with the default options of the class (you can
       #   overwrite default options here.)
-      # * +block+ is the callback for streamed highlighting.
       #
       # Else, a Tokens object is used.
-      def initialize code='', options = {}, &block
+      def initialize code='', options = {}
         raise "I am only the basic Scanner class. I can't scan "\
           "anything. :( Use my subclasses." if self.class == Scanner
         
@@ -119,14 +118,7 @@ module CodeRay
 
         super Scanner.normify(code)
 
-        @tokens = options[:tokens]
-        if @options[:stream]
-          raise NotImplementedError unless @tokens.is_a? Encoders::Encoder
-        else
-          warn "warning in CodeRay::Scanner.new: Block given, "\
-            "but :stream is #{@options[:stream]}" if block_given?
-          @tokens ||= Tokens.new
-        end
+        @tokens = options[:tokens] || Tokens.new
         @tokens.scanner = self if @tokens.respond_to? :scanner=
 
         setup
@@ -158,14 +150,9 @@ module CodeRay
       def tokenize new_string=nil, options = {}
         options = @options.merge(options)
         self.string = new_string if new_string
-        @cached_tokens =
-          if @options[:stream]  # :stream must have been set already
-            reset unless new_string
-            scan_tokens @tokens, options
-            @tokens
-          else
-            scan_tokens @tokens, options
-          end
+        reset unless new_string
+        scan_tokens @tokens, options
+        @cached_tokens = @tokens
       end
       
       # Caches the result of tokenize.
@@ -173,11 +160,6 @@ module CodeRay
         @cached_tokens ||= tokenize
       end
       
-      # Whether the scanner is in streaming mode.
-      def streaming?
-        !!@options[:stream]
-      end
-
       # Traverses the tokens.
       def each &block
         tokens.each(&block)
diff --git a/lib/coderay/scanners/c.rb b/lib/coderay/scanners/c.rb
index 45ca42e..bd182e8 100644
--- a/lib/coderay/scanners/c.rb
+++ b/lib/coderay/scanners/c.rb
@@ -4,8 +4,6 @@ module Scanners
   # Scanner for C.
   class C < Scanner
 
-    include Streamable
-    
     register_for :c
     file_extension 'c'
     
diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb
index 7531892..cb79768 100644
--- a/lib/coderay/scanners/cpp.rb
+++ b/lib/coderay/scanners/cpp.rb
@@ -6,8 +6,6 @@ module Scanners
   # Aliases: +cplusplus+, c++
   class CPlusPlus < Scanner
 
-    include Streamable
-    
     register_for :cpp
     file_extension 'cpp'
     title 'C++'
diff --git a/lib/coderay/scanners/debug.rb b/lib/coderay/scanners/debug.rb
index 0f2b89f..deaa30b 100644
--- a/lib/coderay/scanners/debug.rb
+++ b/lib/coderay/scanners/debug.rb
@@ -7,7 +7,6 @@ module Scanners
   # Parses the output of the Encoders::Debug encoder.
   class Debug < Scanner
 
-    include Streamable
     register_for :debug
     file_extension 'raydebug'
     title 'CodeRay Token Dump'
diff --git a/lib/coderay/scanners/groovy.rb b/lib/coderay/scanners/groovy.rb
index fdbbbc7..2334bc6 100644
--- a/lib/coderay/scanners/groovy.rb
+++ b/lib/coderay/scanners/groovy.rb
@@ -6,7 +6,6 @@ module Scanners
   # Scanner for Groovy.
   class Groovy < Java
     
-    include Streamable
     register_for :groovy
     
     # TODO: check list of keywords
diff --git a/lib/coderay/scanners/html.rb b/lib/coderay/scanners/html.rb
index 8f71e0e..7fab3cc 100644
--- a/lib/coderay/scanners/html.rb
+++ b/lib/coderay/scanners/html.rb
@@ -8,7 +8,6 @@ module Scanners
   # See also: Scanners::XML
   class HTML < Scanner
 
-    include Streamable
     register_for :html
     
     KINDS_NOT_LOC = [
diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb
index e7becda..427e754 100644
--- a/lib/coderay/scanners/java.rb
+++ b/lib/coderay/scanners/java.rb
@@ -4,7 +4,6 @@ module Scanners
   # Scanner for Java.
   class Java < Scanner
 
-    include Streamable
     register_for :java
     helper :builtin_types
     
diff --git a/lib/coderay/scanners/java_script.rb b/lib/coderay/scanners/java_script.rb
index 3ae8d80..f97e6da 100644
--- a/lib/coderay/scanners/java_script.rb
+++ b/lib/coderay/scanners/java_script.rb
@@ -6,8 +6,6 @@ module Scanners
   # Aliases: +ecmascript+, +ecma_script+, +javascript+
   class JavaScript < Scanner
     
-    include Streamable
-    
     register_for :java_script
     file_extension 'js'
     
diff --git a/lib/coderay/scanners/json.rb b/lib/coderay/scanners/json.rb
index 668fd82..b36811f 100644
--- a/lib/coderay/scanners/json.rb
+++ b/lib/coderay/scanners/json.rb
@@ -4,8 +4,6 @@ module Scanners
   # Scanner for JSON (JavaScript Object Notation).
   class JSON < Scanner
     
-    include Streamable
-    
     register_for :json
     file_extension 'json'
     
diff --git a/lib/coderay/scanners/nitro_xhtml.rb b/lib/coderay/scanners/nitro_xhtml.rb
index ba8ee71..cf8c5aa 100644
--- a/lib/coderay/scanners/nitro_xhtml.rb
+++ b/lib/coderay/scanners/nitro_xhtml.rb
@@ -9,7 +9,6 @@ module Scanners
   # Alias: +nitro+
   class NitroXHTML < Scanner
     
-    include Streamable
     register_for :nitro_xhtml
     file_extension :xhtml
     title 'Nitro XHTML'
diff --git a/lib/coderay/scanners/plaintext.rb b/lib/coderay/scanners/plaintext.rb
index e176403..b8ea5f5 100644
--- a/lib/coderay/scanners/plaintext.rb
+++ b/lib/coderay/scanners/plaintext.rb
@@ -11,8 +11,6 @@ module Scanners
     register_for :plaintext, :plain
     title 'Plain text'
     
-    include Streamable
-    
     KINDS_NOT_LOC = [:plain]  # :nodoc:
     
   protected
diff --git a/lib/coderay/scanners/python.rb b/lib/coderay/scanners/python.rb
index 568ed57..b01248a 100644
--- a/lib/coderay/scanners/python.rb
+++ b/lib/coderay/scanners/python.rb
@@ -7,8 +7,6 @@ module Scanners
   # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
   class Python < Scanner
     
-    include Streamable
-    
     register_for :python
     file_extension 'py'
     
diff --git a/lib/coderay/scanners/rhtml.rb b/lib/coderay/scanners/rhtml.rb
index 064a92c..1922948 100644
--- a/lib/coderay/scanners/rhtml.rb
+++ b/lib/coderay/scanners/rhtml.rb
@@ -7,7 +7,6 @@ module Scanners
   # Scanner for HTML ERB templates.
   class RHTML < Scanner
     
-    include Streamable
     register_for :rhtml
     title 'HTML ERB Template'
     
diff --git a/lib/coderay/scanners/ruby.rb b/lib/coderay/scanners/ruby.rb
index dcbfce0..a2e5fed 100644
--- a/lib/coderay/scanners/ruby.rb
+++ b/lib/coderay/scanners/ruby.rb
@@ -17,8 +17,6 @@ module Scanners
   # Alias: +irb+
   class Ruby < Scanner
 
-    include Streamable
-
     register_for :ruby
     file_extension 'rb'