Merge pull request #138 from rubychan/remove-dump

Remove dump/undump functionality from Tokens

5 files changed, 14 insertions, 136 deletions

diff --git a/bench/bench.rb b/bench/bench.rb
index 1889eed..1958c73 100644
--- a/bench/bench.rb
+++ b/bench/bench.rb
@@ -14,11 +14,10 @@ require 'coderay'
 lang = ARGV.fetch(0) do
   puts <<-HELP
 Usage:
-  ruby bench.rb (c|ruby|dump) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
+  ruby bench.rb (c|ruby) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
 
   SIZE defaults to 100 kB (= 100,000 bytes).
   SIZE = 0 means the whole input.
-  SIZE is ignored when dump is input.
 
 -p generates a profile (slow! use with SIZE = 1)
 -o shows the output
@@ -48,10 +47,6 @@ if format == 'comp'
   end
 end
 
-$dump_input = lang == 'dump'
-$dump_output = format == 'dump'
-require 'coderay/helpers/gzip_simple.rb' if $dump_input
-
 def here fn = nil
   return MYDIR unless fn
   File.join here, fn
@@ -66,59 +61,38 @@ N.times do
 
   data = nil
   File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
-  if $dump_input
-    @size = CodeRay::Tokens.load(data).text.size
-  else
-    raise 'Example file is empty.' if data.empty?
-    unless @size.zero?
-      data += data until data.size >= @size
-      data = data[0, @size]
-    end
-    @size = data.size
+  raise 'Example file is empty.' if data.empty?
+  unless @size.zero?
+    data += data until data.size >= @size
+    data = data[0, @size]
   end
-
+  @size = data.size
+  
   options = {
     :tab_width => 2,
     # :line_numbers => :inline,
     :css => $style ? :style : :class,
   }
-  $hl = CodeRay.encoder(format, options) unless $dump_output
+  $hl = CodeRay.encoder(format, options)
   time = bm.report('CodeRay') do
     if $stream || true
-      if $dump_input
-        raise 'Can\'t stream dump.'
-      elsif $dump_output
-        raise 'Can\'t dump stream.'
-      end
       $o = $hl.encode(data, lang, options)
     else
-      if $dump_input
-        tokens = CodeRay::Tokens.load data
-      else
-        tokens = CodeRay.scan(data, lang)
-      end
+      tokens = CodeRay.scan(data, lang)
       tokens.optimize! if $optimize
-      if $dump_output
-        $o = tokens.optimize.dump
-      else
-        $o = tokens.encode($hl)
-      end
+      $o = tokens.encode($hl)
     end
   end
-  $file_created = here('test.' +
-    ($dump_output ? 'dump' : $hl.file_extension))
+  $file_created = here('test.' + $hl.file_extension)
   File.open($file_created, 'wb') do |f|
     # f.write $o
   end
-  Dir.chdir(here) do
-    FileUtils.copy 'test.dump', 'example.dump' if $dump_output
-  end
-
+  
   time_real = time.real
-
+  
   puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
   puts $o if ARGV.include? '-o'
-
+  
 end
 end
 puts "Files created: #$file_created"
diff --git a/lib/coderay/encoders/debug.rb b/lib/coderay/encoders/debug.rb
index 61520a1..f4db330 100644
--- a/lib/coderay/encoders/debug.rb
+++ b/lib/coderay/encoders/debug.rb
@@ -9,7 +9,6 @@ module Encoders
   #
   # You cannot fully restore the tokens information from the
   # output, because consecutive :space tokens are merged.
-  # Use Tokens#dump for caching purposes.
   # 
   # See also: Scanners::Debug
   class Debug < Encoder
diff --git a/lib/coderay/helpers/gzip.rb b/lib/coderay/helpers/gzip.rb
deleted file mode 100644
index 245014a..0000000
--- a/lib/coderay/helpers/gzip.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-module CodeRay
-  
-  # A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
-  module GZip
-    
-    require 'zlib'
-    
-    # The default zipping level. 7 zips good and fast.
-    DEFAULT_GZIP_LEVEL = 7
-    
-    # Unzips the given string +s+.
-    #
-    # Example:
-    #   require 'gzip_simple'
-    #   print GZip.gunzip(File.read('adresses.gz'))
-    def GZip.gunzip s
-      Zlib::Inflate.inflate s
-    end
-    
-    # Zips the given string +s+.
-    #
-    # Example:
-    #   require 'gzip_simple'
-    #   File.open('adresses.gz', 'w') do |file
-    #     file.write GZip.gzip('Mum: 0123 456 789', 9)
-    #   end
-    #
-    # If you provide a +level+, you can control how strong
-    # the string is compressed:
-    # - 0: no compression, only convert to gzip format
-    # - 1: compress fast
-    # - 7: compress more, but still fast (default)
-    # - 8: compress more, slower
-    # - 9: compress best, very slow
-    def GZip.gzip s, level = DEFAULT_GZIP_LEVEL
-      Zlib::Deflate.new(level).deflate s, Zlib::FINISH
-    end
-    
-  end
-  
-end
diff --git a/lib/coderay/tokens.rb b/lib/coderay/tokens.rb
index ad59b7f..54358d4 100644
--- a/lib/coderay/tokens.rb
+++ b/lib/coderay/tokens.rb
@@ -1,8 +1,5 @@
 module CodeRay
   
-  # GZip library for writing and reading token dumps.
-  autoload :GZip, coderay_path('helpers', 'gzip')
-  
   # The Tokens class represents a list of tokens returned from
   # a Scanner. It's actually just an Array with a few helper methods.
   #
@@ -148,53 +145,11 @@ module CodeRay
       parts
     end
     
-    # Dumps the object into a String that can be saved
-    # in files or databases.
-    #
-    # The dump is created with Marshal.dump;
-    # In addition, it is gzipped using GZip.gzip.
-    #
-    # The returned String object includes Undumping
-    # so it has an #undump method. See Tokens.load.
-    #
-    # You can configure the level of compression,
-    # but the default value 7 should be what you want
-    # in most cases as it is a good compromise between
-    # speed and compression rate.
-    #
-    # See GZip module.
-    def dump gzip_level = 7
-      dump = Marshal.dump self
-      dump = GZip.gzip dump, gzip_level
-      dump.extend Undumping
-    end
-    
     # Return the actual number of tokens.
     def count
       size / 2
     end
     
-    # Include this module to give an object an #undump
-    # method.
-    #
-    # The string returned by Tokens.dump includes Undumping.
-    module Undumping
-      # Calls Tokens.load with itself.
-      def undump
-        Tokens.load self
-      end
-    end
-    
-    # Undump the object using Marshal.load, then
-    # unzip it using GZip.gunzip.
-    #
-    # The result is commonly a Tokens object, but
-    # this is not guaranteed.
-    def Tokens.load dump
-      dump = GZip.gunzip dump
-      @dump = Marshal.load dump
-    end
-    
     alias text_token push
     def begin_group kind; push :begin_group, kind end
     def end_group kind; push :end_group, kind end
diff --git a/test/unit/tokens.rb b/test/unit/tokens.rb
index 86dc632..73b0fd5 100644
--- a/test/unit/tokens.rb
+++ b/test/unit/tokens.rb
@@ -18,15 +18,6 @@ class TokensTest < Test::Unit::TestCase
     assert_equal tokens.count, 4
   end
   
-  def test_dump_undump
-    tokens = make_tokens
-    tokens2 = nil
-    assert_nothing_raised do
-      tokens2 = tokens.dump.undump
-    end
-    assert_equal tokens, tokens2
-  end
-  
   def test_to_s
     assert_equal 'string()', make_tokens.to_s
   end