start to use a faster string buffer

8 files changed, 116 insertions, 48 deletions

diff --git a/Rakefile b/Rakefile
index e2f950c..edb3731 100644
--- a/Rakefile
+++ b/Rakefile
@@ -158,12 +158,14 @@ desc "Benchmarking parser"
 task :benchmark_parser do
   ENV['RUBYOPT'] = "-Ilib:ext #{ENV['RUBYOPT']}"
   myruby 'benchmarks/parser_benchmark.rb'
+  myruby 'benchmarks/parser2_benchmark.rb'
 end
 
 desc "Benchmarking generator"
 task :benchmark_generator do
   ENV['RUBYOPT'] = "-Ilib:ext #{ENV['RUBYOPT']}"
   myruby 'benchmarks/generator_benchmark.rb'
+  myruby 'benchmarks/generator2_benchmark.rb'
 end
 
 desc "Benchmarking library"
diff --git a/benchmarks/generator2_benchmark.rb b/benchmarks/generator2_benchmark.rb
index 60ba17f..bd99db9 100755
--- a/benchmarks/generator2_benchmark.rb
+++ b/benchmarks/generator2_benchmark.rb
@@ -182,22 +182,22 @@ if $0 == __FILE__
     Generator2BenchmarkYajl.run
   else
     system "#{RAKE_PATH} clean"
-#    system "#{RUBY_PATH} #$0 rails"
-#    system "#{RUBY_PATH} #$0 pure"
-    system "#{RAKE_PATH} compile_ext"
-    system "#{RUBY_PATH} #$0 ext"
-    system "#{RUBY_PATH} #$0 yajl"
+    system "#{RUBY_PATH} #$0 rails"
+    system "#{RUBY_PATH} #$0 pure"
+#    system "#{RAKE_PATH} compile_ext"
+#    system "#{RUBY_PATH} #$0 ext"
+#    system "#{RUBY_PATH} #$0 yajl"
     Bullshit.compare do
       output_filename File.join(File.dirname(__FILE__), 'data', 'Generator2BenchmarkComparison.log')
 
-      benchmark Generator2BenchmarkExt,    :generator_fast,    :load => yes
-      benchmark Generator2BenchmarkExt,    :generator_safe,    :load => yes
-      benchmark Generator2BenchmarkExt,    :generator_pretty,  :load => yes
-#      benchmark Generator2BenchmarkPure,   :generator_fast,    :load => yes
-#      benchmark Generator2BenchmarkPure,   :generator_safe,    :load => yes
-#      benchmark Generator2BenchmarkPure,   :generator_pretty,  :load => yes
-#      benchmark Generator2BenchmarkRails,  :generator,         :load => yes
-      benchmark Generator2BenchmarkYajl,   :generator,         :load => yes
+#      benchmark Generator2BenchmarkExt,    :generator_fast,    :load => yes
+#      benchmark Generator2BenchmarkExt,    :generator_safe,    :load => yes
+#      benchmark Generator2BenchmarkExt,    :generator_pretty,  :load => yes
+      benchmark Generator2BenchmarkPure,   :generator_fast,    :load => yes
+      benchmark Generator2BenchmarkPure,   :generator_safe,    :load => yes
+      benchmark Generator2BenchmarkPure,   :generator_pretty,  :load => yes
+      benchmark Generator2BenchmarkRails,  :generator,         :load => yes
+#      benchmark Generator2BenchmarkYajl,   :generator,         :load => yes
     end
   end
 end
diff --git a/benchmarks/generator_benchmark.rb b/benchmarks/generator_benchmark.rb
index 539fb91..69d4f17 100755
--- a/benchmarks/generator_benchmark.rb
+++ b/benchmarks/generator_benchmark.rb
@@ -186,20 +186,20 @@ if $0 == __FILE__
     system "#{RAKE_PATH} clean"
     system "#{RUBY_PATH} #$0 rails"
     system "#{RUBY_PATH} #$0 pure"
-    system "#{RAKE_PATH} compile_ext"
-    system "#{RUBY_PATH} #$0 ext"
-    system "#{RUBY_PATH} #$0 yajl"
+#    system "#{RAKE_PATH} compile_ext"
+#    system "#{RUBY_PATH} #$0 ext"
+#    system "#{RUBY_PATH} #$0 yajl"
     Bullshit.compare do
       output_filename File.join(File.dirname(__FILE__), 'data', 'GeneratorBenchmarkComparison.log')
 
-      benchmark GeneratorBenchmarkExt,    :generator_fast,    :load => yes
-      benchmark GeneratorBenchmarkExt,    :generator_safe,    :load => yes
-      benchmark GeneratorBenchmarkExt,    :generator_pretty,  :load => yes
+#      benchmark GeneratorBenchmarkExt,    :generator_fast,    :load => yes
+#      benchmark GeneratorBenchmarkExt,    :generator_safe,    :load => yes
+#      benchmark GeneratorBenchmarkExt,    :generator_pretty,  :load => yes
       benchmark GeneratorBenchmarkPure,   :generator_fast,    :load => yes
       benchmark GeneratorBenchmarkPure,   :generator_safe,    :load => yes
       benchmark GeneratorBenchmarkPure,   :generator_pretty,  :load => yes
       benchmark GeneratorBenchmarkRails,  :generator,         :load => yes
-      benchmark GeneratorBenchmarkYajl,   :generator,         :load => yes
+#      benchmark GeneratorBenchmarkYajl,   :generator,         :load => yes
     end
   end
 end
diff --git a/benchmarks/parser_benchmark.rb b/benchmarks/parser_benchmark.rb
index 87aba31..7ac027e 100755
--- a/benchmarks/parser_benchmark.rb
+++ b/benchmarks/parser_benchmark.rb
@@ -221,9 +221,9 @@ if $0 == __FILE__
     ParserBenchmarkYajl.run
   else
     system "#{RAKE_PATH} clean"
-    system "#{RUBY_PATH} #$0 yaml"
-    system "#{RUBY_PATH} #$0 rails"
-    system "#{RUBY_PATH} #$0 pure"
+    #system "#{RUBY_PATH} #$0 yaml"
+    #system "#{RUBY_PATH} #$0 rails"
+    #system "#{RUBY_PATH} #$0 pure"
     system "#{RAKE_PATH} compile_ext"
     system "#{RUBY_PATH} #$0 ext"
     system "#{RUBY_PATH} #$0 yajl"
@@ -231,10 +231,10 @@ if $0 == __FILE__
       output_filename File.join(File.dirname(__FILE__), 'data', 'ParserBenchmarkComparison.log')
 
       benchmark ParserBenchmarkExt,   :parser, :load => yes
-      benchmark ParserBenchmarkPure,  :parser, :load => yes
-      benchmark ParserBenchmarkYAML,  :parser, :load => yes
-      benchmark ParserBenchmarkRails, :parser, :load => yes
-      benchmark ParserBenchmarkYajl, :parser, :load => yes
+      #benchmark ParserBenchmarkPure,  :parser, :load => yes
+      #benchmark ParserBenchmarkYAML,  :parser, :load => yes
+      #benchmark ParserBenchmarkRails, :parser, :load => yes
+      benchmark ParserBenchmarkYajl,  :parser, :load => yes
     end
   end
 end
diff --git a/ext/json/ext/generator/fbuffer.c b/ext/json/ext/generator/fbuffer.c
new file mode 100644
index 0000000..507a432
--- /dev/null
+++ b/ext/json/ext/generator/fbuffer.c
@@ -0,0 +1,49 @@
+#include "ruby.h"
+#include "fbuffer.h"
+
+FBuffer *fbuffer_alloc()
+{
+    FBuffer *fb = ALLOC(FBuffer);
+    memset((void *) fb, 0, sizeof(FBuffer));
+    return fb;
+}
+
+void fbuffer_free(FBuffer *fb)
+{
+    if (fb->ptr) ruby_xfree(fb->ptr);
+    ruby_xfree(fb);
+}
+
+inline void fbuffer_inc_capa(FBuffer *fb, unsigned int requested)
+{
+    unsigned int required;
+
+    if (!fb->ptr) {
+        fb->ptr = ALLOC_N(unsigned char, FBUFFER_INITIAL_LENGTH);
+        fb->capa = FBUFFER_INITIAL_LENGTH;
+    }
+
+    for (required = fb->capa; requested > required - fb->len; required <<= 1);
+
+    if (required > fb->capa) {
+        fb->ptr = (unsigned char *) REALLOC_N((long*) fb->ptr, unsigned char, required);
+        fb->capa = required;
+    }
+}
+
+inline void fbuffer_append(FBuffer *fb, const unsigned char *newstr, unsigned int len)
+{
+    if (len > 0) {
+        fbuffer_inc_capa(fb, len);
+        memcpy(fb->ptr + fb->len, newstr, len);
+        fb->len += len;
+    }
+}
+
+inline void fbuffer_append_char(FBuffer *fb, const unsigned char newchr)
+{
+    fbuffer_inc_capa(fb, 1);
+    *(fb->ptr + fb->len) = newchr;
+    fb->len++;
+}
+
diff --git a/ext/json/ext/generator/fbuffer.h b/ext/json/ext/generator/fbuffer.h
new file mode 100644
index 0000000..d789da8
--- /dev/null
+++ b/ext/json/ext/generator/fbuffer.h
@@ -0,0 +1,17 @@
+typedef struct FBufferStruct {
+    unsigned char *ptr;
+    unsigned int len;
+    unsigned int capa;
+} FBuffer;
+
+#define FBUFFER_INITIAL_LENGTH 4096
+
+#define FBUFFER_PTR(fb) (fb->ptr)
+
+#define FBUFFER_LEN(fb) (fb->len)
+
+FBuffer *fbuffer_alloc();
+void fbuffer_free(FBuffer *fb);
+inline void fbuffer_inc_capa(FBuffer *fb, unsigned int requested);
+inline void fbuffer_append(FBuffer *fb, const unsigned char *newstr, unsigned int len);
+inline void fbuffer_append_char(FBuffer *fb, const unsigned char newchr);
diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c
index d2dfe52..78b4ff9 100644
--- a/ext/json/ext/parser/parser.c
+++ b/ext/json/ext/parser/parser.c
@@ -21,8 +21,8 @@
 #ifdef HAVE_RUBY_ENCODING_H
 #include "ruby/encoding.h"
 #define FORCE_UTF8(obj) rb_enc_associate((obj), rb_utf8_encoding())
-static VALUE mEncoding_ASCII_8BIT, mEncoding_UTF_8, mEncoding_UTF_16BE,
-    mEncoding_UTF_16LE, mEncoding_UTF_32BE, mEncoding_UTF_32LE;
+static VALUE CEncoding_ASCII_8BIT, CEncoding_UTF_8, CEncoding_UTF_16BE,
+    CEncoding_UTF_16LE, CEncoding_UTF_32BE, CEncoding_UTF_32LE;
 static ID i_encoding, i_encode, i_encode_bang, i_force_encoding;
 #else
 #define FORCE_UTF8(obj)
@@ -1499,28 +1499,28 @@ inline static VALUE convert_encoding(VALUE source)
 #ifdef HAVE_RUBY_ENCODING_H
     {
         VALUE encoding = rb_funcall(source, i_encoding, 0);
-        if (encoding == mEncoding_ASCII_8BIT) {
+        if (encoding == CEncoding_ASCII_8BIT) {
             if (len >= 4 &&  ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0) {
                 source = rb_str_dup(source);
-                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_32BE);
-                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+                rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_32BE);
+                source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
             } else if (len >= 4 && ptr[0] == 0 && ptr[2] == 0) {
                 source = rb_str_dup(source);
-                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_16BE);
-                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+                rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_16BE);
+                source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
             } else if (len >= 4 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == 0) {
                 source = rb_str_dup(source);
-                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_32LE);
-                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+                rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_32LE);
+                source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
             } else if (len >= 4 && ptr[1] == 0 && ptr[3] == 0) {
                 source = rb_str_dup(source);
-                rb_funcall(source, i_force_encoding, 1, mEncoding_UTF_16LE);
-                source = rb_funcall(source, i_encode_bang, 1, mEncoding_UTF_8);
+                rb_funcall(source, i_force_encoding, 1, CEncoding_UTF_16LE);
+                source = rb_funcall(source, i_encode_bang, 1, CEncoding_UTF_8);
             } else {
                 FORCE_UTF8(source);
             }
         } else {
-            source = rb_funcall(source, i_encode, 1, mEncoding_UTF_8);
+            source = rb_funcall(source, i_encode, 1, CEncoding_UTF_8);
         }
     }
 #else
@@ -1863,12 +1863,12 @@ void Init_parser()
     i_object_class = rb_intern("object_class");
     i_array_class = rb_intern("array_class");
 #ifdef HAVE_RUBY_ENCODING_H
-    mEncoding_UTF_8 = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-8"));
-    mEncoding_UTF_16BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16be"));
-    mEncoding_UTF_16LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16le"));
-    mEncoding_UTF_32BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32be"));
-    mEncoding_UTF_32LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32le"));
-    mEncoding_ASCII_8BIT = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("ascii-8bit"));
+    CEncoding_UTF_8 = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-8"));
+    CEncoding_UTF_16BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16be"));
+    CEncoding_UTF_16LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-16le"));
+    CEncoding_UTF_32BE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32be"));
+    CEncoding_UTF_32LE = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("utf-32le"));
+    CEncoding_ASCII_8BIT = rb_funcall(rb_path2class("Encoding"), rb_intern("find"), 1, rb_str_new2("ascii-8bit"));
     i_encoding = rb_intern("encoding");
     i_encode = rb_intern("encode");
     i_encode_bang = rb_intern("encode!");
diff --git a/tests/test_json_generate.rb b/tests/test_json_generate.rb
index e725e6f..e72c562 100755
--- a/tests/test_json_generate.rb
+++ b/tests/test_json_generate.rb
@@ -91,13 +91,13 @@ EOT
     #assert s.check_circular
     h = { 1=>2 }
     h[3] = h
-    assert_raises(JSON::CircularDatastructure) {  generate(h) }
-    assert_raises(JSON::CircularDatastructure) {  generate(h, s) }
+    assert_raises(JSON::NestingError) {  generate(h) }
+    assert_raises(JSON::NestingError) {  generate(h, s) }
     s = JSON.state.new(:check_circular => true)
     #assert s.check_circular
     a = [ 1, 2 ]
     a << a
-    assert_raises(JSON::CircularDatastructure) {  generate(a, s) }
+    assert_raises(JSON::NestingError) {  generate(a, s) }
   end
 
   def test_allow_nan