From d0af8aa9f11bfd9c6ded625f5591db25d42153ad Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 23 May 2010 21:10:49 +0900 Subject: ruby: rdoc --- ruby/pack.c | 132 ++++++++++++++++++++++++++++--- ruby/rbinit.c | 12 +++ ruby/unpack.c | 244 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 370 insertions(+), 18 deletions(-) (limited to 'ruby') diff --git a/ruby/pack.c b/ruby/pack.c index 387bab6..7f56923 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -51,6 +51,16 @@ static ID s_append; rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); \ } + +/* + * Document-method: NilClass#to_msgpack + * + * call-seq: + * nil.to_msgpack(out = '') -> String + * + * Serializes the nil into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -58,6 +68,16 @@ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: TrueClass#to_msgpack + * + * call-seq: + * true.to_msgpack(out = '') -> String + * + * Serializes the true into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -65,6 +85,16 @@ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: FalseClass#to_msgpack + * + * call-seq: + * false.to_msgpack(out = '') -> String + * + * Serializes false into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -73,6 +103,15 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self } +/* + * Document-method: Fixnum#to_msgpack + * + * call-seq: + * fixnum.to_msgpack(out = '') -> String + * + * Serializes the Fixnum into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -85,6 +124,15 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) #define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign) #endif +/* + * Document-method: Bignum#to_msgpack + * + * call-seq: + * bignum.to_msgpack(out = '') -> String + * + * Serializes the Bignum into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -97,6 +145,15 @@ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: Float#to_msgpack + * + * call-seq: + * float.to_msgpack(out = '') -> String + * + * Serializes the Float into raw bytes. + */ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -104,6 +161,15 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: String#to_msgpack + * + * call-seq: + * string.to_msgpack(out = '') -> String + * + * Serializes the String into raw bytes. + */ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -112,6 +178,15 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: Symbol#to_msgpack + * + * call-seq: + * symbol.to_msgpack(out = '') -> String + * + * Serializes the Symbol into raw bytes. + */ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -122,6 +197,16 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: Array#to_msgpack + * + * call-seq: + * array.to_msgpack(out = '') -> String + * + * Serializes the Array into raw bytes. + * This calls to_msgpack method reflectively for internal elements. + */ static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -134,6 +219,7 @@ static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + #ifndef RHASH_SIZE // Ruby 1.8 #define RHASH_SIZE(h) (RHASH(h)->tbl ? RHASH(h)->tbl->num_entries : 0) #endif @@ -146,6 +232,15 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out return ST_CONTINUE; } +/* + * Document-method: Hash#to_msgpack + * + * call-seq: + * hash.to_msgpack(out = '') -> String + * + * Serializes the Hash into raw bytes. + * This calls to_msgpack method reflectively for internal keys and values. + */ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -155,6 +250,17 @@ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) } +/** + * Document-method: MessagePack.pack + * + * call-seq: + * MessagePack.pack(object, out = '') -> String + * + * Serializes the object into raw bytes. The encoding of the string is ASCII-8BIT on Ruby 1.9. + * This method is same as object.to_msgpack(out = ''). + * + * _out_ is an object that implements *<<* method like String or IO. + */ static VALUE MessagePack_pack(int argc, VALUE* argv, VALUE self) { VALUE out; @@ -173,16 +279,22 @@ void Init_msgpack_pack(VALUE mMessagePack) { s_to_msgpack = rb_intern("to_msgpack"); s_append = rb_intern("<<"); - rb_define_method_id(rb_cNilClass, s_to_msgpack, MessagePack_NilClass_to_msgpack, -1); - rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); - rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); - rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1); - rb_define_method_id(rb_cBignum, s_to_msgpack, MessagePack_Bignum_to_msgpack, -1); - rb_define_method_id(rb_cFloat, s_to_msgpack, MessagePack_Float_to_msgpack, -1); - rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); - rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); - rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); - rb_define_method_id(rb_cSymbol, s_to_msgpack, MessagePack_Symbol_to_msgpack, -1); + + rb_define_method(rb_cNilClass, "to_msgpack", MessagePack_NilClass_to_msgpack, -1); + rb_define_method(rb_cTrueClass, "to_msgpack", MessagePack_TrueClass_to_msgpack, -1); + rb_define_method(rb_cFalseClass, "to_msgpack", MessagePack_FalseClass_to_msgpack, -1); + rb_define_method(rb_cFixnum, "to_msgpack", MessagePack_Fixnum_to_msgpack, -1); + rb_define_method(rb_cBignum, "to_msgpack", MessagePack_Bignum_to_msgpack, -1); + rb_define_method(rb_cFloat, "to_msgpack", MessagePack_Float_to_msgpack, -1); + rb_define_method(rb_cString, "to_msgpack", MessagePack_String_to_msgpack, -1); + rb_define_method(rb_cArray, "to_msgpack", MessagePack_Array_to_msgpack, -1); + rb_define_method(rb_cHash, "to_msgpack", MessagePack_Hash_to_msgpack, -1); + rb_define_method(rb_cSymbol, "to_msgpack", MessagePack_Symbol_to_msgpack, -1); + + /** + * MessagePack module is defined in rbinit.c file. + * mMessagePack = rb_define_module("MessagePack"); + */ rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1); } diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 80d1d8c..050abde 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -20,6 +20,18 @@ static VALUE mMessagePack; +/** + * Document-module: MessagePack + * + * MessagePack is a binary-based efficient object serialization library. + * It enables to exchange structured objects between many languages like JSON. + * But unlike JSON, it is very fast and small. + * + * require 'msgpack' + * msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" + * MessagePack.unpack(msg) #=> [1,2,3] + * + */ void Init_msgpack(void) { mMessagePack = rb_define_module("MessagePack"); diff --git a/ruby/unpack.c b/ruby/unpack.c index 3a95e5a..9a2c457 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -205,6 +205,12 @@ static int template_execute_wrap(msgpack_unpack_t* mp, static VALUE cUnpacker; + + +/** + * Document-module: MessagePack::UnpackerError + * + */ static VALUE eUnpackError; @@ -243,6 +249,22 @@ static ID append_method_of(VALUE stream) } } +/** + * Document-method: MessagePack::Unpacker#initialize + * + * call-seq: + * MessagePack::Unpacker.new(stream = nil) + * + * Creates instance of MessagePack::Unpacker. + * + * You can specify a _stream_ for input stream. + * It is required to implement *sysread* or *readpartial* method. + * + * With the input stream, buffers will be feeded into the deserializer automatically. + * + * Without the input stream, use *feed* method manually. Or you can manage the buffer manually + * with *execute*, *finished?*, *data* and *reset* methods. + */ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) { VALUE stream; @@ -268,12 +290,29 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) return self; } + +/** + * Document-method: MessagePack::Unpacker#stream + * + * call-seq: + * unpacker.stream + * + * Gets the input stream. + */ static VALUE MessagePack_Unpacker_stream_get(VALUE self) { UNPACKER(self, mp); return mp->user.stream; } +/** + * Document-method: MessagePack::Unpacker#stream= + * + * call-seq: + * unpacker.stream = stream + * + * Resets the input stream. You can set nil not to use input stream. + */ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) { UNPACKER(self, mp); @@ -282,6 +321,15 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) return val; } + +/** + * Document-method: MessagePack::Unpacker#feed + * + * call-seq: + * unpacker.feed(data) + * + * Fills the internal buffer with the specified buffer. + */ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) { UNPACKER(self, mp); @@ -290,6 +338,20 @@ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) return Qnil; } +/** + * Document-method: MessagePack::Unpacker#fill + * + * call-seq: + * unpacker.fill -> length of read data + * + * Fills the internal buffer using the input stream. + * + * If the input stream is not specified, it returns nil. + * You can set it on *initialize* or *stream=* methods. + * + * This methods raises exceptions that _stream.sysread_ or + * _stream.readpartial_ method raises. + */ static VALUE MessagePack_Unpacker_fill(VALUE self) { UNPACKER(self, mp); @@ -313,6 +375,18 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) return LONG2FIX(len); } + +/** + * Document-method: MessagePack::Unpacker#each + * + * call-seq: + * unpacker.each {|object| } + * + * Deserializes objects repeatedly. This calls *fill* method automatically. + * + * UnpackError is throw when parse error is occured. + * This method raises exceptions that *fill* method raises. + */ static VALUE MessagePack_Unpacker_each(VALUE self) { UNPACKER(self, mp); @@ -352,6 +426,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self) return Qnil; } + static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen) { msgpack_unpack_t mp; @@ -376,12 +451,34 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon } } +/** + * Document-method: MessagePack::Unpacker.unpack_limit + * + * call-seq: + * MessagePack::Unpacker.unpack_limit(data, limit) -> object + * + * Deserializes one object over the specified buffer upto _limit_ bytes. + * + * UnpackError is throw when parse error is occured, the buffer is insufficient + * to deserialize one object or there are extra bytes. + */ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); return MessagePack_unpack_impl(self, data, NUM2ULONG(limit)); } +/** + * Document-method: MessagePack::Unpacker.unpack + * + * call-seq: + * MessagePack::Unpacker.unpack(data) -> object + * + * Deserializes one object over the specified buffer. + * + * UnpackError is throw when parse error is occured, the buffer is insufficient + * to deserialize one object or there are extra bytes. + */ static VALUE MessagePack_unpack(VALUE self, VALUE data) { CHECK_STRING_TYPE(data); @@ -411,7 +508,20 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, } } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#execute_limit + * + * call-seq: + * unpacker.unpack_limit(data, offset, limit) -> next offset + * + * Deserializes one object over the specified buffer from _offset_ bytes upto _limit_ bytes. + * + * This method doesn't use the internal buffer. + * + * Call *reset()* method before calling this method again. + * + * UnpackError is throw when parse error is occured. + */ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, VALUE off, VALUE limit) { @@ -420,7 +530,24 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, (size_t)NUM2ULONG(off), (size_t)NUM2ULONG(limit)); } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#execute + * + * call-seq: + * unpacker.unpack(data, offset) -> next offset + * + * Deserializes one object over the specified buffer from _offset_ bytes. + * + * This method doesn't use the internal buffer. + * + * Call *reset()* method before calling this method again. + * + * This returns offset that was parsed to. + * Use *finished?* method to check an object is deserialized and call *data* + * method if it returns true. + * + * UnpackError is throw when parse error is occured. + */ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) { CHECK_STRING_TYPE(data); @@ -428,7 +555,16 @@ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) (size_t)NUM2ULONG(off), (size_t)RSTRING_LEN(data)); } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#finished? + * + * call-seq: + * unpacker.finished? + * + * Returns true if an object is ready to get with data method. + * + * Use this method with execute method. + */ static VALUE MessagePack_Unpacker_finished_p(VALUE self) { UNPACKER(self, mp); @@ -438,14 +574,30 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self) return Qfalse; } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#data + * + * call-seq: + * unpacker.data + * + * Gets the object deserialized by execute method. + * + * Use this method with execute method. + */ static VALUE MessagePack_Unpacker_data(VALUE self) { UNPACKER(self, mp); return template_data(mp); } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#reset + * + * call-seq: + * unpacker.reset + * + * Resets the internal state of the unpacker. + */ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); @@ -467,20 +619,96 @@ void Init_msgpack_unpack(VALUE mMessagePack) eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); + rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1); + + /* Buffered API */ rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1); rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0); rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0); rb_define_method(cUnpacker, "stream", MessagePack_Unpacker_stream_get, 0); rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1); - rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); - rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); - /* backward compatibility */ + /* Unbuffered API */ rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); + + /** + * MessagePack module is defined in rbinit.c file. + * mMessagePack = rb_define_module("MessagePack"); + */ + rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); + rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); } +/** + * Document-module: MessagePack::Unpacker + * + * Deserializer class that includes Buffered API and Unbuffered API. + * + * + * Buffered API uses the internal buffer of the Unpacker. + * Following code uses Buffered API with an input stream: + * + * # create an unpacker with input stream. + * pac = MessagePack::Unpacker.new(stdin) + * + * # deserialize object one after another. + * pac.each {|obj| + * # ... + * } + * + * + * Following code doesn't use the input stream and feeds buffer + * using *fill* method. This is useful to use special stream + * or with event-driven I/O library. + * + * # create an unpacker without input stream. + * pac = MessagePack::Unpacker.new() + * + * # feed buffer to the internal buffer. + * pac.feed(input_bytes) + * + * # deserialize object one after another. + * pac.each {|obj| + * # ... + * } + * + * You can manage the buffer manually with the combination of + * *execute*, *finished?*, *data* and *reset* method. + * + * # create an unpacker. + * pac = MessagePack::Unpacker.new() + * + * # manage buffer and offset manually. + * offset = 0 + * buffer = '' + * + * # read some data into the buffer. + * buffer << [1,2,3].to_msgpack + * buffer << [4,5,6].to_msgpack + * + * while true + * offset = pac.execute(buffer, offset) + * + * if pac.finished? + * obj = pac.data + * + * buffer.slice!(0, offset) + * offset = 0 + * pac.reset + * + * # do something with the object + * # ... + * + * # repeat execution if there are more data. + * next unless buffer.empty? + * end + * + * break + * end + */ + -- cgit v1.2.1 From dbebe9771b276bd286b2ccdb8ac88dd17a83524c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 May 2010 02:55:58 +0900 Subject: ruby: update rdoc --- ruby/pack.c | 5 ----- ruby/rbinit.c | 8 ++++++++ ruby/unpack.c | 7 ++++--- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'ruby') diff --git a/ruby/pack.c b/ruby/pack.c index 7f56923..bbeac4a 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -59,7 +59,6 @@ static ID s_append; * nil.to_msgpack(out = '') -> String * * Serializes the nil into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -76,7 +75,6 @@ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) * true.to_msgpack(out = '') -> String * * Serializes the true into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -93,7 +91,6 @@ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) * false.to_msgpack(out = '') -> String * * Serializes false into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -110,7 +107,6 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self * fixnum.to_msgpack(out = '') -> String * * Serializes the Fixnum into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -131,7 +127,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) * bignum.to_msgpack(out = '') -> String * * Serializes the Bignum into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 050abde..4ad6beb 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -27,10 +27,18 @@ static VALUE mMessagePack; * It enables to exchange structured objects between many languages like JSON. * But unlike JSON, it is very fast and small. * + * You can install MessagePack with rubygems. + * + * gem install msgpack + * + * Simple usage is as follows. + * * require 'msgpack' * msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" * MessagePack.unpack(msg) #=> [1,2,3] * + * Use Unpacker class for streaming deserialization. + * */ void Init_msgpack(void) { diff --git a/ruby/unpack.c b/ruby/unpack.c index 9a2c457..dec40c6 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -654,7 +654,7 @@ void Init_msgpack_unpack(VALUE mMessagePack) * Following code uses Buffered API with an input stream: * * # create an unpacker with input stream. - * pac = MessagePack::Unpacker.new(stdin) + * pac = MessagePack::Unpacker.new(STDIN) * * # deserialize object one after another. * pac.each {|obj| @@ -663,8 +663,8 @@ void Init_msgpack_unpack(VALUE mMessagePack) * * * Following code doesn't use the input stream and feeds buffer - * using *fill* method. This is useful to use special stream - * or with event-driven I/O library. + * manually. This is useful to use special stream or with + * event-driven I/O library. * * # create an unpacker without input stream. * pac = MessagePack::Unpacker.new() @@ -677,6 +677,7 @@ void Init_msgpack_unpack(VALUE mMessagePack) * # ... * } * + * * You can manage the buffer manually with the combination of * *execute*, *finished?*, *data* and *reset* method. * -- cgit v1.2.1 From 26bc835c7e702769375201391bf4965b08c52516 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 04:30:49 +0900 Subject: ruby: buffer rewinding --- ruby/unpack.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) (limited to 'ruby') diff --git a/ruby/unpack.c b/ruby/unpack.c index dec40c6..ba20ae6 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -27,6 +27,11 @@ static ID s_readpartial; int s_ascii_8bit; #endif +static ID s_slice_bang; +#ifdef RUBY_VM +static ID s_clear; +#endif + typedef struct { int finished; VALUE source; @@ -125,6 +130,7 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) { *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } +//{ *o = rb_str_new(p, l); return 0; } #include "msgpack/unpack_template.h" @@ -249,6 +255,14 @@ static ID append_method_of(VALUE stream) } } +#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE +#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024) +#endif + +#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE +#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024) +#endif + /** * Document-method: MessagePack::Unpacker#initialize * @@ -283,9 +297,9 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) template_init(mp); mp->user.finished = 0; mp->user.offset = 0; - mp->user.buffer = rb_str_new("",0); + mp->user.buffer = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); mp->user.stream = stream; - mp->user.streambuf = rb_str_new("",0); + mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE); mp->user.stream_append_method = append_method_of(stream); return self; } @@ -322,6 +336,63 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) } +#ifdef RUBY_VM +# ifndef STR_SHARED +# define STR_SHARED FL_USER2 +# endif +# ifndef STR_NOEMBED +# define STR_NOEMBED FL_USER1 +# endif +# ifndef STR_ASSOC +# define STR_ASSOC FL_USER3 +# endif +# ifndef STR_NOCAPA_P +# define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,STR_SHARED|STR_ASSOC)) +# endif +# define NEED_MORE_CAPA(s,size) (!STR_NOCAPA_P(s) && RSTRING(s)->as.heap.aux.capa < size) +#else +# ifndef STR_NOCAPA +# ifndef STR_ASSOC +# define STR_ASSOC FL_USER3 +# endif +# ifndef ELTS_SHARED +# define ELTS_SHARED FL_USER2 +# endif +# define STR_NOCAPA (ELTS_SHARED|STR_ASSOC) +# endif +# define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size) +#endif + +static void try_rewind_buffer(msgpack_unpack_t* mp, size_t required) +{ + VALUE buffer = mp->user.buffer; + + size_t need_capa = RSTRING_LEN(buffer) + required; + + if(NEED_MORE_CAPA(buffer, need_capa)) { + /* FIXME +#ifdef RUBY_VM + if(RSTRING_LEN(buffer) <= mp->user.offset) { + rb_funcall(buffer, s_clear, 0); + mp->user.offset = 0; + return; + } +#endif + rb_funcall(buffer, s_slice_bang, 2, LONG2FIX(0), LONG2FIX(mp->user.offset)); + mp->user.offset = 0; + */ + size_t not_parsed = RSTRING_LEN(buffer) - mp->user.offset; + size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE * 2; + while(nsize < not_parsed + required) { + nsize *= 2; + } + VALUE nbuffer = rb_str_buf_new(nsize); + rb_str_buf_cat(nbuffer, RSTRING_PTR(buffer)+mp->user.offset, not_parsed); + mp->user.buffer = nbuffer; + mp->user.offset = 0; + } +} + /** * Document-method: MessagePack::Unpacker#feed * @@ -334,6 +405,7 @@ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) { UNPACKER(self, mp); StringValue(data); + try_rewind_buffer(mp, RSTRING_LEN(data)); rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); return Qnil; } @@ -363,12 +435,13 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) long len; if(RSTRING_LEN(mp->user.buffer) == 0) { rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(64*1024), mp->user.buffer); + LONG2FIX(MSGPACK_UNPACKER_BUFFER_INIT_SIZE), mp->user.buffer); len = RSTRING_LEN(mp->user.buffer); } else { rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(64*1024), mp->user.streambuf); + LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE), mp->user.streambuf); len = RSTRING_LEN(mp->user.streambuf); + try_rewind_buffer(mp, len); rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); } @@ -616,6 +689,11 @@ void Init_msgpack_unpack(VALUE mMessagePack) s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); #endif + s_slice_bang = rb_intern("slice!"); +#ifdef RUBY_VM + s_clear = rb_intern("clear"); +#endif + eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); -- cgit v1.2.1 From 5fa589691c780735fae153e7e6907906663349a1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 07:01:28 +0900 Subject: ruby: use malloc/realloc for stream buffer --- ruby/unpack.c | 178 ++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 117 insertions(+), 61 deletions(-) (limited to 'ruby') diff --git a/ruby/unpack.c b/ruby/unpack.c index ba20ae6..c4b5e29 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -27,16 +27,17 @@ static ID s_readpartial; int s_ascii_8bit; #endif -static ID s_slice_bang; -#ifdef RUBY_VM -static ID s_clear; -#endif +struct unpack_buffer { + size_t size; + size_t free; + char* ptr; +}; typedef struct { int finished; VALUE source; size_t offset; - VALUE buffer; + struct unpack_buffer buffer; VALUE stream; VALUE streambuf; ID stream_append_method; @@ -129,8 +130,16 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, #endif static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) -{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } +//{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } //{ *o = rb_str_new(p, l); return 0; } +{ + if(u->source == Qnil || l <= COW_MIN_SIZE) { + *o = rb_str_new(p, l); + } else { + *o = rb_str_substr(u->source, p - b, l); + } + return 0; +} #include "msgpack/unpack_template.h" @@ -209,6 +218,27 @@ static int template_execute_wrap(msgpack_unpack_t* mp, return ret; } +static int template_execute_wrap_each(msgpack_unpack_t* mp, + const char* ptr, size_t dlen, size_t* from) +{ + VALUE args[4] = { + (VALUE)mp, + (VALUE)ptr, + (VALUE)dlen, + (VALUE)from, + }; + + // FIXME execute実行中はmp->topが更新されないのでGC markが機能しない + rb_gc_disable(); + + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, Qnil); + + rb_gc_enable(); + + return ret; +} + static VALUE cUnpacker; @@ -220,15 +250,26 @@ static VALUE cUnpacker; static VALUE eUnpackError; +#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE +#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024) +#endif + +#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE +#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024) +#endif + static void MessagePack_Unpacker_free(void* data) { - if(data) { free(data); } + if(data) { + msgpack_unpack_t* mp = (msgpack_unpack_t*)data; + free(mp->user.buffer.ptr); + free(mp); + } } static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; - rb_gc_mark(mp->user.buffer); rb_gc_mark(mp->user.stream); rb_gc_mark(mp->user.streambuf); for(i=0; i < mp->top; ++i) { @@ -255,14 +296,6 @@ static ID append_method_of(VALUE stream) } } -#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE -#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024) -#endif - -#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE -#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024) -#endif - /** * Document-method: MessagePack::Unpacker#initialize * @@ -297,7 +330,9 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) template_init(mp); mp->user.finished = 0; mp->user.offset = 0; - mp->user.buffer = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); + mp->user.buffer.size = 0; + mp->user.buffer.free = 0; + mp->user.buffer.ptr = NULL; mp->user.stream = stream; mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE); mp->user.stream_append_method = append_method_of(stream); @@ -363,34 +398,66 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) # define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size) #endif -static void try_rewind_buffer(msgpack_unpack_t* mp, size_t required) +static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) { - VALUE buffer = mp->user.buffer; + struct unpack_buffer* buffer = &mp->user.buffer; + + if(buffer->size == 0) { + char* tmp = (char*)malloc(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); + // FIXME check tmp == NULL + buffer->ptr = tmp; + buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE; + buffer->size = 0; + + } else if(buffer->size <= mp->user.offset) { + /* clear buffer and rewind offset */ + buffer->free += buffer->size; + buffer->size = 0; + mp->user.offset = 0; + } - size_t need_capa = RSTRING_LEN(buffer) + required; + if(len <= buffer->free) { + /* enough free space: just copy */ + memcpy(buffer->ptr+buffer->size, ptr, len); + buffer->size += len; + buffer->free -= len; + return; + } - if(NEED_MORE_CAPA(buffer, need_capa)) { - /* FIXME -#ifdef RUBY_VM - if(RSTRING_LEN(buffer) <= mp->user.offset) { - rb_funcall(buffer, s_clear, 0); - mp->user.offset = 0; - return; + size_t csize = buffer->size + buffer->free; + + if(mp->user.offset <= buffer->size / 2) { + /* parsed less than half: realloc and copy */ + csize *= 2; + while(csize < buffer->size + len) { + csize *= 2; } -#endif - rb_funcall(buffer, s_slice_bang, 2, LONG2FIX(0), LONG2FIX(mp->user.offset)); - mp->user.offset = 0; - */ - size_t not_parsed = RSTRING_LEN(buffer) - mp->user.offset; - size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE * 2; - while(nsize < not_parsed + required) { - nsize *= 2; + char* tmp = (char*)realloc(buffer->ptr, csize); + // FIXME check tmp == NULL + memcpy(tmp + buffer->size, ptr, len); + buffer->ptr = tmp; + buffer->free = csize - buffer->size; + return; + } + + size_t not_parsed = buffer->size - mp->user.offset; + + if(csize < not_parsed + len) { + /* more buffer size */ + csize *= 2; + while(csize < not_parsed + len) { + csize *= 2; } - VALUE nbuffer = rb_str_buf_new(nsize); - rb_str_buf_cat(nbuffer, RSTRING_PTR(buffer)+mp->user.offset, not_parsed); - mp->user.buffer = nbuffer; - mp->user.offset = 0; + char* tmp = (char*)realloc(buffer->ptr, csize); + // FIXME check tmp == NULL + buffer->ptr = tmp; } + + memcpy(buffer->ptr+not_parsed, ptr, not_parsed); + buffer->size = not_parsed; + buffer->free = csize - buffer->size; + buffer->ptr = buffer->ptr; + mp->user.offset = 0; } /** @@ -405,8 +472,7 @@ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) { UNPACKER(self, mp); StringValue(data); - try_rewind_buffer(mp, RSTRING_LEN(data)); - rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); + feed_buffer(mp, RSTRING_PTR(data), RSTRING_LEN(data)); return Qnil; } @@ -432,18 +498,12 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) return Qnil; } - long len; - if(RSTRING_LEN(mp->user.buffer) == 0) { - rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(MSGPACK_UNPACKER_BUFFER_INIT_SIZE), mp->user.buffer); - len = RSTRING_LEN(mp->user.buffer); - } else { - rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE), mp->user.streambuf); - len = RSTRING_LEN(mp->user.streambuf); - try_rewind_buffer(mp, len); - rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); - } + rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, + LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE), + mp->user.streambuf); + + size_t len = RSTRING_LEN(mp->user.streambuf); + feed_buffer(mp, RSTRING_PTR(mp->user.streambuf), len); return LONG2FIX(len); } @@ -470,7 +530,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self) #endif while(1) { - if(RSTRING_LEN(mp->user.buffer) <= mp->user.offset) { + if(mp->user.buffer.size <= mp->user.offset) { do_fill: { VALUE len = MessagePack_Unpacker_fill(self); @@ -480,8 +540,9 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } } - ret = template_execute_wrap(mp, mp->user.buffer, - RSTRING_LEN(mp->user.buffer), &mp->user.offset); + ret = template_execute_wrap_each(mp, + mp->user.buffer.ptr, mp->user.buffer.size, + &mp->user.offset); if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -689,11 +750,6 @@ void Init_msgpack_unpack(VALUE mMessagePack) s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); #endif - s_slice_bang = rb_intern("slice!"); -#ifdef RUBY_VM - s_clear = rb_intern("clear"); -#endif - eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); -- cgit v1.2.1 From 94c39985079cff23d85d333ea69d554216ac79e0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 07:43:05 +0900 Subject: ruby: update gemspec --- ruby/README | 26 +++++++++++++++++--------- ruby/msgpack.gemspec | 6 +++--- ruby/pack.h | 2 +- ruby/rbinit.c | 4 ++-- ruby/unpack.h | 2 +- 5 files changed, 24 insertions(+), 16 deletions(-) (limited to 'ruby') diff --git a/ruby/README b/ruby/README index 859ae2b..051a769 100644 --- a/ruby/README +++ b/ruby/README @@ -1,29 +1,37 @@ = MessagePack - == Description +MessagePack is a binary-based efficient object serialization library. +It enables to exchange structured objects between many languages like JSON. +But unlike JSON, it is very fast and small. -== Installation +Simple usage is as follows: -=== Archive Installation + require 'msgpack' + msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" + MessagePack.unpack(msg) #=> [1,2,3] - rake install +Use MessagePack::Unpacker for streaming deserialization. -=== Gem Installation - gem install msgpack +== Installation +=== Archive Installation -== Features/Problems + ruby extconf.rb + make + make install +=== Gem Installation -== Synopsis + gem install msgpack == Copyright Author:: frsyuki -Copyright:: Copyright (c) 2008-2009 frsyuki +Copyright:: Copyright (c) 2008-2010 FURUHASHI Sadayuki License:: Apache License, Version 2.0 + diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index c5e8c8c..0f08628 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,14 +1,14 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.9" + s.version = "0.4.0" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" s.homepage = "http://msgpack.sourceforge.net/" s.rubyforge_project = "msgpack" - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.has_rdoc = true + s.rdoc_options = ["ext"] s.require_paths = ["lib"] s.files = Dir["ext/**/*", "msgpack/**/*", "test/**/*"] s.test_files = Dir["test/test_*.rb"] diff --git a/ruby/pack.h b/ruby/pack.h index c9b08a4..f162a86 100644 --- a/ruby/pack.h +++ b/ruby/pack.h @@ -1,7 +1,7 @@ /* * MessagePack for Ruby packing routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 4ad6beb..ad51f6b 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ static VALUE mMessagePack; * * gem install msgpack * - * Simple usage is as follows. + * Simple usage is as follows: * * require 'msgpack' * msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" diff --git a/ruby/unpack.h b/ruby/unpack.h index ce2a8de..91d3eb7 100644 --- a/ruby/unpack.h +++ b/ruby/unpack.h @@ -1,7 +1,7 @@ /* * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. -- cgit v1.2.1 From 47185d757ebee52cc06775a843b3bf06292d8bf1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 07:55:02 +0900 Subject: ruby: version 0.4.0 --- ruby/unpack.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'ruby') diff --git a/ruby/unpack.c b/ruby/unpack.c index c4b5e29..c93af35 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -130,8 +130,6 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, #endif static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) -//{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } -//{ *o = rb_str_new(p, l); return 0; } { if(u->source == Qnil || l <= COW_MIN_SIZE) { *o = rb_str_new(p, l); -- cgit v1.2.1 From 293293c23cf6672827248910f133b22d0d58fcf3 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 18:01:27 +0900 Subject: ruby: set mp->user.source = Qnil before tempalte_execute_do on Unpacker#each --- ruby/unpack.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) (limited to 'ruby') diff --git a/ruby/unpack.c b/ruby/unpack.c index c93af35..65852b9 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -156,15 +156,25 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha rb_raise(rb_eTypeError, "instance of String needed"); \ } - -static VALUE template_execute_rescue(VALUE nouse) -{ - rb_gc_enable(); #ifdef RUBY_VM - rb_exc_raise(rb_errinfo()); +#define RERAISE rb_exc_raise(rb_errinfo()) #else - rb_exc_raise(ruby_errinfo); +#define RERAISE rb_exc_raise(ruby_errinfo) #endif + + +static VALUE template_execute_rescue(VALUE data) +{ + rb_gc_enable(); + VALUE* resc = (VALUE*)data; + rb_enc_set_index(resc[0], (int)resc[1]); + RERAISE; +} + +static VALUE template_execute_rescue_each(VALUE nouse) +{ + rb_gc_enable(); + RERAISE; } static VALUE template_execute_do(VALUE argv) @@ -192,7 +202,6 @@ static int template_execute_wrap(msgpack_unpack_t* mp, }; #ifdef HAVE_RUBY_ENCODING_H - // FIXME encodingをASCII-8BITにする int enc_orig = rb_enc_get_index(str); rb_enc_set_index(str, s_ascii_8bit); #endif @@ -202,10 +211,10 @@ static int template_execute_wrap(msgpack_unpack_t* mp, mp->user.source = str; - int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue, Qnil); + VALUE resc[2] = {str, enc_orig}; - mp->user.source = Qnil; + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, (VALUE)resc); rb_gc_enable(); @@ -229,8 +238,10 @@ static int template_execute_wrap_each(msgpack_unpack_t* mp, // FIXME execute実行中はmp->topが更新されないのでGC markが機能しない rb_gc_disable(); + mp->user.source = Qnil; + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue, Qnil); + template_execute_rescue_each, Qnil); rb_gc_enable(); @@ -401,8 +412,7 @@ static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) struct unpack_buffer* buffer = &mp->user.buffer; if(buffer->size == 0) { - char* tmp = (char*)malloc(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); - // FIXME check tmp == NULL + char* tmp = ALLOC_N(char, MSGPACK_UNPACKER_BUFFER_INIT_SIZE); buffer->ptr = tmp; buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE; buffer->size = 0; @@ -430,8 +440,7 @@ static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) while(csize < buffer->size + len) { csize *= 2; } - char* tmp = (char*)realloc(buffer->ptr, csize); - // FIXME check tmp == NULL + char* tmp = REALLOC_N(buffer->ptr, char, csize); memcpy(tmp + buffer->size, ptr, len); buffer->ptr = tmp; buffer->free = csize - buffer->size; @@ -446,8 +455,7 @@ static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) while(csize < not_parsed + len) { csize *= 2; } - char* tmp = (char*)realloc(buffer->ptr, csize); - // FIXME check tmp == NULL + char* tmp = REALLOC_N(buffer->ptr, char, csize); buffer->ptr = tmp; } -- cgit v1.2.1 From 3fbcde4bd74e00e208b10aa00c389256de0ba317 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 18:11:09 +0900 Subject: ruby: don't use rb_enc_set/get on ruby 1.8 --- ruby/msgpack.gemspec | 2 +- ruby/unpack.c | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) (limited to 'ruby') diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 0f08628..e10622f 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.4.0" + s.version = "0.4.1" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" diff --git a/ruby/unpack.c b/ruby/unpack.c index 65852b9..0f7b9f0 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -163,15 +163,17 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #endif -static VALUE template_execute_rescue(VALUE data) +#ifdef HAVE_RUBY_ENCODING_H +static VALUE template_execute_rescue_enc(VALUE data) { rb_gc_enable(); VALUE* resc = (VALUE*)data; rb_enc_set_index(resc[0], (int)resc[1]); RERAISE; } +#endif -static VALUE template_execute_rescue_each(VALUE nouse) +static VALUE template_execute_rescue(VALUE nouse) { rb_gc_enable(); RERAISE; @@ -211,10 +213,14 @@ static int template_execute_wrap(msgpack_unpack_t* mp, mp->user.source = str; +#ifdef HAVE_RUBY_ENCODING_H VALUE resc[2] = {str, enc_orig}; - int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue, (VALUE)resc); + template_execute_rescue_enc, (VALUE)resc); +#else + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, Qnil); +#endif rb_gc_enable(); @@ -241,7 +247,7 @@ static int template_execute_wrap_each(msgpack_unpack_t* mp, mp->user.source = Qnil; int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue_each, Qnil); + template_execute_rescue, Qnil); rb_gc_enable(); -- cgit v1.2.1 From d4049fe593ae4465e7a258d138c2166571a0f1a7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 1 Jun 2010 16:35:21 +0900 Subject: ruby: add test/test_cases.rb --- ruby/makegem.sh | 4 +- ruby/msgpack_test.rb | 267 ----------------------------------------- ruby/test/test_cases.rb | 46 ++++++++ ruby/test/test_helper.rb | 4 + ruby/test/test_pack_unpack.rb | 268 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 321 insertions(+), 268 deletions(-) delete mode 100644 ruby/msgpack_test.rb create mode 100644 ruby/test/test_cases.rb create mode 100644 ruby/test/test_pack_unpack.rb (limited to 'ruby') diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 5ea66f1..827f452 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -13,7 +13,9 @@ cp ../msgpack/pack_template.h msgpack/ cp ../msgpack/unpack_define.h msgpack/ cp ../msgpack/unpack_template.h msgpack/ cp ../msgpack/sysdep.h msgpack/ -cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > test/msgpack_test.rb +cp ../test/cases.mpac test/ +cp ../test/cases_compact.mpac test/ +cp ../test/cases.json test/ gem build msgpack.gemspec diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb deleted file mode 100644 index 8cbb586..0000000 --- a/ruby/msgpack_test.rb +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env ruby -require 'msgpack' -require 'test/unit' - -class MessagePackTestFormat < Test::Unit::TestCase - def self.it(name, &block) - define_method("test_#{name}", &block) - end - - it "nil" do - check 1, nil - end - - it "true" do - check 1, true - end - - it "false" do - check 1, false - end - - it "zero" do - check 1, 0 - end - - it "positive fixnum" do - check 1, 1 - check 1, (1<<6) - check 1, (1<<7)-1 - end - - it "positive int 8" do - check 1, -1 - check 2, (1<<7) - check 2, (1<<8)-1 - end - - it "positive int 16" do - check 3, (1<<8) - check 3, (1<<16)-1 - end - - it "positive int 32" do - check 5, (1<<16) - check 5, (1<<32)-1 - end - - it "positive int 64" do - check 9, (1<<32) - check 9, (1<<64)-1 - end - - it "negative fixnum" do - check 1, -1 - check 1, -((1<<5)-1) - check 1, -(1<<5) - end - - it "negative int 8" do - check 2, -((1<<5)+1) - check 2, -(1<<7) - end - - it "negative int 16" do - check 3, -((1<<7)+1) - check 3, -(1<<15) - end - - it "negative int 32" do - check 5, -((1<<15)+1) - check 5, -(1<<31) - end - - it "negative int 64" do - check 9, -((1<<31)+1) - check 9, -(1<<63) - end - - it "double" do - check 9, 1.0 - check 9, 0.1 - check 9, -0.1 - check 9, -1.0 - end - - it "fixraw" do - check_raw 1, 0 - check_raw 1, (1<<5)-1 - end - - it "raw 16" do - check_raw 3, (1<<5) - check_raw 3, (1<<16)-1 - end - - it "raw 32" do - check_raw 5, (1<<16) - #check_raw 5, (1<<32)-1 # memory error - end - - it "fixarray" do - check_array 1, 0 - check_array 1, (1<<4)-1 - end - - it "array 16" do - check_array 3, (1<<4) - check_array 3, (1<<16)-1 - end - - it "array 32" do - check_array 5, (1<<16) - #check_array 5, (1<<32)-1 # memory error - end - - it "nil" do - match nil, "\xc0" - end - - it "false" do - match false, "\xc2" - end - - it "true" do - match true, "\xc3" - end - - it "0" do - match 0, "\x00" - end - - it "127" do - match 127, "\x7f" - end - - it "128" do - match 128, "\xcc\x80" - end - - it "256" do - match 256, "\xcd\x01\x00" - end - - it "-1" do - match -1, "\xff" - end - - it "-33" do - match -33, "\xd0\xdf" - end - - it "-129" do - match -129, "\xd1\xff\x7f" - end - - it "{1=>1}" do - match ({1=>1}), "\x81\x01\x01" - end - - it "1.0" do - match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00" - end - - it "[]" do - match [], "\x90" - end - - it "[0, 1, ..., 14]" do - match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" - end - - it "[0, 1, ..., 15]" do - match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - end - - it "{}" do - match ({}), "\x80" - end - - it "{0=>0, 1=>1, ..., 14=>14}" do - a = (0..14).to_a; - match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a" - end - - it "{0=>0, 1=>1, ..., 15=>15}" do - a = (0..15).to_a; - match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a" - end - -# it "fixmap" do -# check_map 1, 0 -# check_map 1, (1<<4)-1 -# end -# -# it "map 16" do -# check_map 3, (1<<4) -# check_map 3, (1<<16)-1 -# end -# -# it "map 32" do -# check_map 5, (1<<16) -# #check_map 5, (1<<32)-1 # memory error -# end - - it "gc mark" do - obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] - num = 4 - raw = obj.to_msgpack * num - pac = MessagePack::Unpacker.new - parsed = 0 - raw.split(//).each do |b| - pac.feed(b) - pac.each {|o| - GC.start - assert_equal(obj, o) - parsed += 1 - } - GC.start - end - assert_equal(parsed, num) - end - - it "streaming backward compatibility" do - obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] - num = 4 - raw = obj.to_msgpack * num - pac = MessagePack::Unpacker.new - buffer = "" - nread = 0 - parsed = 0 - raw.split(//).each do |b| - buffer << b - nread = pac.execute(buffer, nread) - if pac.finished? - o = pac.data - assert_equal(obj, o) - parsed += 1 - pac.reset - buffer.slice!(0, nread) - nread = 0 - next unless buffer.empty? - end - end - assert_equal(parsed, num) - end - - private - def check(len, obj) - v = obj.to_msgpack - assert_equal(v.length, len) - assert_equal(MessagePack.unpack(v), obj) - end - - def check_raw(overhead, num) - check num+overhead, " "*num - end - - def check_array(overhead, num) - check num+overhead, Array.new(num) - end - - def match(obj, buf) - assert_equal(obj.to_msgpack, buf) - assert_equal(MessagePack::unpack(buf), obj) - end -end - diff --git a/ruby/test/test_cases.rb b/ruby/test/test_cases.rb new file mode 100644 index 0000000..bfb752e --- /dev/null +++ b/ruby/test/test_cases.rb @@ -0,0 +1,46 @@ +#!/usr/bin/env ruby +here = File.dirname(__FILE__) +require "#{here}/test_helper" + +begin +require 'json' +rescue LoadError +require 'rubygems' +require 'json' +end + +CASES_PATH = "#{here}/cases.mpac" +CASES_COMPACT_PATH = "#{here}/cases_compact.mpac" +CASES_JSON_PATH = "#{here}/cases.json" + +class MessagePackTestCases < Test::Unit::TestCase + def feed_file(path) + pac = MessagePack::Unpacker.new + pac.feed File.read(path) + pac + end + + def test_compare_compact + pac = feed_file(CASES_PATH) + cpac = feed_file(CASES_COMPACT_PATH) + + objs = []; pac.each {| obj| objs << obj } + cobjs = []; cpac.each {|cobj| cobjs << cobj } + + objs.zip(cobjs).each {|obj, cobj| + assert_equal(obj, cobj) + } + end + + def test_compare_json + pac = feed_file(CASES_PATH) + + objs = []; pac.each {|obj| objs << obj } + jobjs = JSON.load File.read(CASES_JSON_PATH) + + objs.zip(jobjs) {|obj, jobj| + assert_equal(obj, jobj) + } + end +end + diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 6a63489..19226ef 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -1,3 +1,7 @@ require 'test/unit' +begin +require File.dirname(__FILE__) + '/../msgpack' +rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' +end diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb new file mode 100644 index 0000000..e22bab3 --- /dev/null +++ b/ruby/test/test_pack_unpack.rb @@ -0,0 +1,268 @@ +#!/usr/bin/env ruby +require File.dirname(__FILE__)+'/test_helper' + +class MessagePackTestPackUnpack < Test::Unit::TestCase + def self.it(name, &block) + define_method("test_#{name}", &block) + end + + it "nil" do + check 1, nil + end + + it "true" do + check 1, true + end + + it "false" do + check 1, false + end + + it "zero" do + check 1, 0 + end + + it "positive fixnum" do + check 1, 1 + check 1, (1<<6) + check 1, (1<<7)-1 + end + + it "positive int 8" do + check 1, -1 + check 2, (1<<7) + check 2, (1<<8)-1 + end + + it "positive int 16" do + check 3, (1<<8) + check 3, (1<<16)-1 + end + + it "positive int 32" do + check 5, (1<<16) + check 5, (1<<32)-1 + end + + it "positive int 64" do + check 9, (1<<32) + check 9, (1<<64)-1 + end + + it "negative fixnum" do + check 1, -1 + check 1, -((1<<5)-1) + check 1, -(1<<5) + end + + it "negative int 8" do + check 2, -((1<<5)+1) + check 2, -(1<<7) + end + + it "negative int 16" do + check 3, -((1<<7)+1) + check 3, -(1<<15) + end + + it "negative int 32" do + check 5, -((1<<15)+1) + check 5, -(1<<31) + end + + it "negative int 64" do + check 9, -((1<<31)+1) + check 9, -(1<<63) + end + + it "double" do + check 9, 1.0 + check 9, 0.1 + check 9, -0.1 + check 9, -1.0 + end + + it "fixraw" do + check_raw 1, 0 + check_raw 1, (1<<5)-1 + end + + it "raw 16" do + check_raw 3, (1<<5) + check_raw 3, (1<<16)-1 + end + + it "raw 32" do + check_raw 5, (1<<16) + #check_raw 5, (1<<32)-1 # memory error + end + + it "fixarray" do + check_array 1, 0 + check_array 1, (1<<4)-1 + end + + it "array 16" do + check_array 3, (1<<4) + check_array 3, (1<<16)-1 + end + + it "array 32" do + check_array 5, (1<<16) + #check_array 5, (1<<32)-1 # memory error + end + + it "nil" do + match nil, "\xc0" + end + + it "false" do + match false, "\xc2" + end + + it "true" do + match true, "\xc3" + end + + it "0" do + match 0, "\x00" + end + + it "127" do + match 127, "\x7f" + end + + it "128" do + match 128, "\xcc\x80" + end + + it "256" do + match 256, "\xcd\x01\x00" + end + + it "-1" do + match -1, "\xff" + end + + it "-33" do + match -33, "\xd0\xdf" + end + + it "-129" do + match -129, "\xd1\xff\x7f" + end + + it "{1=>1}" do + match ({1=>1}), "\x81\x01\x01" + end + + it "1.0" do + match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00" + end + + it "[]" do + match [], "\x90" + end + + it "[0, 1, ..., 14]" do + match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + end + + it "[0, 1, ..., 15]" do + match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + end + + it "{}" do + match ({}), "\x80" + end + +## FIXME +# it "{0=>0, 1=>1, ..., 14=>14}" do +# a = (0..14).to_a; +# match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a" +# end +# +# it "{0=>0, 1=>1, ..., 15=>15}" do +# a = (0..15).to_a; +# match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a" +# end + +## FIXME +# it "fixmap" do +# check_map 1, 0 +# check_map 1, (1<<4)-1 +# end +# +# it "map 16" do +# check_map 3, (1<<4) +# check_map 3, (1<<16)-1 +# end +# +# it "map 32" do +# check_map 5, (1<<16) +# #check_map 5, (1<<32)-1 # memory error +# end + + it "gc mark" do + obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + num = 4 + raw = obj.to_msgpack * num + pac = MessagePack::Unpacker.new + parsed = 0 + raw.split(//).each do |b| + pac.feed(b) + pac.each {|o| + GC.start + assert_equal(obj, o) + parsed += 1 + } + GC.start + end + assert_equal(parsed, num) + end + + it "streaming backward compatibility" do + obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + num = 4 + raw = obj.to_msgpack * num + pac = MessagePack::Unpacker.new + buffer = "" + nread = 0 + parsed = 0 + raw.split(//).each do |b| + buffer << b + nread = pac.execute(buffer, nread) + if pac.finished? + o = pac.data + assert_equal(obj, o) + parsed += 1 + pac.reset + buffer.slice!(0, nread) + nread = 0 + next unless buffer.empty? + end + end + assert_equal(parsed, num) + end + + private + def check(len, obj) + v = obj.to_msgpack + assert_equal(v.length, len) + assert_equal(MessagePack.unpack(v), obj) + end + + def check_raw(overhead, num) + check num+overhead, " "*num + end + + def check_array(overhead, num) + check num+overhead, Array.new(num) + end + + def match(obj, buf) + assert_equal(obj.to_msgpack, buf) + assert_equal(MessagePack::unpack(buf), obj) + end +end + -- cgit v1.2.1 From 9c3ed173b1c497cfe9bdfff2e5e695e5a3377e0a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 3 Jun 2010 21:51:40 +0900 Subject: ruby: fixes buffering routine --- ruby/unpack.c | 98 ++++++++++++++++++++++++----------------------------------- 1 file changed, 39 insertions(+), 59 deletions(-) (limited to 'ruby') diff --git a/ruby/unpack.c b/ruby/unpack.c index 0f7b9f0..65ae476 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -386,90 +386,70 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) } -#ifdef RUBY_VM -# ifndef STR_SHARED -# define STR_SHARED FL_USER2 -# endif -# ifndef STR_NOEMBED -# define STR_NOEMBED FL_USER1 -# endif -# ifndef STR_ASSOC -# define STR_ASSOC FL_USER3 -# endif -# ifndef STR_NOCAPA_P -# define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,STR_SHARED|STR_ASSOC)) -# endif -# define NEED_MORE_CAPA(s,size) (!STR_NOCAPA_P(s) && RSTRING(s)->as.heap.aux.capa < size) -#else -# ifndef STR_NOCAPA -# ifndef STR_ASSOC -# define STR_ASSOC FL_USER3 -# endif -# ifndef ELTS_SHARED -# define ELTS_SHARED FL_USER2 -# endif -# define STR_NOCAPA (ELTS_SHARED|STR_ASSOC) -# endif -# define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size) -#endif - -static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) +static void reserve_buffer(msgpack_unpack_t* mp, size_t require) { struct unpack_buffer* buffer = &mp->user.buffer; if(buffer->size == 0) { - char* tmp = ALLOC_N(char, MSGPACK_UNPACKER_BUFFER_INIT_SIZE); + size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE; + while(nsize < require) { + nsize *= 2; + } + char* tmp = ALLOC_N(char, nsize); buffer->ptr = tmp; - buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE; + buffer->free = nsize; buffer->size = 0; + return; + } - } else if(buffer->size <= mp->user.offset) { + if(buffer->size <= mp->user.offset) { /* clear buffer and rewind offset */ buffer->free += buffer->size; buffer->size = 0; mp->user.offset = 0; } - if(len <= buffer->free) { - /* enough free space: just copy */ - memcpy(buffer->ptr+buffer->size, ptr, len); - buffer->size += len; - buffer->free -= len; + if(require <= buffer->free) { + /* enough free space */ return; } - size_t csize = buffer->size + buffer->free; + size_t nsize = (buffer->size + buffer->free) * 2; if(mp->user.offset <= buffer->size / 2) { - /* parsed less than half: realloc and copy */ - csize *= 2; - while(csize < buffer->size + len) { - csize *= 2; + /* parsed less than half: realloc only */ + while(nsize < buffer->size + require) { + nsize *= 2; } - char* tmp = REALLOC_N(buffer->ptr, char, csize); - memcpy(tmp + buffer->size, ptr, len); + char* tmp = REALLOC_N(buffer->ptr, char, nsize); + buffer->free = nsize - buffer->size; buffer->ptr = tmp; - buffer->free = csize - buffer->size; - return; - } - size_t not_parsed = buffer->size - mp->user.offset; - - if(csize < not_parsed + len) { - /* more buffer size */ - csize *= 2; - while(csize < not_parsed + len) { - csize *= 2; + } else { + /* parsed more than half: realloc and move */ + size_t not_parsed = buffer->size - mp->user.offset; + while(nsize < not_parsed + require) { + nsize *= 2; } - char* tmp = REALLOC_N(buffer->ptr, char, csize); + char* tmp = REALLOC_N(buffer->ptr, char, nsize); + memcpy(tmp, tmp + mp->user.offset, not_parsed); + buffer->free = nsize - buffer->size; + buffer->size = not_parsed; buffer->ptr = tmp; + mp->user.offset = 0; } +} - memcpy(buffer->ptr+not_parsed, ptr, not_parsed); - buffer->size = not_parsed; - buffer->free = csize - buffer->size; - buffer->ptr = buffer->ptr; - mp->user.offset = 0; +static inline void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) +{ + struct unpack_buffer* buffer = &mp->user.buffer; + + if(buffer->free < len) { + reserve_buffer(mp, len); + } + memcpy(buffer->ptr + buffer->size, ptr, len); + buffer->size += len; + buffer->free -= len; } /** -- cgit v1.2.1 From 251090406a642d968e314db3aa1d5240db00598a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 3 Jun 2010 21:52:01 +0900 Subject: ruby: adds a test case for buffering --- ruby/test/test_pack_unpack.rb | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'ruby') diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb index e22bab3..9dff44f 100644 --- a/ruby/test/test_pack_unpack.rb +++ b/ruby/test/test_pack_unpack.rb @@ -203,6 +203,37 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase # #check_map 5, (1<<32)-1 # memory error # end + it "buffer" do + str = "a"*32*1024*4 + raw = str.to_msgpack + pac = MessagePack::Unpacker.new + + len = 0 + parsed = false + + n = 655 + time = raw.size / n + time += 1 unless raw.size % n == 0 + off = 0 + + time.times do + assert(!parsed) + + fe = raw[off, n] + assert(fe.length > 0) + off += fe.length + + pac.feed fe + pac.each {|obj| + assert(!parsed) + assert_equal(obj, str) + parsed = true + } + end + + assert(parsed) + end + it "gc mark" do obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] num = 4 -- cgit v1.2.1 From b3e0ad13030cdaee442582a8dd16c46518e9d6c9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 3 Jun 2010 22:00:15 +0900 Subject: ruby: 0.4.2 --- ruby/msgpack.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ruby') diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index e10622f..fb6338a 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.4.1" + s.version = "0.4.2" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" -- cgit v1.2.1 From 9fffa9800ae4b09180d2b892f1f70215534a874b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 14:54:09 +0900 Subject: ruby: fixes RDoc of Unpacker#execute and Unpacker#execute_impl --- ruby/makegem.sh | 2 ++ ruby/unpack.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'ruby') diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 827f452..d21f06a 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,6 +19,8 @@ cp ../test/cases.json test/ gem build msgpack.gemspec +rdoc rbinit.c pack.c unpack.c + if [ $? -eq 0 ]; then rm -rf ext msgpack test/msgpack_test.rb fi diff --git a/ruby/unpack.c b/ruby/unpack.c index 65ae476..151dbf4 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -638,7 +638,7 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, * Document-method: MessagePack::Unpacker#execute_limit * * call-seq: - * unpacker.unpack_limit(data, offset, limit) -> next offset + * unpacker.execute_limit(data, offset, limit) -> next offset * * Deserializes one object over the specified buffer from _offset_ bytes upto _limit_ bytes. * @@ -660,7 +660,7 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, * Document-method: MessagePack::Unpacker#execute * * call-seq: - * unpacker.unpack(data, offset) -> next offset + * unpacker.execute(data, offset) -> next offset * * Deserializes one object over the specified buffer from _offset_ bytes. * -- cgit v1.2.1 From 34a29cd0a50eea4a0e008fe3947c86179d536540 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 14:54:40 +0900 Subject: ruby: fixes SEGV problem caused by GC bug at MessagePack_Unpacker_mark. --- ruby/test/test_helper.rb | 1 + ruby/unpack.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'ruby') diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 19226ef..bf9fee8 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -5,3 +5,4 @@ rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' end +GC.stress = true diff --git a/ruby/unpack.c b/ruby/unpack.c index 151dbf4..0948151 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -287,6 +287,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) unsigned int i; rb_gc_mark(mp->user.stream); rb_gc_mark(mp->user.streambuf); + rb_gc_mark_maybe(template_data(mp)); for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); rb_gc_mark_maybe(mp->stack[i].map_key); @@ -297,6 +298,17 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) { VALUE obj; msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1); + + // rb_gc_mark (not _maybe) is used for following member objects. + mp->user.stream = Qnil; + mp->user.streambuf = Qnil; + + mp->user.finished = 0; + mp->user.offset = 0; + mp->user.buffer.size = 0; + mp->user.buffer.free = 0; + mp->user.buffer.ptr = NULL; + obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, MessagePack_Unpacker_free, mp); return obj; @@ -343,14 +355,10 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) UNPACKER(self, mp); template_init(mp); - mp->user.finished = 0; - mp->user.offset = 0; - mp->user.buffer.size = 0; - mp->user.buffer.free = 0; - mp->user.buffer.ptr = NULL; mp->user.stream = stream; mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE); mp->user.stream_append_method = append_method_of(stream); + return self; } -- cgit v1.2.1 From 123ae024c6d5c217f18a9444c61b292145227278 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 15:12:52 +0900 Subject: ruby: MessagePack::VERSION constant --- ruby/extconf.rb | 3 ++- ruby/msgpack.gemspec | 3 ++- ruby/rbinit.c | 4 +++- ruby/test/test_helper.rb | 2 +- ruby/test/test_pack_unpack.rb | 4 ++++ ruby/version.rb | 3 +++ 6 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 ruby/version.rb (limited to 'ruby') diff --git a/ruby/extconf.rb b/ruby/extconf.rb index e6d4bd6..eb6a389 100644 --- a/ruby/extconf.rb +++ b/ruby/extconf.rb @@ -1,4 +1,5 @@ require 'mkmf' -$CFLAGS << " -I.. -Wall -O4" +require './version.rb' +$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"] create_makefile('msgpack') diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index fb6338a..95a2bd0 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,8 @@ +require './version.rb' Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.4.2" + s.version = MessagePack::VERSION s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" diff --git a/ruby/rbinit.c b/ruby/rbinit.c index ad51f6b..28a8bfe 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -43,7 +43,9 @@ static VALUE mMessagePack; void Init_msgpack(void) { mMessagePack = rb_define_module("MessagePack"); + + rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION)); + Init_msgpack_unpack(mMessagePack); Init_msgpack_pack(mMessagePack); } - diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index bf9fee8..80d7806 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -5,4 +5,4 @@ rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' end -GC.stress = true +#GC.stress = true diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb index 9dff44f..25bde81 100644 --- a/ruby/test/test_pack_unpack.rb +++ b/ruby/test/test_pack_unpack.rb @@ -276,6 +276,10 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase assert_equal(parsed, num) end + it "MessagePack::VERSION constant" do + p MessagePack::VERSION + end + private def check(len, obj) v = obj.to_msgpack diff --git a/ruby/version.rb b/ruby/version.rb new file mode 100644 index 0000000..b156620 --- /dev/null +++ b/ruby/version.rb @@ -0,0 +1,3 @@ +module MessagePack + VERSION = "0.4.3" +end -- cgit v1.2.1 From 20de730541475516aa7a6361af1d1b5e4ea574b8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 29 Jun 2010 15:39:47 +0900 Subject: ruby: 0.4.3 --- ruby/ChangeLog | 6 ++++++ ruby/makegem.sh | 1 + 2 files changed, 7 insertions(+) (limited to 'ruby') diff --git a/ruby/ChangeLog b/ruby/ChangeLog index e69de29..d3a7282 100644 --- a/ruby/ChangeLog +++ b/ruby/ChangeLog @@ -0,0 +1,6 @@ + +2010-06-29 version 0.4.3: + + * Adds MessagePack::VERSION constant + * Fixes SEGV problem caused by GC bug at MessagePack_Unpacker_mark + diff --git a/ruby/makegem.sh b/ruby/makegem.sh index d21f06a..bf30cd4 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -8,6 +8,7 @@ cp pack.h ext/ cp rbinit.c ext/ cp unpack.c ext/ cp unpack.h ext/ +cp version.rb ext/ cp ../msgpack/pack_define.h msgpack/ cp ../msgpack/pack_template.h msgpack/ cp ../msgpack/unpack_define.h msgpack/ -- cgit v1.2.1 From b5c78de2ddf82783a6f80a199b68927d1a1747ca Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 06:30:16 +0900 Subject: ruby: converts encodings into UTF-8 on Ruby 1.9 --- ruby/encoding.h | 33 ++++++++++++++++++++++ ruby/pack.c | 21 ++++++++++++-- ruby/rbinit.c | 15 ++++++++++ ruby/test/test_encoding.rb | 68 ++++++++++++++++++++++++++++++++++++++++++++++ ruby/test/test_helper.rb | 4 ++- ruby/unpack.c | 38 +++----------------------- 6 files changed, 141 insertions(+), 38 deletions(-) create mode 100644 ruby/encoding.h create mode 100644 ruby/test/test_encoding.rb (limited to 'ruby') diff --git a/ruby/encoding.h b/ruby/encoding.h new file mode 100644 index 0000000..2ad3fd7 --- /dev/null +++ b/ruby/encoding.h @@ -0,0 +1,33 @@ +/* + * MessagePack for Ruby + * + * Copyright (C) 2008-2010 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ENCODING_H__ +#define ENCODING_H__ + + +#ifdef HAVE_RUBY_ENCODING_H +#include "ruby/encoding.h" +#define MSGPACK_RUBY_ENCODING +extern int s_enc_utf8; +extern int s_enc_ascii8bit; +extern int s_enc_usascii; +extern VALUE s_enc_utf8_value; +#endif + + +#endif /* encoding.h */ + diff --git a/ruby/pack.c b/ruby/pack.c index bbeac4a..35878c7 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -16,6 +16,8 @@ * limitations under the License. */ #include "ruby.h" +#include "encoding.h" + #include "msgpack/pack_define.h" static ID s_to_msgpack; @@ -131,7 +133,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - // FIXME bignum if(RBIGNUM_SIGN(self)) { // positive msgpack_pack_uint64(out, rb_big2ull(self)); } else { // negative @@ -168,6 +169,14 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); +#ifdef MSGPACK_RUBY_ENCODING + int enc = ENCODING_GET(self); + if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) { + if(!ENC_CODERANGE_ASCIIONLY(self)) { + self = rb_str_encode(self, s_enc_utf8_value, 0, Qnil); + } + } +#endif msgpack_pack_raw(out, RSTRING_LEN(self)); msgpack_pack_raw_body(out, RSTRING_PTR(self), RSTRING_LEN(self)); return out; @@ -184,12 +193,16 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) */ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) { +#ifdef MSGPACK_RUBY_ENCODING + return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self))); +#else ARG_BUFFER(out, argc, argv); const char* name = rb_id2name(SYM2ID(self)); size_t len = strlen(name); msgpack_pack_raw(out, len); msgpack_pack_raw_body(out, name, len); return out; +#endif } @@ -205,7 +218,8 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_array(out, RARRAY_LEN(self)); + // FIXME check sizeof(long) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX + msgpack_pack_array(out, (unsigned int)RARRAY_LEN(self)); VALUE* p = RARRAY_PTR(self); VALUE* const pend = p + RARRAY_LEN(self); for(;p != pend; ++p) { @@ -239,7 +253,8 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_map(out, RHASH_SIZE(self)); + // FIXME check sizeof(st_index_t) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX + msgpack_pack_map(out, (unsigned int)RHASH_SIZE(self)); rb_hash_foreach(self, MessagePack_Hash_to_msgpack_foreach, out); return out; } diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 28a8bfe..4678159 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -17,9 +17,17 @@ */ #include "pack.h" #include "unpack.h" +#include "encoding.h" static VALUE mMessagePack; +#ifdef MSGPACK_RUBY_ENCODING +int s_enc_utf8; +int s_enc_ascii8bit; +int s_enc_usascii; +VALUE s_enc_utf8_value; +#endif + /** * Document-module: MessagePack * @@ -46,6 +54,13 @@ void Init_msgpack(void) rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION)); +#ifdef MSGPACK_RUBY_ENCODING + s_enc_ascii8bit = rb_ascii8bit_encindex(); + s_enc_utf8 = rb_utf8_encindex(); + s_enc_usascii = rb_usascii_encindex(); + s_enc_utf8_value = rb_enc_from_encoding(rb_utf8_encoding()); +#endif + Init_msgpack_unpack(mMessagePack); Init_msgpack_pack(mMessagePack); } diff --git a/ruby/test/test_encoding.rb b/ruby/test/test_encoding.rb new file mode 100644 index 0000000..2cf0767 --- /dev/null +++ b/ruby/test/test_encoding.rb @@ -0,0 +1,68 @@ +#!/usr/bin/env ruby +require File.dirname(__FILE__)+'/test_helper' + +if RUBY_VERSION < "1.9" + exit +end + +class MessagePackTestEncoding < Test::Unit::TestCase + def self.it(name, &block) + define_method("test_#{name}", &block) + end + + it "US-ASCII" do + check_unpack "abc".force_encoding("US-ASCII") + end + + it "UTF-8 ascii" do + check_unpack "abc".force_encoding("UTF-8") + end + + it "UTF-8 mbstr" do + check_unpack "\xE3\x81\x82".force_encoding("UTF-8") + end + + it "UTF-8 invalid" do + check_unpack "\xD0".force_encoding("UTF-8") + end + + it "ASCII-8BIT" do + check_unpack "\xD0".force_encoding("ASCII-8BIT") + end + + it "EUC-JP" do + x = "\xA4\xA2".force_encoding("EUC-JP") + check_unpack(x) + end + + it "EUC-JP invalid" do + begin + "\xD0".force_encoding("EUC-JP").to_msgpack + assert(false) + rescue Encoding::InvalidByteSequenceError + assert(true) + end + end + + private + def check_unpack(str) + if str.encoding.to_s == "ASCII-8BIT" + should_str = str.dup.force_encoding("UTF-8") + else + should_str = str.encode("UTF-8") + end + + raw = str.to_msgpack + r = MessagePack.unpack(str.to_msgpack) + assert_equal(r.encoding.to_s, "UTF-8") + assert_equal(r, should_str.force_encoding("UTF-8")) + + if str.valid_encoding? + sym = str.to_sym + r = MessagePack.unpack(sym.to_msgpack) + assert_equal(r.encoding.to_s, "UTF-8") + assert_equal(r, should_str.force_encoding("UTF-8")) + end + end +end + diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 80d7806..4def861 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -5,4 +5,6 @@ rescue LoadError require File.dirname(__FILE__) + '/../lib/msgpack' end -#GC.stress = true +if ENV["GC_STRESS"] + GC.stress = true +end diff --git a/ruby/unpack.c b/ruby/unpack.c index 0948151..3c5e350 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,17 +16,13 @@ * limitations under the License. */ #include "ruby.h" +#include "encoding.h" #include "msgpack/unpack_define.h" static ID s_sysread; static ID s_readpartial; -#ifdef HAVE_RUBY_ENCODING_H -#include "ruby/encoding.h" -int s_ascii_8bit; -#endif - struct unpack_buffer { size_t size; size_t free; @@ -136,6 +132,9 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } else { *o = rb_str_substr(u->source, p - b, l); } +#ifdef MSGPACK_RUBY_ENCODING + ENCODING_SET(*o, s_enc_utf8); +#endif return 0; } @@ -163,16 +162,6 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #endif -#ifdef HAVE_RUBY_ENCODING_H -static VALUE template_execute_rescue_enc(VALUE data) -{ - rb_gc_enable(); - VALUE* resc = (VALUE*)data; - rb_enc_set_index(resc[0], (int)resc[1]); - RERAISE; -} -#endif - static VALUE template_execute_rescue(VALUE nouse) { rb_gc_enable(); @@ -203,31 +192,16 @@ static int template_execute_wrap(msgpack_unpack_t* mp, (VALUE)from, }; -#ifdef HAVE_RUBY_ENCODING_H - int enc_orig = rb_enc_get_index(str); - rb_enc_set_index(str, s_ascii_8bit); -#endif - // FIXME execute実行中はmp->topが更新されないのでGC markが機能しない rb_gc_disable(); mp->user.source = str; -#ifdef HAVE_RUBY_ENCODING_H - VALUE resc[2] = {str, enc_orig}; - int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue_enc, (VALUE)resc); -#else int ret = (int)rb_rescue(template_execute_do, (VALUE)args, template_execute_rescue, Qnil); -#endif rb_gc_enable(); -#ifdef HAVE_RUBY_ENCODING_H - rb_enc_set_index(str, enc_orig); -#endif - return ret; } @@ -746,10 +720,6 @@ void Init_msgpack_unpack(VALUE mMessagePack) s_sysread = rb_intern("sysread"); s_readpartial = rb_intern("readpartial"); -#ifdef HAVE_RUBY_ENCODING_H - s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); -#endif - eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); -- cgit v1.2.1 From 09b47cc536ebd951c231fd5e09b4382a25b98020 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 07:00:19 +0900 Subject: ruby: fixes compatibility with ruby-1.8.5 --- ruby/compat.h | 60 +++++++++++++++++++++++++++++++++++++++++++ ruby/encoding.h | 33 ------------------------ ruby/pack.c | 6 ++--- ruby/rbinit.c | 6 ++--- ruby/test/test_pack_unpack.rb | 12 ++++++--- ruby/unpack.c | 12 +++------ 6 files changed, 77 insertions(+), 52 deletions(-) create mode 100644 ruby/compat.h delete mode 100644 ruby/encoding.h (limited to 'ruby') diff --git a/ruby/compat.h b/ruby/compat.h new file mode 100644 index 0000000..98c8881 --- /dev/null +++ b/ruby/compat.h @@ -0,0 +1,60 @@ +/* + * MessagePack for Ruby + * + * Copyright (C) 2008-2010 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef COMPAT_H__ +#define COMPAT_H__ + + +#ifdef HAVE_RUBY_ENCODING_H +#include "ruby/encoding.h" +#define COMPAT_HAVE_ENCODING +extern int s_enc_utf8; +extern int s_enc_ascii8bit; +extern int s_enc_usascii; +extern VALUE s_enc_utf8_value; +#endif + +#ifdef RUBY_VM +#define COMPAT_RERAISE rb_exc_raise(rb_errinfo()) +#else +#define COMPAT_RERAISE rb_exc_raise(ruby_errinfo) +#endif + + +/* ruby 1.8.5 */ +#ifndef RSTRING_PTR +#define RSTRING_PTR(s) (RSTRING(s)->ptr) +#endif + +/* ruby 1.8.5 */ +#ifndef RSTRING_LEN +#define RSTRING_LEN(s) (RSTRING(s)->len) +#endif + +/* ruby 1.8.5 */ +#ifndef RARRAY_PTR +#define RARRAY_PTR(s) (RARRAY(s)->ptr) +#endif + +/* ruby 1.8.5 */ +#ifndef RARRAY_LEN +#define RARRAY_LEN(s) (RARRAY(s)->len) +#endif + + +#endif /* compat.h */ + diff --git a/ruby/encoding.h b/ruby/encoding.h deleted file mode 100644 index 2ad3fd7..0000000 --- a/ruby/encoding.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * MessagePack for Ruby - * - * Copyright (C) 2008-2010 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef ENCODING_H__ -#define ENCODING_H__ - - -#ifdef HAVE_RUBY_ENCODING_H -#include "ruby/encoding.h" -#define MSGPACK_RUBY_ENCODING -extern int s_enc_utf8; -extern int s_enc_ascii8bit; -extern int s_enc_usascii; -extern VALUE s_enc_utf8_value; -#endif - - -#endif /* encoding.h */ - diff --git a/ruby/pack.c b/ruby/pack.c index 35878c7..49b69fc 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -16,7 +16,7 @@ * limitations under the License. */ #include "ruby.h" -#include "encoding.h" +#include "compat.h" #include "msgpack/pack_define.h" @@ -169,7 +169,7 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING int enc = ENCODING_GET(self); if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) { if(!ENC_CODERANGE_ASCIIONLY(self)) { @@ -193,7 +193,7 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) */ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) { -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self))); #else ARG_BUFFER(out, argc, argv); diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 4678159..1d1cbc6 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -17,11 +17,11 @@ */ #include "pack.h" #include "unpack.h" -#include "encoding.h" +#include "compat.h" static VALUE mMessagePack; -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING int s_enc_utf8; int s_enc_ascii8bit; int s_enc_usascii; @@ -54,7 +54,7 @@ void Init_msgpack(void) rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION)); -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING s_enc_ascii8bit = rb_ascii8bit_encindex(); s_enc_utf8 = rb_utf8_encindex(); s_enc_usascii = rb_usascii_encindex(); diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb index 25bde81..545e593 100644 --- a/ruby/test/test_pack_unpack.rb +++ b/ruby/test/test_pack_unpack.rb @@ -153,7 +153,8 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase end it "{1=>1}" do - match ({1=>1}), "\x81\x01\x01" + obj = {1=>1} + match obj, "\x81\x01\x01" end it "1.0" do @@ -165,15 +166,18 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase end it "[0, 1, ..., 14]" do - match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + obj = (0..14).to_a + match obj, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" end it "[0, 1, ..., 15]" do - match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + obj = (0..15).to_a + match obj, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" end it "{}" do - match ({}), "\x80" + obj = {} + match obj, "\x80" end ## FIXME diff --git a/ruby/unpack.c b/ruby/unpack.c index 3c5e350..2d10e75 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,7 +16,7 @@ * limitations under the License. */ #include "ruby.h" -#include "encoding.h" +#include "compat.h" #include "msgpack/unpack_define.h" @@ -132,7 +132,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } else { *o = rb_str_substr(u->source, p - b, l); } -#ifdef MSGPACK_RUBY_ENCODING +#ifdef COMPAT_HAVE_ENCODING ENCODING_SET(*o, s_enc_utf8); #endif return 0; @@ -155,17 +155,11 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha rb_raise(rb_eTypeError, "instance of String needed"); \ } -#ifdef RUBY_VM -#define RERAISE rb_exc_raise(rb_errinfo()) -#else -#define RERAISE rb_exc_raise(ruby_errinfo) -#endif - static VALUE template_execute_rescue(VALUE nouse) { rb_gc_enable(); - RERAISE; + COMPAT_RERAISE; } static VALUE template_execute_do(VALUE argv) -- cgit v1.2.1 From 71a1cb01842787e2fa897f023addd88337542915 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 31 Aug 2010 09:29:01 +0900 Subject: fixes compatibility with Rubinius --- ruby/compat.h | 17 +++++++++++++++++ ruby/extconf.rb | 2 +- ruby/pack.c | 8 ++------ 3 files changed, 20 insertions(+), 7 deletions(-) (limited to 'ruby') diff --git a/ruby/compat.h b/ruby/compat.h index 98c8881..d7a2ca7 100644 --- a/ruby/compat.h +++ b/ruby/compat.h @@ -35,6 +35,23 @@ extern VALUE s_enc_utf8_value; #endif +/* ruby 1.8 and Rubinius */ +#ifndef RBIGNUM_POSITIVE_P +# ifdef RUBINIUS +# define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue) +# else +# define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign) +# endif +#endif + + +/* Rubinius */ +#ifdef RUBINIUS +static inline void rb_gc_enable() { return; } +static inline void rb_gc_disable() { return; } +#endif + + /* ruby 1.8.5 */ #ifndef RSTRING_PTR #define RSTRING_PTR(s) (RSTRING(s)->ptr) diff --git a/ruby/extconf.rb b/ruby/extconf.rb index eb6a389..f1d44ec 100644 --- a/ruby/extconf.rb +++ b/ruby/extconf.rb @@ -1,5 +1,5 @@ require 'mkmf' require './version.rb' -$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"] +$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\" -g] create_makefile('msgpack') diff --git a/ruby/pack.c b/ruby/pack.c index 49b69fc..8ce46aa 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -118,10 +118,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) } -#ifndef RBIGNUM_SIGN // Ruby 1.8 -#define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign) -#endif - /* * Document-method: Bignum#to_msgpack * @@ -133,9 +129,9 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - if(RBIGNUM_SIGN(self)) { // positive + if(RBIGNUM_POSITIVE_P(self)) { msgpack_pack_uint64(out, rb_big2ull(self)); - } else { // negative + } else { msgpack_pack_int64(out, rb_big2ll(self)); } return out; -- cgit v1.2.1