[ruby-dev:36091] Re: x86_64環境でMarshal.dumpの不具合

なかだです。

At Tue, 2 Sep 2008 00:41:51 +0900,
Shugo Maeda wrote in [ruby-dev:36086]:
> dump中にGCが発生すると、一時的に生成したBignumオブジェクトのアドレスが
> たまたま同じになってしまうため、arg->dataの検索時に誤って以前のオブジェクト
> にヒットしてリンクとして処理されてしまうようです。
> 他にもdump中に一時的にオブジェクトを生成している部分がもしあれば、同じよう
> な問題が発生しそうですね。
> 
> 対応としてはarg->dataをHashにするとかでしょうか。

それでは解決しませんでした。

hashやeql?の再定義の影響を避けるためには、arg->dataのキーを
VALUEの値自体(INT2FIX(obj)など)にしなくてはなりませんが、そうす
るとマークの対象ではなくなってしまいます。

具体的には、test/ruby/marshaltest.rb:test_range_cyclicと両立させ
ることは不可能です。

もしかしたら、[ruby-dev:34312]もこれが原因かも知れませんねぇ。

# marshal.cでは r16761 もバックポートしたほうがいいと思います。


* gc.c (rb_mark_set): new function to mark keys.

* marshal.c (struct dump_arg, struct load_arg): added wrappers to mark
  data entries.  backport from trunk r13527,r13528,r13961,r16533.
  [ruby-dev:36082]


Index: intern.h
===================================================================
--- intern.h	(revision 19063)
+++ intern.h	(working copy)
@@ -258,4 +258,5 @@ char *rb_source_filename _((const char*)
 void rb_gc_mark_locations _((VALUE*, VALUE*));
 void rb_mark_tbl _((struct st_table*));
+void rb_mark_set _((struct st_table*));
 void rb_mark_hash _((struct st_table*));
 void rb_gc_mark_maybe _((VALUE));
Index: gc.c
===================================================================
--- gc.c	(revision 19063)
+++ gc.c	(working copy)
@@ -735,4 +735,29 @@ rb_mark_tbl(tbl)
 
 static int
+mark_key(key, value, lev)
+    VALUE key, value;
+    int lev;
+{
+    gc_mark(key, lev);
+    return ST_CONTINUE;
+}
+
+static void
+mark_set(tbl, lev)
+    st_table *tbl;
+    int lev;
+{
+    if (!tbl) return;
+    st_foreach(tbl, mark_key, lev);
+}
+
+void
+rb_mark_set(tbl)
+    st_table *tbl;
+{
+    mark_set(tbl, 0);
+}
+
+static int
 mark_keyvalue(key, value, lev)
     VALUE key;
Index: marshal.c
===================================================================
--- marshal.c	(revision 19063)
+++ marshal.c	(working copy)
@@ -85,14 +85,4 @@ static ID s_dump_data, s_load_data, s_al
 static ID s_getc, s_read, s_write, s_binmode;
 
-static void
-reentrant_check(obj, sym)
-    VALUE obj;
-    ID sym;
-{
-    if (obj && RBASIC(obj)->klass) {
-        rb_raise(rb_eRuntimeError, "%s reentered", rb_id2name(sym));
-    }
-}
-
 struct dump_arg {
     VALUE obj;
@@ -101,4 +91,5 @@ struct dump_arg {
     st_table *data;
     int taint;
+    VALUE wrapper;
 };
 
@@ -109,4 +100,25 @@ struct dump_call_arg {
 };
 
+static void
+check_dump_arg(arg, sym)
+    struct dump_arg *arg;
+    ID sym;
+{
+    if (!DATA_PTR(arg->wrapper)) {
+        rb_raise(rb_eRuntimeError, "Marshal.dump reentered at %s",
+		 rb_id2name(sym));
+    }
+}
+
+static void
+mark_dump_arg(ptr)
+    void *ptr;
+{
+    struct dump_arg *p = ptr;
+    if (!ptr)
+        return;
+    rb_mark_set(p->data);
+}
+
 static VALUE
 class2path(klass)
@@ -516,5 +528,5 @@ w_object(obj, arg, limit)
 
 	    v = rb_funcall(obj, s_mdump, 0, 0);
-	    reentrant_check(arg->str, s_mdump);
+	    check_dump_arg(arg, s_mdump);
 	    w_class(TYPE_USRMARSHAL, obj, arg, Qfalse);
 	    w_object(v, arg, limit);
@@ -526,5 +538,5 @@ w_object(obj, arg, limit)
 
 	    v = rb_funcall(obj, s_dump, 1, INT2NUM(limit));
-	    reentrant_check(arg->str, s_dump);
+	    check_dump_arg(arg, s_dump);
 	    if (TYPE(v) != T_STRING) {
 		rb_raise(rb_eTypeError, "_dump() must return string");
@@ -671,5 +683,5 @@ w_object(obj, arg, limit)
 		}
 		v = rb_funcall(obj, s_dump_data, 0);
-		reentrant_check(arg->str, s_dump_data);
+		check_dump_arg(arg, s_dump_data);
 		w_class(TYPE_DATA, obj, arg, Qtrue);
 		w_object(v, arg, limit);
@@ -704,7 +716,9 @@ dump_ensure(arg)
     struct dump_arg *arg;
 {
-    if (RBASIC(arg->str)->klass) return 0; /* ignore reentrant */
+    if (!DATA_PTR(arg->wrapper)) return 0;
     st_free_table(arg->symbols);
     st_free_table(arg->data);
+    DATA_PTR(arg->wrapper) = 0;
+    arg->wrapper = 0;
     if (arg->taint) {
 	OBJ_TAINT(arg->str);
@@ -773,5 +787,5 @@ marshal_dump(argc, argv)
 	if (rb_respond_to(port, s_binmode)) {
 	    rb_funcall2(port, s_binmode, 0, 0);
-	    reentrant_check(arg.str, s_dump_data);
+	    check_dump_arg(&arg, s_dump_data);
 	}
     }
@@ -783,4 +797,5 @@ marshal_dump(argc, argv)
     arg.data    = st_init_numtable();
     arg.taint   = Qfalse;
+    arg.wrapper = Data_Wrap_Struct(rb_cData, mark_dump_arg, 0, &arg);
     c_arg.obj   = obj;
     c_arg.arg   = &arg;
@@ -800,9 +815,31 @@ struct load_arg {
     long offset;
     st_table *symbols;
-    VALUE data;
+    st_table *data;
     VALUE proc;
     int taint;
+    VALUE wrapper;
 };
 
+static void
+check_load_arg(arg, sym)
+    struct load_arg *arg;
+    ID sym;
+{
+    if (!DATA_PTR(arg->wrapper)) {
+        rb_raise(rb_eRuntimeError, "Marshal.load reentered at %s",
+		 rb_id2name(sym));
+    }
+}
+
+static void
+mark_load_arg(ptr)
+    void *ptr;
+{
+    struct load_arg *p = ptr;
+    if (!ptr)
+        return;
+    rb_mark_tbl(p->data);
+}
+
 static VALUE r_object _((struct load_arg *arg));
 
@@ -824,5 +861,5 @@ r_byte(arg)
 	VALUE src = arg->src;
 	VALUE v = rb_funcall2(src, s_getc, 0, 0);
-	reentrant_check(arg->data, s_getc);
+	check_load_arg(arg, s_getc);
 	if (NIL_P(v)) rb_eof_error();
 	c = (unsigned char)FIX2INT(v);
@@ -905,5 +942,5 @@ r_bytes0(len, arg)
 	VALUE n = LONG2NUM(len);
 	str = rb_funcall2(src, s_read, 1, &n);
-	reentrant_check(arg->data, s_read);
+	check_load_arg(arg, s_read);
 	if (NIL_P(str)) goto too_short;
 	StringValue(str);
@@ -968,5 +1005,5 @@ r_entry(v, arg)
     struct load_arg *arg;
 {
-    rb_hash_aset(arg->data, INT2FIX(RHASH(arg->data)->tbl->num_entries), v);
+    st_insert(arg->data, arg->data->num_entries, (st_data_t)v);
     if (arg->taint) OBJ_TAINT(v);
     return v;
@@ -1024,12 +1061,13 @@ r_object0(arg, proc, ivp, extmod)
     int type = r_byte(arg);
     long id;
+    st_data_t link;
 
     switch (type) {
       case TYPE_LINK:
 	id = r_long(arg);
-	v = rb_hash_aref(arg->data, LONG2FIX(id));
-	if (NIL_P(v)) {
+ 	if (!st_lookup(arg->data, (st_data_t)id, &link)) {
 	    rb_raise(rb_eArgError, "dump format error (unlinked)");
 	}
+	v = (st_data_t)link;
 	return v;
 
@@ -1256,5 +1294,5 @@ r_object0(arg, proc, ivp, extmod)
 	    }
 	    v = rb_funcall(klass, s_load, 1, data);
-	    reentrant_check(arg->data, s_load);
+	    check_load_arg(arg, s_load);
 	    r_entry(v, arg);
 	}
@@ -1280,5 +1318,5 @@ r_object0(arg, proc, ivp, extmod)
 	    data = r_object(arg);
 	    rb_funcall(v, s_mload, 1, data);
-	    reentrant_check(arg->data, s_mload);
+	    check_load_arg(arg, s_mload);
 	}
         break;
@@ -1307,5 +1345,5 @@ r_object0(arg, proc, ivp, extmod)
 	       }
 	       v = rb_funcall(klass, s_alloc, 0);
-	       reentrant_check(arg->data, s_alloc);
+	       check_load_arg(arg, s_alloc);
            }
 	   else {
@@ -1322,5 +1360,5 @@ r_object0(arg, proc, ivp, extmod)
            }
            rb_funcall(v, s_load_data, 1, r_object0(arg, 0, 0, extmod));
-	   reentrant_check(arg->data, s_load_data);
+	   check_load_arg(arg, s_load_data);
        }
        break;
@@ -1366,5 +1404,5 @@ r_object0(arg, proc, ivp, extmod)
     if (proc) {
 	rb_funcall(proc, s_call, 1, v);
-	reentrant_check(arg->data, s_call);
+	check_load_arg(arg, s_call);
     }
     return v;
@@ -1389,6 +1427,9 @@ load_ensure(arg)
     struct load_arg *arg;
 {
-    if (RBASIC(arg->data)->klass) return 0; /* ignore reentrant */
+    if (!DATA_PTR(arg->wrapper)) return 0;
     st_free_table(arg->symbols);
+    st_free_table(arg->data);
+    DATA_PTR(arg->wrapper) = 0;
+    arg->wrapper = 0;
     return 0;
 }
@@ -1432,5 +1473,8 @@ marshal_load(argc, argv)
     arg.src = port;
     arg.offset = 0;
-    arg.data = 0;
+    arg.symbols = st_init_numtable();
+    arg.data    = st_init_numtable();
+    arg.proc = 0;
+    arg.wrapper = Data_Wrap_Struct(rb_cData, mark_load_arg, 0, &arg);
 
     major = r_byte(&arg);
@@ -1447,9 +1491,5 @@ marshal_load(argc, argv)
     }
 
-    arg.symbols = st_init_numtable();
-    arg.data   = rb_hash_new();
-    RBASIC(arg.data)->klass = 0;
-    if (NIL_P(proc)) arg.proc = 0;
-    else             arg.proc = proc;
+    if (!NIL_P(proc)) arg.proc = proc;
     v = rb_ensure(load, (VALUE)&arg, load_ensure, (VALUE)&arg);
     RBASIC(arg.data)->klass = rb_cHash;
Index: test/ruby/test_marshal.rb
===================================================================
--- test/ruby/test_marshal.rb	(revision 19063)
+++ test/ruby/test_marshal.rb	(working copy)
@@ -13,9 +13,15 @@ class TestMarshal < Test::Unit::TestCase
 
   def encode(o)
+    stress, GC.stress = GC.stress, true
     Marshal.dump(o)
+  ensure
+    GC.stress = stress
   end
 
   def decode(s)
+    stress, GC.stress = GC.stress, true
     Marshal.load(s)
+  ensure
+    GC.stress = stress
   end
 
Index: test/ruby/marshaltestlib.rb
===================================================================
--- test/ruby/marshaltestlib.rb	(revision 19063)
+++ test/ruby/marshaltestlib.rb	(working copy)
@@ -194,4 +194,10 @@ module MarshalTestLib
   end
 
+  def test_fixnum_64bit
+    obj = [1220278665, 1220278662, 1220278661, 1220278661, 1220278656]
+
+    marshal_equal(obj)
+  end
+
   def test_float
     marshal_equal(-1.0)


-- 
--- 僕の前にBugはない。
--- 僕の後ろにBugはできる。
    中田 伸悦