Re: Bug in Fixnum#& with respect to coercion into a bignum

Hi,

At Sun, 9 Mar 2008 21:26:05 +0900,
Dirkjan Bussink wrote in [ruby-core:15838]:
> The last line fails, but what I would expect is that it the object is  
> coerced correctly. The cause of this is that in fix_and in numeric.c,  
> only a direct Bignum is checked. If y in fix_and is not a Fixnum or  
> Bignum, it should be coerced first and then the result should be  
> checked for the Bignum type. This would easily fix this bug.
> 
> The same problem also applies to fix_or and fix_xor of course.

Thank you for the report.


Index: numeric.c
===================================================================
--- numeric.c	(revision 15737)
+++ numeric.c	(working copy)
@@ -1494,4 +1494,5 @@ SIGNED_VALUE
 rb_num2long(VALUE val)
 {
+  again:
     if (NIL_P(val)) {
 	rb_raise(rb_eTypeError, "no implicit conversion from nil to integer");
@@ -1520,5 +1521,5 @@ rb_num2long(VALUE val)
       default:
 	val = rb_to_int(val);
-	return NUM2LONG(val);
+	goto again;
     }
 }
@@ -2594,4 +2595,13 @@ fix_rev(VALUE num)
 }
 
+static VALUE
+fix_coerce(VALUE x)
+{
+    while (!FIXNUM_P(x) && TYPE(x) != T_BIGNUM) {
+	x = rb_to_int(x);
+    }
+    return x;
+}
+
 /*
  * call-seq:
@@ -2606,8 +2616,8 @@ fix_and(VALUE x, VALUE y)
     long val;
 
-    if (TYPE(y) == T_BIGNUM) {
+    if (!FIXNUM_P(y = fix_coerce(y))) {
 	return rb_big_and(y, x);
     }
-    val = FIX2LONG(x) & NUM2LONG(y);
+    val = FIX2LONG(x) & FIX2LONG(y);
     return LONG2NUM(val);
 }
@@ -2625,8 +2635,8 @@ fix_or(VALUE x, VALUE y)
     long val;
 
-    if (TYPE(y) == T_BIGNUM) {
+    if (!FIXNUM_P(y = fix_coerce(y))) {
 	return rb_big_or(y, x);
     }
-    val = FIX2LONG(x) | NUM2LONG(y);
+    val = FIX2LONG(x) | FIX2LONG(y);
     return LONG2NUM(val);
 }
@@ -2644,8 +2654,8 @@ fix_xor(VALUE x, VALUE y)
     long val;
 
-    if (TYPE(y) == T_BIGNUM) {
+    if (!FIXNUM_P(y = fix_coerce(y))) {
 	return rb_big_xor(y, x);
     }
-    val = FIX2LONG(x) ^ NUM2LONG(y);
+    val = FIX2LONG(x) ^ FIX2LONG(y);
     return LONG2NUM(val);
 }
@@ -2741,5 +2751,5 @@ fix_aref(VALUE fix, VALUE idx)
     long i;
 
-    if (TYPE(idx) == T_BIGNUM) {
+    if (!FIXNUM_P(idx = fix_coerce(idx))) {
 	idx = rb_big_norm(idx);
 	if (!FIXNUM_P(idx)) {
@@ -2749,5 +2759,5 @@ fix_aref(VALUE fix, VALUE idx)
 	}
     }
-    i = NUM2LONG(idx);
+    i = FIX2LONG(idx);
 
     if (i < 0) return INT2FIX(0);


-- 
Nobu Nakada