From: Jared Jennings <jjenning@...>
Date: 2011-06-29T07:33:32+09:00
Subject: OpenSSL in FIPS compliant mode leads to crashes

When a Red Hat Enterprise Linux system is in FIPS compliant mode, the
file /proc/sys/crypto/fips_enabled contains "1". OpenSSL detects this
and acts differently. One thing it does is to refuse to perform an MD5
digest. This failure is detectable by means of the return value of
EVP_DigestInit_ex, but the openssl module in Ruby 1.8.7 ignores this
return value. Then, when openssl goes on to try to use the digest,
OpenSSL prints an error message and calls abort(), killing the Ruby
interpreter.

The following patch makes openssl check the return value of
EVP_DigestInit_ex and raises an exception if this function fails.

(I tried and failed to subscribe to ruby-core, which seemed like a
better place to post this.)

--- ruby-1.8.7.299/ruby-1.8.7-p299/ext/openssl/ossl_digest.c	2008-02-25
02:48:57.000000000 -0600
+++ ruby-1.8.7.299-digestfail/ruby-1.8.7-p299/ext/openssl/ossl_digest.c	2011-06-28
15:56:20.000000000 -0500
@@ -62,7 +62,9 @@

     ret = ossl_digest_alloc(cDigest);
     GetDigest(ret, ctx);
-    EVP_DigestInit_ex(ctx, md, NULL);
+    if(!EVP_DigestInit_ex(ctx, md, NULL)) {
+        rb_raise(rb_eArgError, "digest algorithm init failed");
+    }

     return ret;
 }
@@ -104,7 +106,9 @@
     if (!NIL_P(data)) StringValue(data);

     GetDigest(self, ctx);
-    EVP_DigestInit_ex(ctx, md, NULL);
+    if(!EVP_DigestInit_ex(ctx, md, NULL)) {
+        rb_raise(rb_eArgError, "digest algorithm init failed");
+    }

     if (!NIL_P(data)) return ossl_digest_update(self, data);
     return self;
@@ -138,7 +142,9 @@
     EVP_MD_CTX *ctx;

     GetDigest(self, ctx);
-    EVP_DigestInit_ex(ctx, EVP_MD_CTX_md(ctx), NULL);
+    if(!EVP_DigestInit_ex(ctx, EVP_MD_CTX_md(ctx), NULL)) {
+        rb_raise(rb_eArgError, "digest algorithm init failed");
+    }

     return self;
 }