From: Sam Roberts <sroberts@...>
Date: 2004-12-05T05:18:37+09:00
Subject: [patch] for openssl to support non-pkcs1 padding

Quoteing sroberts@uniserve.com, on Sun, Dec 05, 2004 at 04:03:08AM +0900:
> I want to do unpadded RSA operations with openssl, but I discovered
> that ossl_pkey_rsa.c hard-codes the use of PKCS#1 padding into the
> ruby methods [private/public]_[encrypt/decrypt].

So I wrote a patch, any chance of integrating this?

I think the hack I made to openssl.rb to define the constants is wrong,
but I'm not sure how this is supposed to be done.

Below is unit test and diff.

Thanks,
Sam

+++ osslraw.rb +++
require 'test/unit'

require './ext/openssl/openssl.bundle'

class Padding < Test::Unit::TestCase
  def test_padding

    key = OpenSSL::PKey::RSA.new(512, 3)

    # 1 == PKCS1 padding, 3 = raw

    #
    # No padding
    #

    # Need right size for raw mode
    plain0 = "x" * (512/8)

    cipher = key.private_encrypt(plain0, 3)

    plain1  = key.public_decrypt(cipher, 3)

    assert_equal(plain0, plain1, plain0)

    #
    # PKCS1 padding, using 1 and no arg to get it, all should be equiv
    #

    # Need smaller size for pkcs1 mode
    plain0 = "x" * (512/8 - 11)

    cipherp1 = key.private_encrypt(plain0, 1)

    plain1  = key.public_decrypt(cipherp1, 1)

    assert_equal(plain0, plain1, plain0)

    cipherdef = key.private_encrypt(plain0)

    plain1  = key.public_decrypt(cipherdef)

    assert_equal(plain0, plain1, plain0)

    assert_equal(cipherp1, cipherdef)

    # Failure cases
    assert_raise(ArgumentError) { key.private_encrypt() }
    assert_raise(ArgumentError) { key.private_encrypt("hi", 1, nil) }
    assert_raise(TypeError) { key.private_encrypt(plain0, nil) }
    assert_raise(OpenSSL::PKey::RSAError) { key.private_encrypt(plain0, 666) }

  end
end

+++ patch +++
Index: ext/openssl/ossl_pkey_rsa.c
===================================================================
RCS file: /src/ruby/ext/openssl/ossl_pkey_rsa.c,v
retrieving revision 1.5.2.2
diff -u -r1.5.2.2 ossl_pkey_rsa.c
--- ext/openssl/ossl_pkey_rsa.c	30 Jun 2004 18:34:59 -0000	1.5.2.2
+++ ext/openssl/ossl_pkey_rsa.c	4 Dec 2004 20:10:33 -0000
@@ -228,7 +228,7 @@
 ossl_rsa_to_der(VALUE self)
 {
     EVP_PKEY *pkey;
-    int (*i2d_func)_((const RSA*, unsigned char**));
+    int (*i2d_func)_((RSA*, unsigned char**));
     unsigned char *p;
     long len;
     VALUE str;
@@ -249,21 +249,42 @@
     return str;
 }
 
+static void
+ossl_crypt_args(int argc, VALUE* values, VALUE* buffer, int* padding)
+{
+  if(argc == 0)
+    rb_raise(rb_eArgError, "wrong number of arguments (0 for 1)");
+
+  if(argc > 2)
+    rb_raise(rb_eArgError, "wrong number of arguments (%d for 2)", argc);
+
+  *buffer = values[0];
+
+  /* Default to old behaviour, PKCS#1 padding. */
+  if(argc == 1)
+    *padding = RSA_PKCS1_PADDING;
+  else
+    *padding = (int) NUM2INT(values[1]);
+}
+
 #define ossl_rsa_buf_size(pkey) (RSA_size((pkey)->pkey.rsa)+16)
 
 static VALUE
-ossl_rsa_public_encrypt(VALUE self, VALUE buffer)
+ossl_rsa_public_encrypt(int argc, VALUE* values, VALUE self)
 {
     EVP_PKEY *pkey;
     int buf_len;
     VALUE str;
+    VALUE buffer;
+    int padding;
+    ossl_crypt_args(argc, values, &buffer, &padding);
 	
     GetPKeyRSA(self, pkey);
     StringValue(buffer);
     str = rb_str_new(0, ossl_rsa_buf_size(pkey));
     buf_len = RSA_public_encrypt(RSTRING(buffer)->len, RSTRING(buffer)->ptr,
 				 RSTRING(str)->ptr, pkey->pkey.rsa,
-				 RSA_PKCS1_PADDING);
+				 padding);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
     RSTRING(str)->len = buf_len;
     RSTRING(str)->ptr[buf_len] = 0;
@@ -272,18 +293,21 @@
 }
 
 static VALUE
-ossl_rsa_public_decrypt(VALUE self, VALUE buffer)
+ossl_rsa_public_decrypt(int argc, VALUE* values, VALUE self)
 {
     EVP_PKEY *pkey;
     int buf_len;
     VALUE str;
+    VALUE buffer;
+    int padding;
+    ossl_crypt_args(argc, values, &buffer, &padding);
 
     GetPKeyRSA(self, pkey);
     StringValue(buffer);
     str = rb_str_new(0, ossl_rsa_buf_size(pkey));
     buf_len = RSA_public_decrypt(RSTRING(buffer)->len, RSTRING(buffer)->ptr,
 				 RSTRING(str)->ptr, pkey->pkey.rsa,
-				 RSA_PKCS1_PADDING);
+				 padding);
     if(buf_len < 0) ossl_raise(eRSAError, NULL);
     RSTRING(str)->len = buf_len;
     RSTRING(str)->ptr[buf_len] = 0;
@@ -292,11 +316,14 @@
 }
 
 static VALUE
-ossl_rsa_private_encrypt(VALUE self, VALUE buffer)
+ossl_rsa_private_encrypt(int argc, VALUE* values, VALUE self)
 {
     EVP_PKEY *pkey;
     int buf_len;
     VALUE str;
+    VALUE buffer;
+    int padding;
+    ossl_crypt_args(argc, values, &buffer, &padding);
 	
     GetPKeyRSA(self, pkey);
     if (!RSA_PRIVATE(pkey->pkey.rsa)) {
@@ -306,7 +333,7 @@
     str = rb_str_new(0, ossl_rsa_buf_size(pkey));
     buf_len = RSA_private_encrypt(RSTRING(buffer)->len, RSTRING(buffer)->ptr,
 				  RSTRING(str)->ptr, pkey->pkey.rsa,
-				  RSA_PKCS1_PADDING);
+				  padding);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
     RSTRING(str)->len = buf_len;
     RSTRING(str)->ptr[buf_len] = 0;
@@ -315,11 +342,14 @@
 }
 
 static VALUE
-ossl_rsa_private_decrypt(VALUE self, VALUE buffer)
+ossl_rsa_private_decrypt(int argc, VALUE* values, VALUE self)
 {
     EVP_PKEY *pkey;
     int buf_len;
     VALUE str;
+    VALUE buffer;
+    int padding;
+    ossl_crypt_args(argc, values, &buffer, &padding);
 
     GetPKeyRSA(self, pkey);
     if (!RSA_PRIVATE(pkey->pkey.rsa)) {
@@ -329,7 +359,7 @@
     str = rb_str_new(0, ossl_rsa_buf_size(pkey));
     buf_len = RSA_private_decrypt(RSTRING(buffer)->len, RSTRING(buffer)->ptr,
 				  RSTRING(str)->ptr, pkey->pkey.rsa,
-				  RSA_PKCS1_PADDING);
+				  padding);
     if (buf_len < 0) ossl_raise(eRSAError, NULL);
     RSTRING(str)->len = buf_len;
     RSTRING(str)->ptr[buf_len] = 0;
@@ -469,10 +499,10 @@
     rb_define_alias(cRSA, "to_s", "export");
     rb_define_method(cRSA, "to_der", ossl_rsa_to_der, 0);
     rb_define_method(cRSA, "public_key", ossl_rsa_to_public_key, 0);
-    rb_define_method(cRSA, "public_encrypt", ossl_rsa_public_encrypt, 1);
-    rb_define_method(cRSA, "public_decrypt", ossl_rsa_public_decrypt, 1);
-    rb_define_method(cRSA, "private_encrypt", ossl_rsa_private_encrypt, 1);
-    rb_define_method(cRSA, "private_decrypt", ossl_rsa_private_decrypt, 1);
+    rb_define_method(cRSA, "public_encrypt", ossl_rsa_public_encrypt, -1);
+    rb_define_method(cRSA, "public_decrypt", ossl_rsa_public_decrypt, -1);
+    rb_define_method(cRSA, "private_encrypt", ossl_rsa_private_encrypt, -1);
+    rb_define_method(cRSA, "private_decrypt", ossl_rsa_private_decrypt, -1);
 
     DEF_OSSL_PKEY_BN(cRSA, rsa, n);
     DEF_OSSL_PKEY_BN(cRSA, rsa, e);
Index: ext/openssl/lib/openssl.rb
===================================================================
RCS file: /src/ruby/ext/openssl/lib/openssl.rb,v
retrieving revision 1.1
diff -u -r1.1 openssl.rb
--- ext/openssl/lib/openssl.rb	23 Jul 2003 16:11:29 -0000	1.1
+++ ext/openssl/lib/openssl.rb	4 Dec 2004 20:10:33 -0000
@@ -22,3 +22,10 @@
 require 'openssl/ssl'
 require 'openssl/x509'
 
+module OpenSSL
+ RSA_PKCS1_PADDING	= 1
+ RSA_SSLV23_PADDING	= 2
+ RSA_NO_PADDING		= 3
+ RSA_PKCS1_OAEP_PADDING	= 4
+end
+