Simplify Android key store implementation

Instead of setting RSA_FLAG_SIGN_VER and implementing the rsa_sign callback,
implement rsa_priv_enc. This allows us to let OpenSSL deal with the mess of
wrapping things in a DigestInfo and the like.

BUG=289694
TEST=TLS client auth still works

Review URL: https://chromiumcodereview.appspot.com/23477064

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@223122 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 51 insertions, 101 deletions

diff --git a/net/android/keystore_openssl.cc b/net/android/keystore_openssl.cc
index cd55ece..5ad8473 100644
--- a/net/android/keystore_openssl.cc
+++ b/net/android/keystore_openssl.cc
@@ -35,7 +35,7 @@
 //
 // Generally speaking, OpenSSL provides many different ways to sign
 // digests. This code doesn't support all these cases, only the ones that
-// are required to sign the MAC during the OpenSSL handshake for TLS.
+// are required to sign the digest during the OpenSSL handshake for TLS.
 //
 // The OpenSSL EVP_PKEY type is a generic wrapper around key pairs.
 // Internally, it can hold a pointer to a RSA, DSA or ECDSA structure,
@@ -106,7 +106,6 @@ typedef crypto::ScopedOpenSSL<RSA, RSA_free> ScopedRSA;
 typedef crypto::ScopedOpenSSL<DSA, DSA_free> ScopedDSA;
 typedef crypto::ScopedOpenSSL<EC_KEY, EC_KEY_free> ScopedEC_KEY;
 typedef crypto::ScopedOpenSSL<EC_GROUP, EC_GROUP_free> ScopedEC_GROUP;
-typedef crypto::ScopedOpenSSL<X509_SIG, X509_SIG_free> ScopedX509_SIG;
 
 // Custom RSA_METHOD that uses the platform APIs.
 // Note that for now, only signing through RSA_sign() is really supported.
@@ -133,14 +132,60 @@ int RsaMethodPubDec(int flen,
   return -1;
 }
 
+// See RSA_eay_private_encrypt in
+// third_party/openssl/openssl/crypto/rsa/rsa_eay.c for the default
+// implementation of this function.
 int RsaMethodPrivEnc(int flen,
                      const unsigned char *from,
                      unsigned char *to,
                      RSA *rsa,
                      int padding) {
-  NOTIMPLEMENTED();
-  RSAerr(RSA_F_RSA_PRIVATE_ENCRYPT, RSA_R_RSA_OPERATIONS_NOT_SUPPORTED);
-  return -1;
+  DCHECK_EQ(RSA_PKCS1_PADDING, padding);
+  if (padding != RSA_PKCS1_PADDING) {
+    // TODO(davidben): If we need to, we can implement RSA_NO_PADDING
+    // by using javax.crypto.Cipher and picking either the
+    // "RSA/ECB/NoPadding" or "RSA/ECB/PKCS1Padding" transformation as
+    // appropriate. I believe support for both of these was added in
+    // the same Android version as the "NONEwithRSA"
+    // java.security.Signature algorithm, so the same version checks
+    // for GetRsaLegacyKey should work.
+    RSAerr(RSA_F_RSA_PRIVATE_ENCRYPT, RSA_R_UNKNOWN_PADDING_TYPE);
+    return -1;
+  }
+
+  // Retrieve private key JNI reference.
+  jobject private_key = reinterpret_cast<jobject>(RSA_get_app_data(rsa));
+  if (!private_key) {
+    LOG(WARNING) << "Null JNI reference passed to RsaMethodPrivEnc!";
+    RSAerr(RSA_F_RSA_PRIVATE_ENCRYPT, ERR_R_INTERNAL_ERROR);
+    return -1;
+  }
+
+  base::StringPiece from_piece(reinterpret_cast<const char*>(from), flen);
+  std::vector<uint8> result;
+  // For RSA keys, this function behaves as RSA_private_encrypt with
+  // PKCS#1 padding.
+  if (!RawSignDigestWithPrivateKey(private_key, from_piece, &result)) {
+    LOG(WARNING) << "Could not sign message in RsaMethodPrivEnc!";
+    RSAerr(RSA_F_RSA_PRIVATE_ENCRYPT, ERR_R_INTERNAL_ERROR);
+    return -1;
+  }
+
+  size_t expected_size = static_cast<size_t>(RSA_size(rsa));
+  if (result.size() > expected_size) {
+    LOG(ERROR) << "RSA Signature size mismatch, actual: "
+               <<  result.size() << ", expected <= " << expected_size;
+    RSAerr(RSA_F_RSA_PRIVATE_ENCRYPT, ERR_R_INTERNAL_ERROR);
+    return -1;
+  }
+
+  // Copy result to OpenSSL-provided buffer. RawSignDigestWithPrivateKey
+  // should pad with leading 0s, but if it doesn't, pad the result.
+  size_t zero_pad = expected_size - result.size();
+  memset(to, 0, zero_pad);
+  memcpy(to + zero_pad, &result[0], result.size());
+
+  return expected_size;
 }
 
 int RsaMethodPrivDec(int flen,
@@ -154,8 +199,6 @@ int RsaMethodPrivDec(int flen,
 }
 
 int RsaMethodInit(RSA* rsa) {
-  // Required to ensure that RsaMethodSign will be called.
-  rsa->flags |= RSA_FLAG_SIGN_VER;
   return 0;
 }
 
@@ -173,99 +216,6 @@ int RsaMethodFinish(RSA* rsa) {
   return 0;
 }
 
-// Although these parameters are, per OpenSSL, named |message| and
-// |message_len|, RsaMethodSign is actually passed a message digest,
-// not the original message.
-int RsaMethodSign(int type,
-                  const unsigned char* message,
-                  unsigned int message_len,
-                  unsigned char* signature,
-                  unsigned int* signature_len,
-                  const RSA* rsa) {
-  // Retrieve private key JNI reference.
-  jobject private_key = reinterpret_cast<jobject>(RSA_get_app_data(rsa));
-  if (!private_key) {
-    LOG(WARNING) << "Null JNI reference passed to RsaMethodSign!";
-    return 0;
-  }
-
-  // See RSA_sign in third_party/openssl/openssl/crypto/rsa/rsa_sign.c.
-  base::StringPiece message_piece;
-  std::vector<uint8> buffer;  // To store |message| wrapped in a DigestInfo.
-  if (type == NID_md5_sha1) {
-    // For TLS < 1.2, sign just |message|.
-    message_piece.set(message, static_cast<size_t>(message_len));
-  } else {
-    // For TLS 1.2, wrap |message| in a PKCS #1 DigestInfo before signing.
-    ScopedX509_SIG sig(X509_SIG_new());
-    if (!sig.get())
-      return 0;
-    if (X509_ALGOR_set0(sig.get()->algor,
-                        OBJ_nid2obj(type), V_ASN1_NULL, 0) != 1) {
-      return 0;
-    }
-    if (sig.get()->algor->algorithm == NULL) {
-      RSAerr(RSA_F_RSA_SIGN, RSA_R_UNKNOWN_ALGORITHM_TYPE);
-      return 0;
-    }
-    if (sig.get()->algor->algorithm->length == 0) {
-      RSAerr(RSA_F_RSA_SIGN,
-             RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
-      return 0;
-    }
-    if (ASN1_OCTET_STRING_set(sig.get()->digest, message, message_len) != 1)
-      return 0;
-
-    int len = i2d_X509_SIG(sig.get(), NULL);
-    if (len < 0) {
-      LOG(WARNING) << "Couldn't encode X509_SIG structure";
-      return 0;
-    }
-    buffer.resize(len);
-    // OpenSSL takes a pointer to a pointer so it can kindly increment
-    // it for you.
-    unsigned char* p = &buffer[0];
-    len = i2d_X509_SIG(sig.get(), &p);
-    if (len < 0) {
-      LOG(WARNING) << "Couldn't encode X509_SIG structure";
-      return 0;
-    }
-
-    message_piece.set(&buffer[0], static_cast<size_t>(len));
-  }
-
-  // Sanity-check the size.
-  //
-  // TODO(davidben): Do we need to do this? OpenSSL does, but
-  // RawSignDigestWithPrivateKey does error on sufficiently large
-  // input. However, it doesn't take the padding into account.
-  size_t expected_size = static_cast<size_t>(RSA_size(rsa));
-  if (message_piece.size() > expected_size - RSA_PKCS1_PADDING_SIZE) {
-    RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
-    return 0;
-  }
-
-  // Sign |message_piece| with the private key through JNI.
-  std::vector<uint8> result;
-
-  if (!RawSignDigestWithPrivateKey(
-      private_key, message_piece, &result)) {
-    LOG(WARNING) << "Could not sign message in RsaMethodSign!";
-    return 0;
-  }
-
-  if (result.size() > expected_size) {
-    LOG(ERROR) << "RSA Signature size mismatch, actual: "
-               <<  result.size() << ", expected <= " << expected_size;
-    return 0;
-  }
-
-  // Copy result to OpenSSL-provided buffer
-  memcpy(signature, &result[0], result.size());
-  *signature_len = static_cast<unsigned int>(result.size());
-  return 1;
-}
-
 const RSA_METHOD android_rsa_method = {
   /* .name = */ "Android signing-only RSA method",
   /* .rsa_pub_enc = */ RsaMethodPubEnc,
@@ -281,7 +231,7 @@ const RSA_METHOD android_rsa_method = {
   // it's not valid for the certificate.
   /* .flags = */ RSA_METHOD_FLAG_NO_CHECK,
   /* .app_data = */ NULL,
-  /* .rsa_sign = */ RsaMethodSign,
+  /* .rsa_sign = */ NULL,
   /* .rsa_verify = */ NULL,
   /* .rsa_keygen = */ NULL,
 };