Add SignatureVerifier::VerifyInitRSAPSS for verifying RSA-PSS signatures.

Change the OpenSSL-based SignatureVerifier to use EVP_DigestVerifyInit
instead of EVP_VerifyInit_ex.

Copy the PSS padding verification code from NSS to the NSS-based
SignatureVerifier because the RSA-PSS code in the NSS softoken isn't
exposed via the NSS PK11_ or VFY_ functions yet.

R=agl@chromium.org,rsleevi@chromium.org
BUG=none
TEST=to be added to net_unittests via testing net::quic::ProofVerifier.

Review URL: https://chromiumcodereview.appspot.com/17776003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@209178 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 77 insertions, 22 deletions

diff --git a/crypto/signature_verifier_openssl.cc b/crypto/signature_verifier_openssl.cc
index cab45db..1e71339 100644
--- a/crypto/signature_verifier_openssl.cc
+++ b/crypto/signature_verifier_openssl.cc
@@ -16,8 +16,21 @@
 
 namespace crypto {
 
+namespace {
+
+const EVP_MD* ToOpenSSLDigest(SignatureVerifier::HashAlgorithm hash_alg) {
+  switch (hash_alg) {
+    case SignatureVerifier::SHA1:
+      return EVP_sha1();
+    case SignatureVerifier::SHA256:
+      return EVP_sha256();
+  }
+  return EVP_md_null();
+}
+
+}  // namespace
+
 struct SignatureVerifier::VerifyContext {
-  ScopedOpenSSL<EVP_PKEY, EVP_PKEY_free> public_key;
   ScopedOpenSSL<EVP_MD_CTX, EVP_MD_CTX_destroy> ctx;
 };
 
@@ -35,33 +48,44 @@ bool SignatureVerifier::VerifyInit(const uint8* signature_algorithm,
                                    int signature_len,
                                    const uint8* public_key_info,
                                    int public_key_info_len) {
-  DCHECK(!verify_context_);
-  verify_context_ = new VerifyContext;
   OpenSSLErrStackTracer err_tracer(FROM_HERE);
-
   ScopedOpenSSL<X509_ALGOR, X509_ALGOR_free> algorithm(
       d2i_X509_ALGOR(NULL, &signature_algorithm, signature_algorithm_len));
   if (!algorithm.get())
     return false;
-
   const EVP_MD* digest = EVP_get_digestbyobj(algorithm.get()->algorithm);
-  DCHECK(digest);
+  if (!digest)
+    return false;
 
-  signature_.assign(signature, signature + signature_len);
+  return CommonInit(digest, signature, signature_len, public_key_info,
+                    public_key_info_len, NULL);
+}
 
-  // BIO_new_mem_buf is not const aware, but it does not modify the buffer.
-  char* data = reinterpret_cast<char*>(const_cast<uint8*>(public_key_info));
-  ScopedOpenSSL<BIO, BIO_free_all> bio(BIO_new_mem_buf(data,
-                                                       public_key_info_len));
-  if (!bio.get())
-    return false;
+bool SignatureVerifier::VerifyInitRSAPSS(HashAlgorithm hash_alg,
+                                         HashAlgorithm mask_hash_alg,
+                                         int salt_len,
+                                         const uint8* signature,
+                                         int signature_len,
+                                         const uint8* public_key_info,
+                                         int public_key_info_len) {
+  OpenSSLErrStackTracer err_tracer(FROM_HERE);
+  const EVP_MD* digest = ToOpenSSLDigest(hash_alg);
+  DCHECK(digest);
 
-  verify_context_->public_key.reset(d2i_PUBKEY_bio(bio.get(), NULL));
-  if (!verify_context_->public_key.get())
+  EVP_PKEY_CTX* pkey_ctx;
+  if (!CommonInit(digest, signature, signature_len, public_key_info,
+                  public_key_info_len, &pkey_ctx)) {
     return false;
+  }
 
-  verify_context_->ctx.reset(EVP_MD_CTX_create());
-  int rv = EVP_VerifyInit_ex(verify_context_->ctx.get(), digest, NULL);
+  int rv = EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_PSS_PADDING);
+  if (rv != 1)
+    return false;
+  rv = EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx,
+                                    ToOpenSSLDigest(mask_hash_alg));
+  if (rv != 1)
+    return false;
+  rv = EVP_PKEY_CTX_set_rsa_pss_saltlen(pkey_ctx, salt_len);
   return rv == 1;
 }
 
@@ -69,22 +93,53 @@ void SignatureVerifier::VerifyUpdate(const uint8* data_part,
                                      int data_part_len) {
   DCHECK(verify_context_);
   OpenSSLErrStackTracer err_tracer(FROM_HERE);
-  int rv = EVP_VerifyUpdate(verify_context_->ctx.get(),
-                            data_part, data_part_len);
+  int rv = EVP_DigestVerifyUpdate(verify_context_->ctx.get(),
+                                  data_part, data_part_len);
   DCHECK_EQ(rv, 1);
 }
 
 bool SignatureVerifier::VerifyFinal() {
   DCHECK(verify_context_);
   OpenSSLErrStackTracer err_tracer(FROM_HERE);
-  int rv = EVP_VerifyFinal(verify_context_->ctx.get(),
-                           vector_as_array(&signature_), signature_.size(),
-                           verify_context_->public_key.get());
+  int rv = EVP_DigestVerifyFinal(verify_context_->ctx.get(),
+                                 vector_as_array(&signature_),
+                                 signature_.size());
   DCHECK_GE(rv, 0);
   Reset();
   return rv == 1;
 }
 
+bool SignatureVerifier::CommonInit(const EVP_MD* digest,
+                                   const uint8* signature,
+                                   int signature_len,
+                                   const uint8* public_key_info,
+                                   int public_key_info_len,
+                                   EVP_PKEY_CTX** pkey_ctx) {
+  if (verify_context_)
+    return false;
+
+  verify_context_ = new VerifyContext;
+
+  signature_.assign(signature, signature + signature_len);
+
+  // BIO_new_mem_buf is not const aware, but it does not modify the buffer.
+  char* data = reinterpret_cast<char*>(const_cast<uint8*>(public_key_info));
+  ScopedOpenSSL<BIO, BIO_free_all> bio(BIO_new_mem_buf(data,
+                                                       public_key_info_len));
+  if (!bio.get())
+    return false;
+
+  ScopedOpenSSL<EVP_PKEY, EVP_PKEY_free> public_key(
+      d2i_PUBKEY_bio(bio.get(), NULL));
+  if (!public_key.get())
+    return false;
+
+  verify_context_->ctx.reset(EVP_MD_CTX_create());
+  int rv = EVP_DigestVerifyInit(verify_context_->ctx.get(), pkey_ctx,
+                                digest, NULL, public_key.get());
+  return rv == 1;
+}
+
 void SignatureVerifier::Reset() {
   delete verify_context_;
   verify_context_ = NULL;