crypto: use minimal ASN.1 lengths.

An NSS update has (correctly) become stricter about ASN.1 lengths and is
causing SignatureVerifierTest.VerifyRSAPSS to fail.

This change fixes the test so that it uses minimal (DER) lengths.

(I don't actually have the updated NSS, so this is speculative.)

BUG=447759

Review URL: https://codereview.chromium.org/844073003

Cr-Commit-Position: refs/heads/master@{#310914}

1 files changed, 25 insertions, 18 deletions

diff --git a/crypto/signature_verifier_unittest.cc b/crypto/signature_verifier_unittest.cc
index b521bd7..a661ff7 100644
--- a/crypto/signature_verifier_unittest.cc
+++ b/crypto/signature_verifier_unittest.cc
@@ -1000,6 +1000,23 @@ static bool DecodeTestInput(const char* in, std::vector<uint8>* out) {
   return true;
 }
 
+// PrependASN1Length prepends an ASN.1 serialized length to the beginning of
+// |out|.
+static void PrependASN1Length(std::vector<uint8>* out, size_t len) {
+  if (len < 128) {
+    out->insert(out->begin(), static_cast<uint8>(len));
+  } else if (len < 256) {
+    out->insert(out->begin(), static_cast<uint8>(len));
+    out->insert(out->begin(), 0x81);
+  } else if (len < 0x10000) {
+    out->insert(out->begin(), static_cast<uint8>(len));
+    out->insert(out->begin(), static_cast<uint8>(len >> 8));
+    out->insert(out->begin(), 0x82);
+  } else {
+    CHECK(false) << "ASN.1 length not handled: " << len;
+  }
+}
+
 static bool EncodeRSAPublicKey(const std::vector<uint8>& modulus_n,
                                const std::vector<uint8>& public_exponent_e,
                                std::vector<uint8>* public_key_info) {
@@ -1027,37 +1044,28 @@ static bool EncodeRSAPublicKey(const std::vector<uint8>& modulus_n,
   public_key_info->insert(public_key_info->begin(),
                           public_exponent_e.begin(),
                           public_exponent_e.end());
-  uint8 exponent_size = base::checked_cast<uint8>(public_exponent_e.size());
-  public_key_info->insert(public_key_info->begin(), exponent_size);
+  PrependASN1Length(public_key_info, public_exponent_e.size());
   public_key_info->insert(public_key_info->begin(), kIntegerTag);
 
   // Encode the modulus n as an INTEGER.
   public_key_info->insert(public_key_info->begin(),
                           modulus_n.begin(), modulus_n.end());
-  uint16 modulus_size = base::checked_cast<uint16>(modulus_n.size());
+  size_t modulus_size = modulus_n.size();
   if (modulus_n[0] & 0x80) {
     public_key_info->insert(public_key_info->begin(), 0x00);
     modulus_size++;
   }
-  public_key_info->insert(public_key_info->begin(), modulus_size & 0xff);
-  public_key_info->insert(public_key_info->begin(), (modulus_size >> 8) & 0xff);
-  public_key_info->insert(public_key_info->begin(), 0x82);
+  PrependASN1Length(public_key_info, modulus_size);
   public_key_info->insert(public_key_info->begin(), kIntegerTag);
 
   // Encode the RSAPublicKey SEQUENCE.
-  uint16 info_size = base::checked_cast<uint16>(public_key_info->size());
-  public_key_info->insert(public_key_info->begin(), info_size & 0xff);
-  public_key_info->insert(public_key_info->begin(), (info_size >> 8) & 0xff);
-  public_key_info->insert(public_key_info->begin(), 0x82);
+  PrependASN1Length(public_key_info, public_key_info->size());
   public_key_info->insert(public_key_info->begin(), kSequenceTag);
 
   // Encode the BIT STRING.
   // Number of unused bits.
   public_key_info->insert(public_key_info->begin(), 0x00);
-  info_size = base::checked_cast<uint16>(public_key_info->size());
-  public_key_info->insert(public_key_info->begin(), info_size & 0xff);
-  public_key_info->insert(public_key_info->begin(), (info_size >> 8) & 0xff);
-  public_key_info->insert(public_key_info->begin(), 0x82);
+  PrependASN1Length(public_key_info, public_key_info->size());
   public_key_info->insert(public_key_info->begin(), kBitStringTag);
 
   // Encode the AlgorithmIdentifier.
@@ -1071,10 +1079,7 @@ static bool EncodeRSAPublicKey(const std::vector<uint8>& modulus_n,
                           algorithm, algorithm + sizeof(algorithm));
 
   // Encode the outermost SEQUENCE.
-  info_size = base::checked_cast<uint16>(public_key_info->size());
-  public_key_info->insert(public_key_info->begin(), info_size & 0xff);
-  public_key_info->insert(public_key_info->begin(), (info_size >> 8) & 0xff);
-  public_key_info->insert(public_key_info->begin(), 0x82);
+  PrependASN1Length(public_key_info, public_key_info->size());
   public_key_info->insert(public_key_info->begin(), kSequenceTag);
 
   return true;
@@ -1082,6 +1087,7 @@ static bool EncodeRSAPublicKey(const std::vector<uint8>& modulus_n,
 
 TEST(SignatureVerifierTest, VerifyRSAPSS) {
   for (unsigned int i = 0; i < arraysize(pss_test); i++) {
+    SCOPED_TRACE(i);
     std::vector<uint8> modulus_n;
     std::vector<uint8> public_exponent_e;
     ASSERT_TRUE(DecodeTestInput(pss_test[i].modulus_n, &modulus_n));
@@ -1092,6 +1098,7 @@ TEST(SignatureVerifierTest, VerifyRSAPSS) {
                                    &public_key_info));
 
     for (unsigned int j = 0; j < arraysize(pss_test[i].example); j++) {
+      SCOPED_TRACE(j);
       std::vector<uint8> message;
       std::vector<uint8> salt;
       std::vector<uint8> signature;