Certificate Transparency: Add UMA for SCT presence, type.

Measure the following aspects for Certificate Transparency:
* Number of Signed Certificate Timestamps (SCTs) available with each certificate.
* The distribution of SCTs' origins.
* The distribution of SCTs' validation status.

Measurements are taken each time a certificate is validated, regardless of the resource type served over this SSL connection.

BUG=309578

Review URL: https://codereview.chromium.org/133713002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@246312 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 176 insertions, 20 deletions

diff --git a/net/cert/ct_signed_certificate_timestamp_log_param.cc b/net/cert/ct_signed_certificate_timestamp_log_param.cc
index 7519087..e7bb97c 100644
--- a/net/cert/ct_signed_certificate_timestamp_log_param.cc
+++ b/net/cert/ct_signed_certificate_timestamp_log_param.cc
@@ -27,6 +27,8 @@ const char* OriginToString(ct::SignedCertificateTimestamp::Origin origin) {
       return "tls_extension";
     case ct::SignedCertificateTimestamp::SCT_FROM_OCSP_RESPONSE:
       return "ocsp";
+    case ct::SignedCertificateTimestamp::SCT_ORIGIN_MAX:
+      break;
   }
 
   return "unknown";
diff --git a/net/cert/multi_log_ct_verifier.cc b/net/cert/multi_log_ct_verifier.cc
index 05a32da..5094da0 100644
--- a/net/cert/multi_log_ct_verifier.cc
+++ b/net/cert/multi_log_ct_verifier.cc
@@ -6,6 +6,7 @@
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
+#include "base/metrics/histogram.h"
 #include "net/base/net_errors.h"
 #include "net/base/net_log.h"
 #include "net/cert/ct_log_verifier.h"
@@ -13,10 +14,45 @@
 #include "net/cert/ct_serialization.h"
 #include "net/cert/ct_signed_certificate_timestamp_log_param.h"
 #include "net/cert/ct_verify_result.h"
+#include "net/cert/sct_status_flags.h"
 #include "net/cert/x509_certificate.h"
 
 namespace net {
 
+namespace {
+
+// Record SCT verification status. This metric would help detecting presence
+// of unknown CT logs as well as bad deployments (invalid SCTs).
+void LogSCTStatusToUMA(ct::SCTVerifyStatus status) {
+  UMA_HISTOGRAM_ENUMERATION(
+      "Net.CertificateTransparency.SCTStatus", status, ct::SCT_STATUS_MAX);
+}
+
+// Record SCT origin enum. This metric measure the popularity
+// of the various channels of providing SCTs for a certificate.
+void LogSCTOriginToUMA(ct::SignedCertificateTimestamp::Origin origin) {
+  UMA_HISTOGRAM_ENUMERATION("Net.CertificateTransparency.SCTOrigin",
+                            origin,
+                            ct::SignedCertificateTimestamp::SCT_ORIGIN_MAX);
+}
+
+// Count the number of SCTs that were available for each SSL connection
+// (including SCTs embedded in the certificate).
+// This metric would allow measuring:
+// * Of all SSL connections, how many had SCTs available for validation.
+// * When SCTs are available, how many are available per connection.
+void LogNumSCTsToUMA(const ct::CTVerifyResult& result) {
+  UMA_HISTOGRAM_CUSTOM_COUNTS("Net.CertificateTransparency.SCTsPerConnection",
+                              result.invalid_scts.size() +
+                                  result.verified_scts.size() +
+                                  result.unknown_logs_scts.size(),
+                              1,
+                              10,
+                              11);
+}
+
+}  // namespace
+
 MultiLogCTVerifier::MultiLogCTVerifier() { }
 
 MultiLogCTVerifier::~MultiLogCTVerifier() { }
@@ -104,6 +140,8 @@ int MultiLogCTVerifier::Verify(
       NetLog::TYPE_SIGNED_CERTIFICATE_TIMESTAMPS_CHECKED,
       net_log_checked_callback);
 
+  LogNumSCTsToUMA(*result);
+
   if (has_verified_scts)
     return OK;
 
@@ -128,9 +166,11 @@ bool MultiLogCTVerifier::VerifySCTs(
   for (std::vector<base::StringPiece>::const_iterator it = sct_list.begin();
        it != sct_list.end(); ++it) {
     base::StringPiece encoded_sct(*it);
+    LogSCTOriginToUMA(origin);
 
     scoped_refptr<ct::SignedCertificateTimestamp> decoded_sct;
     if (!DecodeSignedCertificateTimestamp(&encoded_sct, &decoded_sct)) {
+      LogSCTStatusToUMA(ct::SCT_STATUS_NONE);
       // XXX(rsleevi): Should we really just skip over bad SCTs?
       continue;
     }
@@ -152,6 +192,7 @@ bool MultiLogCTVerifier::VerifySingleSCT(
   if (it == logs_.end()) {
     DVLOG(1) << "SCT does not match any known log.";
     result->unknown_logs_scts.push_back(sct);
+    LogSCTStatusToUMA(ct::SCT_STATUS_LOG_UNKNOWN);
     return false;
   }
 
@@ -160,6 +201,7 @@ bool MultiLogCTVerifier::VerifySingleSCT(
   if (!it->second->Verify(expected_entry, *sct)) {
     DVLOG(1) << "Unable to verify SCT signature.";
     result->invalid_scts.push_back(sct);
+    LogSCTStatusToUMA(ct::SCT_STATUS_INVALID);
     return false;
   }
 
@@ -167,9 +209,11 @@ bool MultiLogCTVerifier::VerifySingleSCT(
   if (sct->timestamp > base::Time::Now()) {
     DVLOG(1) << "SCT is from the future!";
     result->invalid_scts.push_back(sct);
+    LogSCTStatusToUMA(ct::SCT_STATUS_INVALID);
     return false;
   }
 
+  LogSCTStatusToUMA(ct::SCT_STATUS_OK);
   result->verified_scts.push_back(sct);
   return true;
 }
diff --git a/net/cert/multi_log_ct_verifier_unittest.cc b/net/cert/multi_log_ct_verifier_unittest.cc
index 1b15d1e..c2ae25e 100644
--- a/net/cert/multi_log_ct_verifier_unittest.cc
+++ b/net/cert/multi_log_ct_verifier_unittest.cc
@@ -8,6 +8,9 @@
 
 #include "base/file_util.h"
 #include "base/files/file_path.h"
+#include "base/metrics/histogram.h"
+#include "base/metrics/histogram_samples.h"
+#include "base/metrics/statistics_recorder.h"
 #include "base/values.h"
 #include "net/base/capturing_net_log.h"
 #include "net/base/net_errors.h"
@@ -17,6 +20,7 @@
 #include "net/cert/ct_serialization.h"
 #include "net/cert/ct_verify_result.h"
 #include "net/cert/pem_tokenizer.h"
+#include "net/cert/sct_status_flags.h"
 #include "net/cert/signed_certificate_timestamp.h"
 #include "net/cert/x509_certificate.h"
 #include "net/test/cert_test_util.h"
@@ -28,6 +32,8 @@ namespace net {
 namespace {
 
 const char kLogDescription[] = "somelog";
+const char kSCTCountHistogram[] =
+    "Net.CertificateTransparency.SCTsPerConnection";
 
 class MultiLogCTVerifierTest : public ::testing::Test {
  public:
@@ -43,6 +49,12 @@ class MultiLogCTVerifierTest : public ::testing::Test {
         der_test_cert.data(),
         der_test_cert.length());
     ASSERT_TRUE(chain_);
+
+    embedded_sct_chain_ =
+        CreateCertificateChainFromFile(GetTestCertsDirectory(),
+                                       "ct-test-embedded-cert.pem",
+                                       X509Certificate::FORMAT_AUTO);
+    ASSERT_TRUE(embedded_sct_chain_);
   }
 
   bool CheckForSingleVerifiedSCTInResult(const ct::CTVerifyResult& result) {
@@ -103,31 +115,83 @@ class MultiLogCTVerifierTest : public ::testing::Test {
     return true;
   }
 
+  std::string GetSCTListWithInvalidSCT() {
+    std::string sct(ct::GetTestSignedCertificateTimestamp());
+
+    // Change a byte inside the Log ID part of the SCT so it does
+    // not match the log used in the tests
+    sct[15] = 't';
+
+    std::string sct_list;
+    ct::EncodeSCTListForTesting(sct, &sct_list);
+    return sct_list;
+  }
+
+  bool VerifySinglePrecertificateChain(scoped_refptr<X509Certificate> chain,
+                                       const BoundNetLog& bound_net_log,
+                                       ct::CTVerifyResult* result) {
+    return verifier_->Verify(
+               chain, std::string(), std::string(), result, bound_net_log) ==
+           OK;
+  }
+
+  bool VerifySinglePrecertificateChain(scoped_refptr<X509Certificate> chain) {
+    ct::CTVerifyResult result;
+    CapturingNetLog net_log;
+    BoundNetLog bound_net_log =
+        BoundNetLog::Make(&net_log, NetLog::SOURCE_CONNECT_JOB);
+
+    return verifier_->Verify(
+               chain, std::string(), std::string(), &result, bound_net_log) ==
+           OK;
+  }
+
   bool CheckPrecertificateVerification(scoped_refptr<X509Certificate> chain) {
     ct::CTVerifyResult result;
     CapturingNetLog net_log;
     BoundNetLog bound_net_log =
       BoundNetLog::Make(&net_log, NetLog::SOURCE_CONNECT_JOB);
-    return (verifier_->Verify(chain, std::string(), std::string(), &result,
-                              bound_net_log) == OK) &&
-        CheckForSingleVerifiedSCTInResult(result) &&
-        CheckForSCTOrigin(
-            result, ct::SignedCertificateTimestamp::SCT_EMBEDDED) &&
-        CheckForEmbeddedSCTInNetLog(net_log);
+    return (VerifySinglePrecertificateChain(chain, bound_net_log, &result) &&
+            CheckForSingleVerifiedSCTInResult(result) &&
+            CheckForSCTOrigin(result,
+                              ct::SignedCertificateTimestamp::SCT_EMBEDDED) &&
+            CheckForEmbeddedSCTInNetLog(net_log));
+  }
+
+  // Histogram-related helper methods
+  int GetValueFromHistogram(std::string histogram_name, int sample_index) {
+    base::Histogram* histogram = static_cast<base::Histogram*>(
+        base::StatisticsRecorder::FindHistogram(histogram_name));
+
+    if (histogram == NULL)
+      return 0;
+
+    scoped_ptr<base::HistogramSamples> samples = histogram->SnapshotSamples();
+    return samples->GetCount(sample_index);
+  }
+
+  int NumConnectionsWithSingleSCT() {
+    return GetValueFromHistogram(kSCTCountHistogram, 1);
+  }
+
+  int NumEmbeddedSCTsInHistogram() {
+    return GetValueFromHistogram("Net.CertificateTransparency.SCTOrigin",
+                                 ct::SignedCertificateTimestamp::SCT_EMBEDDED);
+  }
+
+  int NumValidSCTsInStatusHistogram() {
+    return GetValueFromHistogram("Net.CertificateTransparency.SCTStatus",
+                                 ct::SCT_STATUS_OK);
   }
 
  protected:
   scoped_ptr<MultiLogCTVerifier> verifier_;
   scoped_refptr<X509Certificate> chain_;
+  scoped_refptr<X509Certificate> embedded_sct_chain_;
 };
 
 TEST_F(MultiLogCTVerifierTest, VerifiesEmbeddedSCT) {
-  scoped_refptr<X509Certificate> chain(
-      CreateCertificateChainFromFile(GetTestCertsDirectory(),
-                                     "ct-test-embedded-cert.pem",
-                                     X509Certificate::FORMAT_AUTO));
-  ASSERT_TRUE(chain);
-  ASSERT_TRUE(CheckPrecertificateVerification(chain));
+  ASSERT_TRUE(CheckPrecertificateVerification(embedded_sct_chain_));
 }
 
 TEST_F(MultiLogCTVerifierTest, VerifiesEmbeddedSCTWithPreCA) {
@@ -176,21 +240,58 @@ TEST_F(MultiLogCTVerifierTest,
 
 TEST_F(MultiLogCTVerifierTest,
        IdentifiesSCTFromUnknownLog) {
-  std::string sct(ct::GetTestSignedCertificateTimestamp());
+  std::string sct_list = GetSCTListWithInvalidSCT();
+  ct::CTVerifyResult result;
+
+  EXPECT_NE(OK,
+            verifier_->Verify(
+                chain_, std::string(), sct_list, &result, BoundNetLog()));
+  EXPECT_EQ(1U, result.unknown_logs_scts.size());
+  EXPECT_EQ("", result.unknown_logs_scts[0]->log_description);
+}
 
-  // Change a byte inside the Log ID part of the SCT so it does
-  // not match the log used in the tests
-  sct[15] = 't';
+TEST_F(MultiLogCTVerifierTest, CountsValidSCTsInStatusHistogram) {
+  int num_valid_scts = NumValidSCTsInStatusHistogram();
 
-  std::string sct_list;
-  ASSERT_TRUE(ct::EncodeSCTListForTesting(sct, &sct_list));
+  ASSERT_TRUE(VerifySinglePrecertificateChain(embedded_sct_chain_));
 
+  EXPECT_EQ(num_valid_scts + 1, NumValidSCTsInStatusHistogram());
+}
+
+TEST_F(MultiLogCTVerifierTest, CountsInvalidSCTsInStatusHistogram) {
+  std::string sct_list = GetSCTListWithInvalidSCT();
   ct::CTVerifyResult result;
+  int num_valid_scts = NumValidSCTsInStatusHistogram();
+  int num_invalid_scts = GetValueFromHistogram(
+      "Net.CertificateTransparency.SCTStatus", ct::SCT_STATUS_LOG_UNKNOWN);
+
   EXPECT_NE(OK,
             verifier_->Verify(chain_, std::string(), sct_list, &result,
                               BoundNetLog()));
-  EXPECT_EQ(1U, result.unknown_logs_scts.size());
-  EXPECT_EQ("", result.unknown_logs_scts[0]->log_description);
+
+  ASSERT_EQ(num_valid_scts, NumValidSCTsInStatusHistogram());
+  ASSERT_EQ(num_invalid_scts + 1,
+            GetValueFromHistogram("Net.CertificateTransparency.SCTStatus",
+                                  ct::SCT_STATUS_LOG_UNKNOWN));
+}
+
+TEST_F(MultiLogCTVerifierTest, CountsSingleEmbeddedSCTInConnectionsHistogram) {
+  int old_sct_count = NumConnectionsWithSingleSCT();
+  ASSERT_TRUE(CheckPrecertificateVerification(embedded_sct_chain_));
+  EXPECT_EQ(old_sct_count + 1, NumConnectionsWithSingleSCT());
+}
+
+TEST_F(MultiLogCTVerifierTest, CountsSingleEmbeddedSCTInOriginsHistogram) {
+  int old_embedded_count = NumEmbeddedSCTsInHistogram();
+  ASSERT_TRUE(CheckPrecertificateVerification(embedded_sct_chain_));
+  EXPECT_EQ(old_embedded_count + 1, NumEmbeddedSCTsInHistogram());
+}
+
+TEST_F(MultiLogCTVerifierTest, CountsZeroSCTsCorrectly) {
+  int connections_without_scts = GetValueFromHistogram(kSCTCountHistogram, 0);
+  EXPECT_FALSE(VerifySinglePrecertificateChain(chain_));
+  ASSERT_EQ(connections_without_scts + 1,
+            GetValueFromHistogram(kSCTCountHistogram, 0));
 }
 
 }  // namespace
diff --git a/net/cert/sct_status_flags.h b/net/cert/sct_status_flags.h
index 1bcb422..123c25c89 100644
--- a/net/cert/sct_status_flags.h
+++ b/net/cert/sct_status_flags.h
@@ -10,9 +10,12 @@ namespace net {
 namespace ct {
 
 // The possible verification statuses for a SignedCertificateTimestamp.
+// Note: The numeric values are used within histograms and should not change
+// or be re-assigned.
 enum SCTVerifyStatus {
   // Not a real status, this just prevents a default int value from being
   // mis-interpreseted as a valid status.
+  // Also used to count SCTs that cannot be decoded in the histogram.
   SCT_STATUS_NONE = 0,
 
   // The SCT is from an unknown log, so we cannot verify its signature.
@@ -23,6 +26,9 @@ enum SCTVerifyStatus {
 
   // The SCT is from a known log, and the signature is valid.
   SCT_STATUS_OK = 3,
+
+  // Used to bound the enum values.
+  SCT_STATUS_MAX,
 };
 
 }  // namespace ct
diff --git a/net/cert/signed_certificate_timestamp.h b/net/cert/signed_certificate_timestamp.h
index f065a94..96268e4 100644
--- a/net/cert/signed_certificate_timestamp.h
+++ b/net/cert/signed_certificate_timestamp.h
@@ -88,10 +88,13 @@ struct NET_EXPORT SignedCertificateTimestamp
   };
 
   // Source of the SCT - supplementary, not defined in CT RFC.
+  // Note: The numeric values are used within histograms and should not change
+  // or be re-assigned.
   enum Origin {
     SCT_EMBEDDED = 0,
     SCT_FROM_TLS_EXTENSION = 1,
     SCT_FROM_OCSP_RESPONSE = 2,
+    SCT_ORIGIN_MAX,
   };
 
   SignedCertificateTimestamp();