crypto: disable NSS AES-NI support when AVX is disabled by OS.

When running under Xen, or with certain kernel configurations, it's possible
for the CPU to support AVX but for the operating system not to have configured
it. In this case, CPUID indicates that AVX support exists and NSS will try to
use it for AES-GCM. However, the first AVX instruction will cause an illegal
instruction exception.

This change works around the problem by disabling AES-NI support when AVX
support exists but is not supported by the OS. Sadly this also means that plain
AES instructions are also disabled in this case, but that's better than
crashing.

https://bugzilla.mozilla.org/show_bug.cgi?id=940794

BUG=320524

Review URL: https://codereview.chromium.org/79283002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@236794 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 18 insertions, 0 deletions

diff --git a/crypto/nss_util.cc b/crypto/nss_util.cc
index 87551a8..3b454d0 100644
--- a/crypto/nss_util.cc
+++ b/crypto/nss_util.cc
@@ -23,6 +23,7 @@
 
 #include <vector>
 
+#include "base/cpu.h"
 #include "base/debug/alias.h"
 #include "base/debug/stack_trace.h"
 #include "base/environment.h"
@@ -415,6 +416,8 @@ class NSSInitSingleton {
     // other threads from accessing until the constructor is done.
     thread_checker_.DetachFromThread();
 
+    DisableAESNIIfNeeded();
+
     EnsureNSPRInit();
 
     // We *must* have NSS >= 3.14.3.
@@ -607,6 +610,21 @@ class NSSInitSingleton {
     return db_slot;
   }
 
+  static void DisableAESNIIfNeeded() {
+    if (NSS_VersionCheck("3.15") && !NSS_VersionCheck("3.15.4")) {
+      // Some versions of NSS have a bug that causes AVX instructions to be
+      // used without testing whether XSAVE is enabled by the operating system.
+      // In order to work around this, we disable AES-NI in NSS when we find
+      // that |has_avx()| is false (which includes the XSAVE test). See
+      // https://bugzilla.mozilla.org/show_bug.cgi?id=940794
+      base::CPU cpu;
+
+      if (cpu.has_avx_hardware() && !cpu.has_avx()) {
+        base::Environment::Create()->SetVar("NSS_DISABLE_HW_AES", "1");
+      }
+    }
+  }
+
   // If this is set to true NSS is forced to be initialized without a DB.
   static bool force_nodb_init_;