crypto: disable NSS AES-NI support when AVX is disabled by OS.

When running under Xen, or with certain kernel configurations, it's possible
for the CPU to support AVX but for the operating system not to have configured
it. In this case, CPUID indicates that AVX support exists and NSS will try to
use it for AES-GCM. However, the first AVX instruction will cause an illegal
instruction exception.

This change works around the problem by disabling AES-NI support when AVX
support exists but is not supported by the OS. Sadly this also means that plain
AES instructions are also disabled in this case, but that's better than
crashing.

https://bugzilla.mozilla.org/show_bug.cgi?id=940794

BUG=320524

Review URL: https://codereview.chromium.org/79283002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@236794 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 58 insertions, 23 deletions

diff --git a/base/cpu.cc b/base/cpu.cc
index 78064e2..dec98bd 100644
--- a/base/cpu.cc
+++ b/base/cpu.cc
@@ -8,11 +8,13 @@
 
 #include <algorithm>
 
+#include "base/basictypes.h"
 #include "build/build_config.h"
 
 #if defined(ARCH_CPU_X86_FAMILY)
 #if defined(_MSC_VER)
 #include <intrin.h>
+#include <immintrin.h>  // For _xgetbv()
 #endif
 #endif
 
@@ -33,11 +35,15 @@ CPU::CPU()
     has_ssse3_(false),
     has_sse41_(false),
     has_sse42_(false),
+    has_avx_(false),
+    has_avx_hardware_(false),
     has_non_stop_time_stamp_counter_(false),
     cpu_vendor_("unknown") {
   Initialize();
 }
 
+namespace {
+
 #if defined(ARCH_CPU_X86_FAMILY)
 #ifndef _MSC_VER
 
@@ -53,16 +59,6 @@ void __cpuid(int cpu_info[4], int info_type) {
   );
 }
 
-void __cpuidex(int cpu_info[4], int info_type, int info_index) {
-  __asm__ volatile (
-    "mov %%ebx, %%edi\n"
-    "cpuid\n"
-    "xchg %%edi, %%ebx\n"
-    : "=a"(cpu_info[0]), "=D"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])
-    : "a"(info_type), "c"(info_index)
-  );
-}
-
 #else
 
 void __cpuid(int cpu_info[4], int info_type) {
@@ -73,18 +69,22 @@ void __cpuid(int cpu_info[4], int info_type) {
   );
 }
 
-void __cpuidex(int cpu_info[4], int info_type, int info_index) {
-  __asm__ volatile (
-    "cpuid \n\t"
-    : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3])
-    : "a"(info_type), "c"(info_index)
-  );
+#endif
+
+// _xgetbv returns the value of an Intel Extended Control Register (XCR).
+// Currently only XCR0 is defined by Intel so |xcr| should always be zero.
+uint64 _xgetbv(uint32 xcr) {
+  uint32 eax, edx;
+
+  __asm__ volatile ("xgetbv" : "=a" (eax), "=d" (edx) : "c" (xcr));
+  return (static_cast<uint64>(edx) << 32) | eax;
 }
 
-#endif
-#endif  // _MSC_VER
+#endif  // !_MSC_VER
 #endif  // ARCH_CPU_X86_FAMILY
 
+}  // anonymous namespace
+
 void CPU::Initialize() {
 #if defined(ARCH_CPU_X86_FAMILY)
   int cpu_info[4] = {-1};
@@ -113,14 +113,24 @@ void CPU::Initialize() {
     type_ = (cpu_info[0] >> 12) & 0x3;
     ext_model_ = (cpu_info[0] >> 16) & 0xf;
     ext_family_ = (cpu_info[0] >> 20) & 0xff;
-    has_mmx_ = (cpu_info[3] & 0x00800000) != 0;
-    has_sse_ = (cpu_info[3] & 0x02000000) != 0;
-    has_sse2_ = (cpu_info[3] & 0x04000000) != 0;
-    has_sse3_ = (cpu_info[2] & 0x00000001) != 0;
+    has_mmx_ =   (cpu_info[3] & 0x00800000) != 0;
+    has_sse_ =   (cpu_info[3] & 0x02000000) != 0;
+    has_sse2_ =  (cpu_info[3] & 0x04000000) != 0;
+    has_sse3_ =  (cpu_info[2] & 0x00000001) != 0;
     has_ssse3_ = (cpu_info[2] & 0x00000200) != 0;
     has_sse41_ = (cpu_info[2] & 0x00080000) != 0;
     has_sse42_ = (cpu_info[2] & 0x00100000) != 0;
-    has_avx_ = (cpu_info[2] & 0x10000000) != 0;
+    has_avx_hardware_ =
+                 (cpu_info[2] & 0x10000000) != 0;
+    // AVX instructions will generate an illegal instruction exception unless
+    //   a) they are supported by the CPU,
+    //   b) XSAVE is supported by the CPU and
+    //   c) XSAVE is enabled by the kernel.
+    // See http://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled
+    has_avx_ =
+        has_avx_hardware_ &&
+        (cpu_info[2] & 0x08000000) != 0 /* OSXSAVE */ &&
+        (_xgetbv(0) & 6) == 6 /* XSAVE enabled by kernel */;
   }
 
   // Get the brand string of the cpu.
diff --git a/base/cpu.h b/base/cpu.h
index 509763e..aec24d2 100644
--- a/base/cpu.h
+++ b/base/cpu.h
@@ -46,6 +46,12 @@ class BASE_EXPORT CPU {
   bool has_sse41() const { return has_sse41_; }
   bool has_sse42() const { return has_sse42_; }
   bool has_avx() const { return has_avx_; }
+  // has_avx_hardware returns true when AVX is present in the CPU. This might
+  // differ from the value of |has_avx()| because |has_avx()| also tests for
+  // operating system support needed to actually call AVX instuctions.
+  // Note: you should never need to call this function. It was added in order
+  // to workaround a bug in NSS but |has_avx()| is what you want.
+  bool has_avx_hardware() const { return has_avx_hardware_; }
   bool has_non_stop_time_stamp_counter() const {
     return has_non_stop_time_stamp_counter_;
   }
@@ -71,6 +77,7 @@ class BASE_EXPORT CPU {
   bool has_sse41_;
   bool has_sse42_;
   bool has_avx_;
+  bool has_avx_hardware_;
   bool has_non_stop_time_stamp_counter_;
   std::string cpu_vendor_;
   std::string cpu_brand_;
diff --git a/crypto/nss_util.cc b/crypto/nss_util.cc
index 87551a8..3b454d0 100644
--- a/crypto/nss_util.cc
+++ b/crypto/nss_util.cc
@@ -23,6 +23,7 @@
 
 #include <vector>
 
+#include "base/cpu.h"
 #include "base/debug/alias.h"
 #include "base/debug/stack_trace.h"
 #include "base/environment.h"
@@ -415,6 +416,8 @@ class NSSInitSingleton {
     // other threads from accessing until the constructor is done.
     thread_checker_.DetachFromThread();
 
+    DisableAESNIIfNeeded();
+
     EnsureNSPRInit();
 
     // We *must* have NSS >= 3.14.3.
@@ -607,6 +610,21 @@ class NSSInitSingleton {
     return db_slot;
   }
 
+  static void DisableAESNIIfNeeded() {
+    if (NSS_VersionCheck("3.15") && !NSS_VersionCheck("3.15.4")) {
+      // Some versions of NSS have a bug that causes AVX instructions to be
+      // used without testing whether XSAVE is enabled by the operating system.
+      // In order to work around this, we disable AES-NI in NSS when we find
+      // that |has_avx()| is false (which includes the XSAVE test). See
+      // https://bugzilla.mozilla.org/show_bug.cgi?id=940794
+      base::CPU cpu;
+
+      if (cpu.has_avx_hardware() && !cpu.has_avx()) {
+        base::Environment::Create()->SetVar("NSS_DISABLE_HW_AES", "1");
+      }
+    }
+  }
+
   // If this is set to true NSS is forced to be initialized without a DB.
   static bool force_nodb_init_;