Implement PNaCl crash throttling

Keep a short vector of crash times.  When the vector is full, check that the
crashes occurred within a specified interval.  If so, cause PNaCl loads to fail.

BUG= https://code.google.com/p/nativeclient/issues/detail?id=359
R=dmichael@chromium.org, jln@chromium.org

Review URL: https://codereview.chromium.org/22459002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@216742 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 137 insertions, 62 deletions

diff --git a/ppapi/native_client/src/trusted/plugin/module_ppapi.cc b/ppapi/native_client/src/trusted/plugin/module_ppapi.cc
index 90fe0f1..4204ed4 100644
--- a/ppapi/native_client/src/trusted/plugin/module_ppapi.cc
+++ b/ppapi/native_client/src/trusted/plugin/module_ppapi.cc
@@ -8,81 +8,95 @@
 #include "native_client/src/shared/platform/nacl_time.h"
 #include "native_client/src/trusted/desc/nrd_all_modules.h"
 
-#include "ppapi/c/private/ppb_nacl_private.h"
-#include "ppapi/cpp/module.h"
 
+#include "ppapi/native_client/src/trusted/plugin/module_ppapi.h"
 #include "ppapi/native_client/src/trusted/plugin/nacl_entry_points.h"
 #include "ppapi/native_client/src/trusted/plugin/plugin.h"
 
 namespace plugin {
 
-class ModulePpapi : public pp::Module {
- public:
-  ModulePpapi() : pp::Module(),
-                  init_was_successful_(false),
-                  private_interface_(NULL) {
-    MODULE_PRINTF(("ModulePpapi::ModulePpapi (this=%p)\n",
-                   static_cast<void*>(this)));
-  }
+ModulePpapi::ModulePpapi() : pp::Module(),
+                             init_was_successful_(false),
+                             private_interface_(NULL) {
+  MODULE_PRINTF(("ModulePpapi::ModulePpapi (this=%p)\n",
+                 static_cast<void*>(this)));
+}
 
-  virtual ~ModulePpapi() {
-    if (init_was_successful_) {
-      NaClSrpcModuleFini();
-      NaClNrdAllModulesFini();
-    }
-    MODULE_PRINTF(("ModulePpapi::~ModulePpapi (this=%p)\n",
-                   static_cast<void*>(this)));
+ModulePpapi::~ModulePpapi() {
+  if (init_was_successful_) {
+    NaClSrpcModuleFini();
+    NaClNrdAllModulesFini();
   }
+  MODULE_PRINTF(("ModulePpapi::~ModulePpapi (this=%p)\n",
+                 static_cast<void*>(this)));
+}
 
-  virtual bool Init() {
-    // Ask the browser for an interface which provides missing functions
-    private_interface_ = reinterpret_cast<const PPB_NaCl_Private*>(
-        GetBrowserInterface(PPB_NACL_PRIVATE_INTERFACE));
+bool ModulePpapi::Init() {
+  // Ask the browser for an interface which provides missing functions
+  private_interface_ = reinterpret_cast<const PPB_NaCl_Private*>(
+      GetBrowserInterface(PPB_NACL_PRIVATE_INTERFACE));
 
-    if (NULL == private_interface_) {
-      MODULE_PRINTF(("ModulePpapi::Init failed: "
-                     "GetBrowserInterface returned NULL\n"));
-      return false;
-    }
+  if (NULL == private_interface_) {
+    MODULE_PRINTF(("ModulePpapi::Init failed: "
+                   "GetBrowserInterface returned NULL\n"));
+    return false;
+  }
 
-    launch_nacl_process = reinterpret_cast<LaunchNaClProcessFunc>(
-        private_interface_->LaunchSelLdr);
+  launch_nacl_process = reinterpret_cast<LaunchNaClProcessFunc>(
+      private_interface_->LaunchSelLdr);
 
 #if NACL_LINUX || NACL_OSX
-    // Note that currently we do not need random numbers inside the
-    // NaCl trusted plugin on Unix, but NaClSecureRngModuleInit() is
-    // strict and will raise a fatal error unless we provide it with a
-    // /dev/urandom FD beforehand.
-    NaClSecureRngModuleSetUrandomFd(dup(private_interface_->UrandomFD()));
+  // Note that currently we do not need random numbers inside the
+  // NaCl trusted plugin on Unix, but NaClSecureRngModuleInit() is
+  // strict and will raise a fatal error unless we provide it with a
+  // /dev/urandom FD beforehand.
+  NaClSecureRngModuleSetUrandomFd(dup(private_interface_->UrandomFD()));
 #endif
 
-    // In the plugin, we don't need high resolution time of day.
-    NaClAllowLowResolutionTimeOfDay();
-    NaClNrdAllModulesInit();
-    NaClSrpcModuleInit();
+  // In the plugin, we don't need high resolution time of day.
+  NaClAllowLowResolutionTimeOfDay();
+  NaClNrdAllModulesInit();
+  NaClSrpcModuleInit();
 
 #if NACL_WINDOWS
-    NaClSetBrokerDuplicateHandleFunc(private_interface_->BrokerDuplicateHandle);
+  NaClSetBrokerDuplicateHandleFunc(private_interface_->BrokerDuplicateHandle);
 #endif
 
-    init_was_successful_ = true;
-    return true;
-  }
+  init_was_successful_ = true;
+  return true;
+}
 
-  virtual pp::Instance* CreateInstance(PP_Instance pp_instance) {
-    MODULE_PRINTF(("ModulePpapi::CreateInstance (pp_instance=%" NACL_PRId32
-                   ")\n",
-                   pp_instance));
-    Plugin* plugin = Plugin::New(pp_instance);
-    MODULE_PRINTF(("ModulePpapi::CreateInstance (return %p)\n",
-                   static_cast<void* >(plugin)));
-    return plugin;
+pp::Instance* ModulePpapi::CreateInstance(PP_Instance pp_instance) {
+  MODULE_PRINTF(("ModulePpapi::CreateInstance (pp_instance=%" NACL_PRId32
+                 ")\n",
+                 pp_instance));
+  Plugin* plugin = Plugin::New(pp_instance);
+  MODULE_PRINTF(("ModulePpapi::CreateInstance (return %p)\n",
+                 static_cast<void* >(plugin)));
+  return plugin;
+}
+
+const uint64_t kMaxCrashesPerInterval = 3;
+const uint64_t kCrashesIntervalInSeconds = 120;
+
+void ModulePpapi::RegisterPluginCrash() {
+  PLUGIN_PRINTF(("ModulePpapi::RegisterPluginCrash ()\n"));
+  if (crash_times_.size() == kMaxCrashesPerInterval) {
+    crash_times_.pop_front();
   }
+  int64_t time = NaClGetTimeOfDayMicroseconds();
+  crash_times_.push_back(time);
+}
 
- private:
-  bool init_was_successful_;
-  const PPB_NaCl_Private* private_interface_;
-};
+bool ModulePpapi::IsPluginUnstable() {
+  PLUGIN_PRINTF(("ModulePpapi::IsPluginUnstable ()\n"));
+  if (crash_times_.size() != kMaxCrashesPerInterval) {
+    return false;
+  }
+  int64_t now = NaClGetTimeOfDayMicroseconds();
+  int64_t delta = now - crash_times_.front();
+  return delta / (1000.0 * 1000.0) <= kCrashesIntervalInSeconds;
+}
 
 }  // namespace plugin
 
diff --git a/ppapi/native_client/src/trusted/plugin/module_ppapi.h b/ppapi/native_client/src/trusted/plugin/module_ppapi.h
new file mode 100644
index 0000000..b2e7564
--- /dev/null
+++ b/ppapi/native_client/src/trusted/plugin/module_ppapi.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2013 The Chromium Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include <deque>
+
+#include "ppapi/c/private/ppb_nacl_private.h"
+#include "ppapi/cpp/module.h"
+
+namespace plugin {
+
+class ModulePpapi : public pp::Module {
+ public:
+  ModulePpapi();
+
+  virtual ~ModulePpapi();
+
+  virtual bool Init();
+
+  virtual pp::Instance* CreateInstance(PP_Instance pp_instance);
+
+  // NaCl crash throttling.  If RegisterPluginCrash is called too many times
+  // within a time period, IsPluginUnstable reports true.  As long as
+  // IsPluginUnstable returns true, NaCl modules will fail to load.
+  void RegisterPluginCrash();
+  bool IsPluginUnstable();
+
+ private:
+  bool init_was_successful_;
+  const PPB_NaCl_Private* private_interface_;
+
+  // Crash throttling support.
+  std::deque<int64_t> crash_times_;
+};
+
+}  // namespace plugin
+
+
+namespace pp {
+
+Module* CreateModule();
+
+}  // namespace pp
diff --git a/ppapi/native_client/src/trusted/plugin/plugin.cc b/ppapi/native_client/src/trusted/plugin/plugin.cc
index 5cbb9d7..a4833ee 100644
--- a/ppapi/native_client/src/trusted/plugin/plugin.cc
+++ b/ppapi/native_client/src/trusted/plugin/plugin.cc
@@ -55,6 +55,7 @@
 
 #include "ppapi/native_client/src/trusted/plugin/file_utils.h"
 #include "ppapi/native_client/src/trusted/plugin/json_manifest.h"
+#include "ppapi/native_client/src/trusted/plugin/module_ppapi.h"
 #include "ppapi/native_client/src/trusted/plugin/nacl_entry_points.h"
 #include "ppapi/native_client/src/trusted/plugin/nacl_subprocess.h"
 #include "ppapi/native_client/src/trusted/plugin/nexe_arch.h"
@@ -1009,6 +1010,10 @@ void Plugin::NexeDidCrash(int32_t pp_error) {
   // invocation will just be a no-op, since all the crash log will
   // have been received and we'll just get an EOF indication.
   CopyCrashLogToJsConsole();
+
+  // Remember the nexe crash time, which helps determine the need to throttle.
+  ModulePpapi* module_ppapi = static_cast<ModulePpapi*>(pp::Module::Get());
+  module_ppapi->RegisterPluginCrash();
 }
 
 void Plugin::BitcodeDidTranslate(int32_t pp_error) {
@@ -1204,15 +1209,25 @@ void Plugin::ProcessNaClManifest(const nacl::string& manifest_json) {
     EnqueueProgressEvent(kProgressEventProgress);
     if (pnacl_options.translate()) {
       if (this->nacl_interface()->IsPnaclEnabled()) {
-        pp::CompletionCallback translate_callback =
-            callback_factory_.NewCallback(&Plugin::BitcodeDidTranslate);
-        // Will always call the callback on success or failure.
-        pnacl_coordinator_.reset(
-            PnaclCoordinator::BitcodeToNative(this,
-                                              program_url,
-                                              pnacl_options,
-                                              translate_callback));
-        return;
+        // Check whether PNaCl has been crashing "frequently".  If so, report
+        // a load error.
+        ModulePpapi* module_ppapi =
+            static_cast<ModulePpapi*>(pp::Module::Get());
+        if (module_ppapi->IsPluginUnstable()) {
+          error_info.SetReport(ERROR_PNACL_CRASH_THROTTLED,
+                               "PNaCl has been temporarily disabled because too"
+                               " many crashes have been observed.");
+        } else {
+          pp::CompletionCallback translate_callback =
+              callback_factory_.NewCallback(&Plugin::BitcodeDidTranslate);
+          // Will always call the callback on success or failure.
+          pnacl_coordinator_.reset(
+              PnaclCoordinator::BitcodeToNative(this,
+                                                program_url,
+                                                pnacl_options,
+                                                translate_callback));
+          return;
+        }
       } else {
         error_info.SetReport(ERROR_PNACL_NOT_ENABLED,
                              "PNaCl has been disabled (e.g., by setting "
diff --git a/ppapi/native_client/src/trusted/plugin/plugin_error.h b/ppapi/native_client/src/trusted/plugin/plugin_error.h
index b5eb4d8..ac3fbf4 100644
--- a/ppapi/native_client/src/trusted/plugin/plugin_error.h
+++ b/ppapi/native_client/src/trusted/plugin/plugin_error.h
@@ -96,6 +96,7 @@ enum PluginErrorCode {
   ERROR_PNACL_NOT_ENABLED = 68,
   ERROR_MANIFEST_NOACCESS_URL = 69,
   ERROR_NEXE_NOACCESS_URL = 70,
+  ERROR_PNACL_CRASH_THROTTLED = 71,
   // If you add a code, read the enum comment above on how to update histograms.
   ERROR_MAX
 };