Merge 32395 - For Linux, override malloc and friends so that we can detect and then stop on out of memory.

BUG=27222
TEST=new base unittests

Review URL: http://codereview.chromium.org/391044


git-svn-id: svn://svn.chromium.org/chrome/branches/249/src@32847 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 193 insertions, 1 deletions

diff --git a/base/process_util_linux.cc b/base/process_util_linux.cc
index 10ae0b1..94942d6 100644
--- a/base/process_util_linux.cc
+++ b/base/process_util_linux.cc
@@ -6,6 +6,7 @@
 
 #include <ctype.h>
 #include <dirent.h>
+#include <dlfcn.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/time.h>
@@ -497,8 +498,105 @@ size_t GetSystemCommitCharge() {
   return result_in_kb;
 }
 
+namespace {
+
+void OnNoMemorySize(size_t size) {
+  if (size != 0)
+    CHECK(false) << "Out of memory, size = " << size;
+  CHECK(false) << "Out of memory.";
+}
+
+void OnNoMemory() {
+  OnNoMemorySize(0);
+}
+
+}  // namespace
+
+extern "C" {
+
+#if defined(LINUX_USE_TCMALLOC)
+
+int tc_set_new_mode(int mode);
+
+#else  // defined(LINUX_USE_TCMALLOC)
+
+typedef void* (*malloc_type)(size_t size);
+typedef void* (*valloc_type)(size_t size);
+typedef void* (*pvalloc_type)(size_t size);
+
+typedef void* (*calloc_type)(size_t nmemb, size_t size);
+typedef void* (*realloc_type)(void *ptr, size_t size);
+typedef void* (*memalign_type)(size_t boundary, size_t size);
+
+typedef int (*posix_memalign_type)(void **memptr, size_t alignment,
+                                   size_t size);
+
+// Override the __libc_FOO name too.
+#define DIE_ON_OOM_1(function_name) \
+  _DIE_ON_OOM_1(function_name##_type, function_name) \
+  _DIE_ON_OOM_1(function_name##_type, __libc_##function_name)
+
+#define DIE_ON_OOM_2(function_name, arg1_type) \
+  _DIE_ON_OOM_2(function_name##_type, function_name, arg1_type) \
+  _DIE_ON_OOM_2(function_name##_type, __libc_##function_name, arg1_type)
+
+// posix_memalign doesn't have a __libc_ variant.
+#define DIE_ON_OOM_3INT(function_name) \
+  _DIE_ON_OOM_3INT(function_name##_type, function_name)
+
+#define _DIE_ON_OOM_1(function_type, function_name) \
+  void* function_name(size_t size) { \
+    static function_type original_function = \
+        reinterpret_cast<function_type>(dlsym(RTLD_NEXT, #function_name)); \
+    void* ret = original_function(size); \
+    if (ret == NULL && size != 0) \
+      OnNoMemorySize(size); \
+    return ret; \
+  }
+
+#define _DIE_ON_OOM_2(function_type, function_name, arg1_type) \
+  void* function_name(arg1_type arg1, size_t size) { \
+    static function_type original_function = \
+        reinterpret_cast<function_type>(dlsym(RTLD_NEXT, #function_name)); \
+    void* ret = original_function(arg1, size); \
+    if (ret == NULL && size != 0) \
+      OnNoMemorySize(size); \
+    return ret; \
+  }
+
+#define _DIE_ON_OOM_3INT(function_type, function_name) \
+  int function_name(void** ptr, size_t alignment, size_t size) { \
+    static function_type original_function = \
+        reinterpret_cast<function_type>(dlsym(RTLD_NEXT, #function_name)); \
+    int ret = original_function(ptr, alignment, size); \
+    if (ret == ENOMEM) \
+      OnNoMemorySize(size); \
+    return ret; \
+  }
+
+DIE_ON_OOM_1(malloc)
+DIE_ON_OOM_1(valloc)
+DIE_ON_OOM_1(pvalloc)
+
+DIE_ON_OOM_2(calloc, size_t)
+DIE_ON_OOM_2(realloc, void*)
+DIE_ON_OOM_2(memalign, size_t)
+
+DIE_ON_OOM_3INT(posix_memalign)
+
+#endif  // defined(LINUX_USE_TCMALLOC)
+
+}  // extern C
+
 void EnableTerminationOnOutOfMemory() {
-  // http://crbug.com/27222
+  // Set the new-out of memory handler.
+  std::set_new_handler(&OnNoMemory);
+  // If we're using glibc's allocator, the above functions will override
+  // malloc and friends and make them die on out of memory.
+#if defined(LINUX_USE_TCMALLOC)
+  // For tcmalloc, we just need to tell it to behave like new.
+  tc_set_new_mode(1);
+#endif
   NOTIMPLEMENTED();
 }
 
diff --git a/base/process_util_unittest.cc b/base/process_util_unittest.cc
index 961ac7e..4169dc6 100644
--- a/base/process_util_unittest.cc
+++ b/base/process_util_unittest.cc
@@ -4,6 +4,8 @@
 
 #define _CRT_SECURE_NO_WARNINGS
 
+#include <limits>
+
 #include "base/command_line.h"
 #include "base/eintr_wrapper.h"
 #include "base/file_path.h"
@@ -15,6 +17,7 @@
 
 #if defined(OS_LINUX)
 #include <dlfcn.h>
+#include <malloc.h>
 #endif
 #if defined(OS_POSIX)
 #include <fcntl.h>
@@ -361,4 +364,95 @@ TEST_F(ProcessUtilTest, ParseProcStatCPU) {
 
 #endif  // defined(OS_POSIX)
 
+#if defined(OS_LINUX)
+// TODO(vandebo) make this work on Windows and Mac too.
+
+class OutOfMemoryTest : public testing::Test {
+ public:
+  OutOfMemoryTest()
+      : value_(NULL),
+        // Make test size as large as possible minus a few pages so
+        // that alignment or other rounding doesn't make it wrap.
+        test_size_(std::numeric_limits<std::size_t>::max() - 8192) {
+  }
+
+  virtual void SetUp() {
+    // Must call EnableTerminationOnOutOfMemory() because that is called from
+    // chrome's main function and therefore hasn't been called yet.
+    EnableTerminationOnOutOfMemory();
+  }
+
+  void* value_;
+  size_t test_size_;
+};
+
+TEST_F(OutOfMemoryTest, New) {
+  ASSERT_DEATH(value_ = new char[test_size_], "");
+}
+
+TEST_F(OutOfMemoryTest, Malloc) {
+  ASSERT_DEATH(value_ = malloc(test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, Realloc) {
+  ASSERT_DEATH(value_ = realloc(NULL, test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, Calloc) {
+  ASSERT_DEATH(value_ = calloc(1024, test_size_ / 1024L), "");
+}
+
+TEST_F(OutOfMemoryTest, Valloc) {
+  ASSERT_DEATH(value_ = valloc(test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, Pvalloc) {
+  ASSERT_DEATH(value_ = pvalloc(test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, Memalign) {
+  ASSERT_DEATH(value_ = memalign(4, test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, Posix_memalign) {
+  ASSERT_DEATH(posix_memalign(&value_, 8, test_size_), "");
+}
+
+extern "C" {
+
+void* __libc_malloc(size_t size);
+void* __libc_realloc(void* ptr, size_t size);
+void* __libc_calloc(size_t nmemb, size_t size);
+void* __libc_valloc(size_t size);
+void* __libc_pvalloc(size_t size);
+void* __libc_memalign(size_t alignment, size_t size);
+
+}  // extern "C"
+
+TEST_F(OutOfMemoryTest, __libc_malloc) {
+  ASSERT_DEATH(value_ = __libc_malloc(test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, __libc_realloc) {
+  ASSERT_DEATH(value_ = __libc_realloc(NULL, test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, __libc_calloc) {
+  ASSERT_DEATH(value_ = __libc_calloc(1024, test_size_ / 1024L), "");
+}
+
+TEST_F(OutOfMemoryTest, __libc_valloc) {
+  ASSERT_DEATH(value_ = __libc_valloc(test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, __libc_pvalloc) {
+  ASSERT_DEATH(value_ = __libc_pvalloc(test_size_), "");
+}
+
+TEST_F(OutOfMemoryTest, __libc_memalign) {
+  ASSERT_DEATH(value_ = __libc_memalign(4, test_size_), "");
+}
+
+#endif  // defined(OS_LINUX)
+
 }  // namespace base