Make possible to check memory allocations inside chromium

    
This patch implements UncheckedMalloc and UncheckedCalloc for Linux.
Previously it was only possible on Mac. The motivation is to let callers handle
OOM gracefully instead of aborting.

When tcmalloc is used this is implemented via a weak symbol that is overridden
by tcmalloc. This way we get around the problem that neither base cannot depend
on tcmalloc and vica versa. Unfortunately weak symbols are not supported on Windows.
To make that work on Windows one will have to do some build system craft that is similar
of what we do to replace the system malloc with tcmalloc.

This cl does not try to solve the more controversial problem of disallowing
the OOM handler for third party libraries under special circumstances.

BUG=73751

Review URL: https://codereview.chromium.org/55333002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@258681 0039d316-1c4b-4281-b951-d872f2087c98

10 files changed, 195 insertions, 25 deletions

diff --git a/base/base.gypi b/base/base.gypi
index 7f2d58f..7f4cb1e 100644
--- a/base/base.gypi
+++ b/base/base.gypi
@@ -454,6 +454,7 @@
           'process/launch_posix.cc',
           'process/launch_win.cc',
           'process/memory.h',
+          'process/memory.cc',
           'process/memory_linux.cc',
           'process/memory_mac.mm',
           'process/memory_win.cc',
diff --git a/base/process/memory.cc b/base/process/memory.cc
new file mode 100644
index 0000000..1dbc363
--- /dev/null
+++ b/base/process/memory.cc
@@ -0,0 +1,30 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/process/memory.h"
+
+namespace base {
+
+// Defined in memory_mac.mm for Mac.
+#if !defined(OS_MACOSX)
+
+bool UncheckedCalloc(size_t num_items, size_t size, void** result) {
+  const size_t alloc_size = num_items * size;
+
+  // Overflow check
+  if (size && ((alloc_size / size) != num_items)) {
+    *result = NULL;
+    return false;
+  }
+
+  if (!UncheckedMalloc(alloc_size, result))
+    return false;
+
+  memset(*result, 0, alloc_size);
+  return true;
+}
+
+#endif
+
+}
diff --git a/base/process/memory.h b/base/process/memory.h
index 8187032..a930be7 100644
--- a/base/process/memory.h
+++ b/base/process/memory.h
@@ -61,17 +61,25 @@ const int kMaxOomScore = 1000;
 BASE_EXPORT bool AdjustOOMScore(ProcessId process, int score);
 #endif
 
+// Special allocator functions for callers that want to check for OOM.
+// These will not abort if the allocation fails even if
+// EnableTerminationOnOutOfMemory has been called.
+// This can be useful for huge and/or unpredictable size memory allocations.
+// Please only use this if you really handle the case when the allocation
+// fails. Doing otherwise would risk security.
+// Return value tells whether the allocation succeeded. If it fails |result| is
+// set to NULL, otherwise it holds the memory address.
+BASE_EXPORT WARN_UNUSED_RESULT bool UncheckedMalloc(size_t size,
+                                                    void** result);
+BASE_EXPORT WARN_UNUSED_RESULT bool UncheckedCalloc(size_t num_items,
+                                                    size_t size,
+                                                    void** result);
+
+// TODO(b.kelemen): make Skia use the new interface and remove these.
 #if defined(OS_MACOSX)
-// Very large images or svg canvases can cause huge mallocs.  Skia
-// does tricks on tcmalloc-based systems to allow malloc to fail with
-// a NULL rather than hit the oom crasher.  This replicates that for
-// OSX.
-//
-// IF YOU USE THIS WITHOUT CONSULTING YOUR FRIENDLY OSX DEVELOPER,
-// YOUR CODE IS LIKELY TO BE REVERTED.  THANK YOU.
 BASE_EXPORT void* UncheckedMalloc(size_t size);
 BASE_EXPORT void* UncheckedCalloc(size_t num_items, size_t size);
-#endif  // defined(OS_MACOSX)
+#endif
 
 }  // namespace base
 
diff --git a/base/process/memory_linux.cc b/base/process/memory_linux.cc
index a40c270..e1d452b 100644
--- a/base/process/memory_linux.cc
+++ b/base/process/memory_linux.cc
@@ -12,6 +12,22 @@
 #include "base/process/internal_linux.h"
 #include "base/strings/string_number_conversions.h"
 
+#if defined(USE_TCMALLOC)
+// Used by UncheckedMalloc. If tcmalloc is linked to the executable
+// this will be replaced by a strong symbol that actually implement
+// the semantics and don't call new handler in case the allocation fails.
+extern "C" {
+
+__attribute__((weak, visibility("default")))
+void* tc_malloc_skip_new_handler_weak(size_t size);
+
+void* tc_malloc_skip_new_handler_weak(size_t size) {
+  return malloc(size);
+}
+
+}
+#endif
+
 namespace base {
 
 size_t g_oom_size = 0U;
@@ -182,4 +198,17 @@ bool AdjustOOMScore(ProcessId process, int score) {
   return false;
 }
 
+bool UncheckedMalloc(size_t size, void** result) {
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER) || defined(LEAK_SANITIZER) || \
+    (!defined(LIBC_GLIBC) && !defined(USE_TCMALLOC))
+  *result = malloc(size);
+#elif defined(LIBC_GLIBC) && !defined(USE_TCMALLOC)
+  *result = __libc_malloc(size);
+#elif defined(USE_TCMALLOC)
+  *result = tc_malloc_skip_new_handler_weak(size);
+#endif
+  return *result != NULL;
+}
+
 }  // namespace base
diff --git a/base/process/memory_mac.mm b/base/process/memory_mac.mm
index 3e281cd..6cf1297 100644
--- a/base/process/memory_mac.mm
+++ b/base/process/memory_mac.mm
@@ -502,26 +502,42 @@ id oom_killer_allocWithZone(id self, SEL _cmd, NSZone* zone)
 
 }  // namespace
 
-void* UncheckedMalloc(size_t size) {
+bool UncheckedMalloc(size_t size, void** result) {
   if (g_old_malloc) {
 #if ARCH_CPU_32_BITS
     ScopedClearErrno clear_errno;
     ThreadLocalBooleanAutoReset flag(g_unchecked_alloc.Pointer(), true);
 #endif  // ARCH_CPU_32_BITS
-    return g_old_malloc(malloc_default_zone(), size);
+    *result = g_old_malloc(malloc_default_zone(), size);
+  } else {
+    *result = malloc(size);
   }
-  return malloc(size);
+
+  return *result != NULL;
 }
 
-void* UncheckedCalloc(size_t num_items, size_t size) {
+bool UncheckedCalloc(size_t num_items, size_t size, void** result) {
   if (g_old_calloc) {
 #if ARCH_CPU_32_BITS
     ScopedClearErrno clear_errno;
     ThreadLocalBooleanAutoReset flag(g_unchecked_alloc.Pointer(), true);
 #endif  // ARCH_CPU_32_BITS
-    return g_old_calloc(malloc_default_zone(), num_items, size);
+    *result = g_old_calloc(malloc_default_zone(), num_items, size);
+  } else {
+    *result = calloc(num_items, size);
   }
-  return calloc(num_items, size);
+
+  return *result != NULL;
+}
+
+void* UncheckedMalloc(size_t size) {
+  void* address;
+  return UncheckedMalloc(size, &address) ? address : NULL;
+}
+
+void* UncheckedCalloc(size_t num_items, size_t size) {
+  void* address;
+  return UncheckedCalloc(num_items, size, &address) ? address : NULL;
 }
 
 void EnableTerminationOnOutOfMemory() {
diff --git a/base/process/memory_unittest.cc b/base/process/memory_unittest.cc
index 5a1bcdf..f329fda 100644
--- a/base/process/memory_unittest.cc
+++ b/base/process/memory_unittest.cc
@@ -166,14 +166,14 @@ int tc_set_new_mode(int mode);
 }
 #endif  // defined(USE_TCMALLOC)
 
-class OutOfMemoryDeathTest : public testing::Test {
+class OutOfMemoryTest : public testing::Test {
  public:
-  OutOfMemoryDeathTest()
-      : value_(NULL),
-        // Make test size as large as possible minus a few pages so
-        // that alignment or other rounding doesn't make it wrap.
-        test_size_(std::numeric_limits<std::size_t>::max() - 12 * 1024),
-        signed_test_size_(std::numeric_limits<ssize_t>::max()) {
+  OutOfMemoryTest()
+    : value_(NULL),
+    // Make test size as large as possible minus a few pages so
+    // that alignment or other rounding doesn't make it wrap.
+    test_size_(std::numeric_limits<std::size_t>::max() - 12 * 1024),
+    signed_test_size_(std::numeric_limits<ssize_t>::max()) {
   }
 
 #if defined(USE_TCMALLOC)
@@ -186,6 +186,14 @@ class OutOfMemoryDeathTest : public testing::Test {
   }
 #endif  // defined(USE_TCMALLOC)
 
+ protected:
+  void* value_;
+  size_t test_size_;
+  ssize_t signed_test_size_;
+};
+
+class OutOfMemoryDeathTest : public OutOfMemoryTest {
+ public:
   void SetUpInDeathAssert() {
     // Must call EnableTerminationOnOutOfMemory() because that is called from
     // chrome's main function and therefore hasn't been called yet.
@@ -194,10 +202,6 @@ class OutOfMemoryDeathTest : public testing::Test {
     // should be done inside of the ASSERT_DEATH.
     base::EnableTerminationOnOutOfMemory();
   }
-
-  void* value_;
-  size_t test_size_;
-  ssize_t signed_test_size_;
 };
 
 TEST_F(OutOfMemoryDeathTest, New) {
@@ -375,5 +379,53 @@ TEST_F(OutOfMemoryDeathTest, PsychoticallyBigObjCObject) {
 #endif  // !ARCH_CPU_64_BITS
 #endif  // OS_MACOSX
 
+class OutOfMemoryHandledTest : public OutOfMemoryTest {
+ public:
+  static const size_t kSafeMallocSize = 512;
+  static const size_t kSafeCallocSize = 128;
+  static const size_t kSafeCallocItems = 4;
+
+  virtual void SetUp() {
+    OutOfMemoryTest::SetUp();
+
+    // We enable termination on OOM - just as Chrome does at early
+    // initialization - and test that UncheckedMalloc and  UncheckedCalloc
+    // properly by-pass this in order to allow the caller to handle OOM.
+    base::EnableTerminationOnOutOfMemory();
+  }
+};
+
+// TODO(b.kelemen): make UncheckedMalloc and UncheckedCalloc work
+// on Windows as well.
+
+TEST_F(OutOfMemoryHandledTest, UncheckedMalloc) {
+  EXPECT_TRUE(base::UncheckedMalloc(kSafeMallocSize, &value_));
+  EXPECT_TRUE(value_ != NULL);
+  free(value_);
+
+  EXPECT_FALSE(base::UncheckedMalloc(test_size_, &value_));
+  EXPECT_TRUE(value_ == NULL);
+}
+
+TEST_F(OutOfMemoryHandledTest, UncheckedCalloc) {
+  EXPECT_TRUE(base::UncheckedCalloc(1, kSafeMallocSize, &value_));
+  EXPECT_TRUE(value_ != NULL);
+  const char* bytes = static_cast<const char*>(value_);
+  for (size_t i = 0; i < kSafeMallocSize; ++i)
+    EXPECT_EQ(0, bytes[i]);
+  free(value_);
+
+  EXPECT_TRUE(
+      base::UncheckedCalloc(kSafeCallocItems, kSafeCallocSize, &value_));
+  EXPECT_TRUE(value_ != NULL);
+  bytes = static_cast<const char*>(value_);
+  for (size_t i = 0; i < (kSafeCallocItems * kSafeCallocSize); ++i)
+    EXPECT_EQ(0, bytes[i]);
+  free(value_);
+
+  EXPECT_FALSE(base::UncheckedCalloc(1, test_size_, &value_));
+  EXPECT_TRUE(value_ == NULL);
+}
+
 #endif  // !defined(OS_ANDROID) && !defined(OS_OPENBSD) &&
         // !defined(OS_WIN) && !defined(ADDRESS_SANITIZER)
diff --git a/base/process/memory_win.cc b/base/process/memory_win.cc
index c53a1be..668214c 100644
--- a/base/process/memory_win.cc
+++ b/base/process/memory_win.cc
@@ -82,4 +82,15 @@ HMODULE GetModuleFromAddress(void* address) {
   return instance;
 }
 
+// TODO(b.kelemen): implement it with the required semantics. On Linux this is
+// implemented with a weak symbol that is overridden by tcmalloc. This is
+// neccessary because base cannot have a direct dependency on tcmalloc. Since
+// weak symbols are not supported on Windows this will involve some build time
+// magic, much like what is done for libcrt in order to override the allocation
+// functions.
+bool UncheckedMalloc(size_t size, void** result) {
+  *result = malloc(size);
+  return *result != NULL;
+}
+
 }  // namespace base
diff --git a/third_party/tcmalloc/README.chromium b/third_party/tcmalloc/README.chromium
index 92fbf75..c5f40b19 100644
--- a/third_party/tcmalloc/README.chromium
+++ b/third_party/tcmalloc/README.chromium
@@ -93,3 +93,4 @@ Modifications:
   simulated stack from the embedding application.
 - Inserted spaces around PRIx64, SCNx64 and friends, for c++11 compatibility.
 - Fix sprintf formatting warning in MaybeDumpProfileLocked
+- Added tc_malloc_skip_new_handler.
diff --git a/third_party/tcmalloc/chromium/src/debugallocation.cc b/third_party/tcmalloc/chromium/src/debugallocation.cc
index 7b09e41..6a4f286 100644
--- a/third_party/tcmalloc/chromium/src/debugallocation.cc
+++ b/third_party/tcmalloc/chromium/src/debugallocation.cc
@@ -1430,3 +1430,11 @@ extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
   return MallocExtension::instance()->GetAllocatedSize(ptr);
 }
+
+#if defined(OS_LINUX)
+extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) {
+  void* result = DebugAllocate(size, MallocBlock::kMallocType);
+  MallocHook::InvokeNewHook(result, size);
+  return result;
+}
+#endif
diff --git a/third_party/tcmalloc/chromium/src/tcmalloc.cc b/third_party/tcmalloc/chromium/src/tcmalloc.cc
index 118e40a..018bd8e 100644
--- a/third_party/tcmalloc/chromium/src/tcmalloc.cc
+++ b/third_party/tcmalloc/chromium/src/tcmalloc.cc
@@ -1719,8 +1719,22 @@ extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
   return MallocExtension::instance()->GetAllocatedSize(ptr);
 }
 
+#if defined(OS_LINUX)
+extern "C" void* PERFTOOLS_DLL_DECL tc_malloc_skip_new_handler(size_t size) {
+  void* result = do_malloc(size);
+  MallocHook::InvokeNewHook(result, size);
+  return result;
+}
+#endif
+
 #endif  // TCMALLOC_USING_DEBUGALLOCATION
 
+#if defined(OS_LINUX)
+// Alias the weak symbol in chromium to our implementation.
+extern "C" __attribute__((visibility("default"), alias("tc_malloc_skip_new_handler")))
+void* tc_malloc_skip_new_handler_weak(size_t size);
+#endif
+
 // --- Validation implementation with an extra mark ----------------------------
 // We will put a mark at the extreme end of each allocation block.  We make
 // sure that we always allocate enough "extra memory" that we can fit in the